From 1cd40c7bb7ef14d298476cc0a1888c571f8b4884 Mon Sep 17 00:00:00 2001 From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com> Date: Sun, 19 Nov 2023 00:48:55 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E3=80=90=E5=92=96?= =?UTF-8?q?=E5=95=A1=E5=8E=85=E7=9A=84=E4=B8=80=E5=A4=A9=E3=80=91=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=E4=B8=89=E4=B8=AA=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/behavior_lib/act/Make.py | 2 +- robowaiter/llm_client/data/fix_questions.txt | 20 +++- .../VLN_greet_and_order.py | 63 ++++++++--- .../VLN_greet_and_order_1118.py | 102 ++++++++++++++++++ robowaiter/scene/tasks/VLM/VLM_order.py | 8 +- 5 files changed, 173 insertions(+), 22 deletions(-) create mode 100644 robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order_1118.py diff --git a/robowaiter/behavior_lib/act/Make.py b/robowaiter/behavior_lib/act/Make.py index f3e9b38..32dd194 100644 --- a/robowaiter/behavior_lib/act/Make.py +++ b/robowaiter/behavior_lib/act/Make.py @@ -34,7 +34,7 @@ class Make(Act): info["add"] |= {f'On({arg},WaterTable)'} elif arg == cls.valid_args[2]: info["add"] |= {f'On({arg},Bar)'} - info['cost'] = 10 + info['cost'] = 2 return info def _update(self) -> ptree.common.Status: diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt index e4f6a17..55a05c0 100644 --- a/robowaiter/llm_client/data/fix_questions.txt +++ b/robowaiter/llm_client/data/fix_questions.txt @@ -26,10 +26,17 @@ create_sub_task create_sub_task {"goal":"At(Robot,BrightTable1)"} + +嘿，RoboWaiter，过来一下！ +我在来的路上啦，请稍等噢！ +create_sub_task +{"goal":"At(Robot,BrightTable6)"} + 你们这有什么饮料嘛？ 我们咖啡厅提供各种口味咖啡，水，冰红茶，酸奶等饮品，还提供点心蛋糕等甜品。您想点什么？ -来杯卡布奇诺吧，我在靠窗边的高脚桌那。 + +来杯卡布奇诺吧。 ok，请稍等！ create_sub_task {"goal":"On(Coffee,BrightTable6)"} @@ -38,6 +45,17 @@ create_sub_task 麻烦啦！ 没事儿，为您服务是我的荣幸！ +大厅的桌子好啊，快带我去呀！ +好的好的，跟我来，我带你去找找。 +create_sub_task +{"goal":"At(Robot,WaterTable)"} + + +我想来杯水，帮我孩子拿个酸奶吧。 +ok，没问题，请稍等！ +create_sub_task +{"goal":"On(Water,WaterTable),On(Yogurt,WaterTable)"} + 来杯酸奶吧。 好的没问题，请稍等！ create_sub_task diff --git a/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py index ff71316..8b1adcd 100644 --- a/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py +++ b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py @@ -14,33 +14,64 @@ class SceneVLM(Scene): super().__init__(robot) # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) - self.scene_flag = 1 + self.scene_flag = 2 self.st1 = 3 - self.st2 = self.st1 + 30 + self.st2 = 3 + # self.st2 = self.st1 + 30 + self.st3 = 3 + self.st4 = 3 self.new_event_list = [ # 场景1：带小女孩找阳光下的空位 - (3, self.add_walker, (5, 230, 1200)), - (3, self.control_walker, (0, False, 200, 60, 520, 0)), - (10, self.customer_say, (0, "早上好呀，我想找个能晒太阳的地方。")), - (10, self.customer_say, (0,"可以带我过去嘛？")), - (20, self.control_walker, (0, False, 50, 140, 1200, 180)), # 小女孩站在了 BrightTable1 旁边就餐啦 + # (3, self.add_walker, (5, 230, 1200)), + # (3, self.control_walker, (0, False, 200, 60, 520, 0)), + # (10, self.customer_say, (0, "早上好呀，我想找个能晒太阳的地方。")), + # (10, self.customer_say, (0,"可以带我过去嘛？")), + # (20, self.control_walker, (0, False, 50, 140, 1200, 180)), # 小女孩站在了 BrightTable1 旁边就餐啦 # 场景2：有个胖胖男人点单交流并要咖啡，帮他送去角落的桌子 - # (3, self.add_walker, (5, 230, 1200)), # 小女孩 + (3, self.add_walker, (5, 230, 1200)), # 小女孩 # 上述准备 - (self.st2, self.add_walker, (26, 60,-140)), - (self.st2, self.add_walker, (10, -80, -180,-45)), # （-150 -250） - (self.st2, self.control_walker, (1, False, 100, 70, 520, 0)), #3 - (self.st2+15, self.customer_say, (1, "你们这有什么饮料嘛？")), # 10 + (self.st2, self.add_walker, (26, -28, -150, 90)), + (self.st2, self.add_walker, (10, -70, -200, -45)), + (self.st2, self.customer_say, (1, "嘿，RoboWaiter，过来一下！")), + (self.st2+10, self.control_walkers_and_say, ([[[1, False, 100, -18, -200, -90, "你们这有什么饮料嘛？"]]])), # 20 胖胖男到了 BrightTable6 (self.st2+15, self.customer_say, (1, "咖啡有哪些呢？")),# 10 - (self.st2+15+5, self.customer_say, (1,"来杯卡布奇诺吧，我在靠窗边的高脚桌那。")), # 15 - (self.st2+15+5+5, self.control_walkers_and_say, ([[[1, False, 100, -30, -200, -90, "麻烦啦！"]]])), # 20 胖胖男到了 BrightTable6 - - + (self.st2+15+5, self.customer_say, (1,"来杯卡布奇诺吧。")), # 15 + # 场景3：有位女士要杯水和冰红茶 + # (3, self.add_walker, (5, 230, 1200)), + # (3, self.add_walker, (26, -30, -200, -90)), + # (3, self.add_walker, (10, -80, -180, -45)), + # # 上述准备 + # (self.st3, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), + # (self.st3, self.add_walker, (32, -80, 850, 135)), # 女 'BrightTable3': (-80, 850, 135), + # (self.st3, self.add_walker, (1, 60, 420, 135)), + # (self.st3, self.control_walker, (5, True, 50, 250, 1200, 180)), #设置id=4 的2小男孩随机游走红随机游走 + # (self.st3, self.add_walker, (48, 60, 520,0)), + # (self.st3 + 3, self.customer_say, (6, "哎呦，今天这么多人，还有空位吗？")),# 女士问 + # (self.st3 + 10, self.customer_say, (6, "我带着孩子呢，想要宽敞亮堂的地方。")), # 女士问 + # # 好的，我明白了，那么您可以选择我们的家庭亲子座，这样可以容纳您的孩子，并且更加宽敞舒适。 + # # 这里可以加一下自主导航和探索，找到一个位置 + # # 好的，我明白了，那么我们推荐您到大厅的桌子，那里的空间比较宽敞，环境也比较明亮，适合带着孩子一起用餐。 + # (self.st3 + 15, self.customer_say, (6, "大厅的桌子好啊，快带我去呀！")), + # (self.st3 + 25, self.control_walker, (6, False, 50,-250, 480, 0)), # #290, 400 + # (self.st3 + 25, self.customer_say, (6, "我想来杯水，帮我孩子拿个酸奶吧。")), + # (self.st3 + 60, self.customer_say, (6, "谢谢！")) + # 场景4：三人排队点单，一个要点心，一个没座位了赠送保温杯 + # (3, self.add_walker, (5, 230, 1200)), + # (3, self.add_walker, (26, -30, -200, -90)), + # (3, self.add_walker, (10, -80, -180, -45)), + # (3, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), + # (3, self.add_walker, (32, -80, 850, 135)), + # (3, self.add_walker, (1, 60, 420, 135)), + # (3, self.add_walker, (48, 60, 520, 0)), + # # # 上述准备 + # (self.st4, self.add_walker, (31, 60, 600, -90)), # 女红色排队 + # (self.st4, self.add_walker, (9, 60, 680, -90)), # 男灰黑色排队 + # (self.st4, self.add_walker, (20, 60, 760, -90)), # 大胖男排队 ] def _reset(self): diff --git a/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order_1118.py b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order_1118.py new file mode 100644 index 0000000..776e4e6 --- /dev/null +++ b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order_1118.py @@ -0,0 +1,102 @@ +""" +视觉语言操作 +机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板 +""" + +import time +from robowaiter.scene.scene import Scene + + + +class SceneVLM(Scene): + + def __init__(self, robot): + super().__init__(robot) + # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) + + self.scene_flag = 1 + self.st1 = 3 + self.st2 = self.st1 + 30 + self.st3 = 3 + + self.new_event_list = [ + + # 场景1：带小女孩找阳光下的空位 + # (3, self.add_walker, (5, 230, 1200)), + # (3, self.control_walker, (0, False, 200, 60, 520, 0)), + # (10, self.customer_say, (0, "早上好呀，我想找个能晒太阳的地方。")), + # (10, self.customer_say, (0,"可以带我过去嘛？")), + # (20, self.control_walker, (0, False, 50, 140, 1200, 180)), # 小女孩站在了 BrightTable1 旁边就餐啦 + + # 场景2：有个胖胖男人点单交流并要咖啡，帮他送去角落的桌子 + # (3, self.add_walker, (5, 230, 1200)), # 小女孩 + # 上述准备 + # (self.st2, self.add_walker, (26, 60,-140)), + # (self.st2, self.add_walker, (10, -80, -180,-45)), # （-150 -250） + # (self.st2, self.control_walker, (1, False, 100, 70, 520, 0)), #3 + # (self.st2+15, self.customer_say, (1, "你们这有什么饮料嘛？")), # 10 + # (self.st2+15, self.customer_say, (1, "咖啡有哪些呢？")),# 10 + # (self.st2+15+5, self.customer_say, (1,"来杯卡布奇诺吧，我在靠窗边的高脚桌那。")), # 15 + # (self.st2+15+5+5, self.control_walkers_and_say, ([[[1, False, 100, -30, -200, -90, "麻烦啦！"]]])), # 20 胖胖男到了 BrightTable6 + + # 场景3：有位女士要杯水和冰红茶 + (3, self.add_walker, (5, 230, 1200)), + (3, self.add_walker, (26, -30, -200, -90)), + (3, self.add_walker, (10, -80, -180, -45)), + # 上述准备 + (self.st3, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), + (self.st3, self.add_walker, (32, -80, 850, 135)), # 女 'BrightTable3': (-80, 850, 135), + (self.st3, self.add_walker, (1, -80, 750, 135)), + (self.st3, self.control_walker, (5, True, 50, 250, 1200, 180)), #设置id=4 的2小男孩随机游走红随机游走 + (self.st3, self.add_walker, (48, 60, 520,0)), + (self.st3 + 3, self.customer_say, (6, "哎呦，今天这么多人，还有空位吗？")),# 女士问 + (self.st3 + 10, self.customer_say, (6, "我带着孩子呢，想要宽敞亮堂的地方。")), # 女士问 + # 好的，我明白了，那么您可以选择我们的家庭亲子座，这样可以容纳您的孩子，并且更加宽敞舒适。 + # 这里可以加一下自主导航和探索，找到一个位置 + # 好的，我明白了，那么我们推荐您到大厅的桌子，那里的空间比较宽敞，环境也比较明亮，适合带着孩子一起用餐。 + (self.st3 + 15, self.customer_say, (6, "大厅的桌子好啊，快带我去呀！")), + (self.st3 + 20, self.control_walker, (6, False, 50,-290, 400, 180)), + + ] + + def _reset(self): + self.gen_obj() + # self.add_walkers([[47, 920]]) + pass + + def _run(self, op_type=10): + # 一个行人从门口走到 吧台 + # 打招呼需要什么 + # 行人说 哪里有位置，想晒个太阳 + # 带领行人去有太阳的地方 + # 行人说 有点热 + # 好的，这就去开空调 + self.walker_followed = False + pass + + def _step(self): + + + if self.scene_flag == 1: + # 如果机器人不在 吧台 + if self.walker_followed: + return + end = [self.status.location.X, self.status.location.Y] + if end[1]>=600 or end[1]<=450 or end[0]>=250: + # if int(self.status.location.X)!=247 or int(self.status.location.X)!=520: + self.walker_followed = True + self.control_walkers_and_say([[0,False,150,end[0],end[1],90,"wd！"]]) + self.scene_flag += 1 + + pass + +if __name__ == '__main__': + import os + from robowaiter.robot.robot import Robot + + robot = Robot() + + # create task + task = SceneVLM(robot) + task.reset() + task.run() diff --git a/robowaiter/scene/tasks/VLM/VLM_order.py b/robowaiter/scene/tasks/VLM/VLM_order.py index 1d773e4..7fa2b57 100644 --- a/robowaiter/scene/tasks/VLM/VLM_order.py +++ b/robowaiter/scene/tasks/VLM/VLM_order.py @@ -12,10 +12,10 @@ class SceneVLM(Scene): # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) self.new_event_list = [ (3, self.add_walker, (20,0,700)), - # (5, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw] - # # (10, self.customer_say, (6,"给我来杯酸奶和咖啡，哦对，再倒一杯水。")), - # (8, self.customer_say, (6, "来杯酸奶吧。")), - # (7, self.control_walker, (6, False, 100, -250, 480, 0)), #(-100,600) + (5, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw] + # (10, self.customer_say, (6,"给我来杯酸奶和咖啡，哦对，再倒一杯水。")), + (8, self.customer_say, (6, "来杯酸奶吧。")), + (7, self.control_walker, (6, False, 100, -250, 480, 0)), #(-100,600) # 有人提出要开空调和关窗帘 # bar (60, 520)