From 8c0896a19e9a12e34ddf64edc31776d76aeb875f Mon Sep 17 00:00:00 2001 From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com> Date: Sun, 19 Nov 2023 16:47:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BA=86=E5=92=96=E5=95=A1?= =?UTF-8?q?=E5=8E=85=E7=9A=84=E4=B8=80=E5=A4=A9=E7=9A=84=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/behavior_lib/_base/Behavior.py | 4 +- robowaiter/behavior_lib/act/GreetCustomer.py | 3 +- robowaiter/behavior_lib/act/ServeCustomer.py | 3 +- .../behavior_lib/cond/NewCustomerComing.py | 2 +- robowaiter/llm_client/data/fix_questions.txt | 22 +++ robowaiter/scene/scene.py | 31 ++-- .../VLN_greet_and_order.py | 160 +++++++++++++----- .../VLN_greet_and_order_1119.py | 135 +++++++++++++++ robowaiter/scene/tasks/OT/Open_tasks_test.py | 8 +- robowaiter/scene/tasks/VLM/VLM.py | 28 +-- 10 files changed, 324 insertions(+), 72 deletions(-) create mode 100644 robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order_1119.py diff --git a/robowaiter/behavior_lib/_base/Behavior.py b/robowaiter/behavior_lib/_base/Behavior.py index 635a22c..f066e77 100644 --- a/robowaiter/behavior_lib/_base/Behavior.py +++ b/robowaiter/behavior_lib/_base/Behavior.py @@ -30,8 +30,8 @@ class Bahavior(ptree.behaviour.Behaviour): # all_place=set() place_xyz_dic={ - 'Bar': (247.0, 520.0, 180.0), #(247.0, 520.0, 100.0) - 'Bar2': (240.0, 40.0, 70.0), + 'Bar': (247.0, 520.0, 100.0), #(247.0, 520.0, 100.0) + 'Bar2': (240.0, 40.0, 100.0), 'WaterTable':(-70.0, 500.0, 107), 'CoffeeTable':(250.0, 310.0, 100.0), 'Table1': (340.0, 900.0, 99.0), diff --git a/robowaiter/behavior_lib/act/GreetCustomer.py b/robowaiter/behavior_lib/act/GreetCustomer.py index 62dae5a..2e87c24 100644 --- a/robowaiter/behavior_lib/act/GreetCustomer.py +++ b/robowaiter/behavior_lib/act/GreetCustomer.py @@ -24,7 +24,8 @@ class GreetCustomer(Act): goal = Act.place_xyz_dic['Bar'] self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0) # self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？") - self.scene.chat_bubble("欢迎光临！") + if self.scene.show_bubble: + self.scene.chat_bubble("欢迎光临！") customer_name = self.scene.state['attention']['customer'] self.scene.state['greeted_customers'].add(customer_name) diff --git a/robowaiter/behavior_lib/act/ServeCustomer.py b/robowaiter/behavior_lib/act/ServeCustomer.py index 0c9f888..f90181f 100644 --- a/robowaiter/behavior_lib/act/ServeCustomer.py +++ b/robowaiter/behavior_lib/act/ServeCustomer.py @@ -22,5 +22,6 @@ class ServeCustomer(Act): goal = Act.place_xyz_dic['Bar'] self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0) - self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？") + if self.scene.show_bubble: + self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？") return ptree.common.Status.RUNNING diff --git a/robowaiter/behavior_lib/cond/NewCustomerComing.py b/robowaiter/behavior_lib/cond/NewCustomerComing.py index a77c190..fd8e693 100644 --- a/robowaiter/behavior_lib/cond/NewCustomerComing.py +++ b/robowaiter/behavior_lib/cond/NewCustomerComing.py @@ -22,7 +22,7 @@ class NewCustomerComing(Cond): for walker in scene.walkers: x, y, yaw = walker.pose.X, walker.pose.Y, walker.pose.Yaw # 到达一定区域就打招呼 - if y >= 450 and y <= 620 and x >= 40 and x <= 100 and yaw>=-10 and yaw <=10: + if y >= 510 and y <= 530 and x >= 50 and x <= 70 and yaw>=-10 and yaw <=10: #450-620 # close_to_bar = True queue_list.append((x,y,walker.name)) diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt index 55a05c0..fd963bb 100644 --- a/robowaiter/llm_client/data/fix_questions.txt +++ b/robowaiter/llm_client/data/fix_questions.txt @@ -32,6 +32,7 @@ create_sub_task create_sub_task {"goal":"At(Robot,BrightTable6)"} + 你们这有什么饮料嘛？ 我们咖啡厅提供各种口味咖啡，水，冰红茶，酸奶等饮品，还提供点心蛋糕等甜品。您想点什么？ @@ -42,6 +43,26 @@ create_sub_task {"goal":"On(Coffee,BrightTable6)"} +我昨天保温杯好像落在你们咖啡厅了，你看到了吗？ +是的，我有印象，保温杯在大厅的2号桌子上。 +get_object_info +{"obj":"2号桌子"} + + +给我来份午餐套餐。 +本店的午餐套餐包括咖啡和点心，请问您是打包还是堂食？ + +打包吧，快点！ +请稍等，马上就好！ +create_sub_task +{"goal":"On(Dessert,Bar),On(Coffee,Bar)"} + + +你可以帮我拿来吗，我在前门的桌子前等你。 +当然，请稍等！ +create_sub_task +{"goal":"On(VacuumCup,Table1)"} + 麻烦啦！ 没事儿，为您服务是我的荣幸！ @@ -61,6 +82,7 @@ create_sub_task create_sub_task {"goal":"On(Yogurt,WaterTable)"} + 好热呀！太阳也好大！ 抱歉，我这就去开空调和关窗帘。 create_sub_task diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py index 0f1d5a4..7cbc2e1 100644 --- a/robowaiter/scene/scene.py +++ b/robowaiter/scene/scene.py @@ -52,7 +52,7 @@ class Scene: event_list = [] new_event_list = [] signal_event_list = [] - show_bubble = False + # show_bubble = True event_signal = "None" default_state = { @@ -64,7 +64,9 @@ class Scene: "sub_goal_list": [], # 子目标列表 "status": None, # 仿真器中的观测信息，见下方详细解释 "condition_set": {'At(Robot,Bar)', 'Is(AC,Off)', - 'Holding(Nothing)','Exist(Yogurt)','Exist(BottledDrink)','On(Yogurt,Bar)','On(BottledDrink,Table1)', + 'Holding(Nothing)','Exist(Yogurt)','Exist(BottledDrink)','On(Yogurt,Bar)','On(BottledDrink,Bar)', + #'Exist(Softdrink)', 'On(Softdrink,Table1)', + 'Exist(VacuumCup)', 'On(VacuumCup,Table2)', 'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)', 'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}, "obj_mem":{}, @@ -95,6 +97,8 @@ class Scene: self.time = 0 self.sub_task_seq = None + self.show_bubble = True + # init robot if robot: robot.set_scene(self) @@ -174,8 +178,9 @@ class Scene: if len(self.signal_event_list)>0: next_event = self.signal_event_list[0] t, func,args = next_event - if t < 0 and self.robot_changed: #一直等待机器人行动，直到机器人无行动 - return + if t < 0: #一直等待机器人行动，直到机器人无行动 + if self.robot_changed: + return if t > 0: time.sleep(t) @@ -183,8 +188,6 @@ class Scene: self.signal_event_list.pop(0) func(*args) - - def deal_event(self): if len(self.event_list)>0: next_event = self.event_list[0] @@ -334,6 +337,12 @@ class Scene: for i in range(len(w)): self.state["customer_mem"][w[i].name] = i + def remove_walkers(self,IDs=[0]): + s = stub.Observe(GrabSim_pb2.SceneID(value=self.sceneID)) + scene = stub.RemoveWalkers(GrabSim_pb2.RemoveList(IDs=IDs, scene=self.sceneID)) + time.sleep(2) + return + def clean_walker(self): stub.CleanWalkers(GrabSim_pb2.SceneID(value=self.sceneID)) @@ -479,7 +488,7 @@ class Scene: name = self.walker_index2mem(name) print(f'{name} say: {sentence}') - if show_bubble: + if self.show_bubble and show_bubble: self.walker_bubble(name,sentence) self.state['chat_list'].append((name,sentence)) @@ -603,9 +612,11 @@ class Scene: ginger_loc = [scene.location.X, scene.location.Y, scene.location.Z] obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=5), # GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9), - GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=9), - - GrabSim_pb2.ObjectList.Object(x=340, y=960, z=88, roll=0, pitch=0, yaw=0, type=9), + # GrabSim_pb2.ObjectList.Object(x=340, y=960, z=88, roll=0, pitch=0, yaw=90, type=7), + # GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=90, type=9), + # GrabSim_pb2.ObjectList.Object(x=340, y=952, z=88, roll=0, pitch=0, yaw=90, type=4), + GrabSim_pb2.ObjectList.Object(x=-102, y=10, z=90, roll=0, pitch=0, yaw=90, type=7), + GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 70, z=95, roll=0, pitch=0, yaw=0, type=9), ] scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID)) diff --git a/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py index 8b1adcd..bfd23a1 100644 --- a/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py +++ b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py @@ -16,62 +16,138 @@ class SceneVLM(Scene): self.scene_flag = 2 self.st1 = 3 - self.st2 = 3 # self.st2 = self.st1 + 30 + # self.st3 = self.st2 + 65 + self.st2 = 3 self.st3 = 3 self.st4 = 3 - self.new_event_list = [ + self.signal_event_list = [ # 场景1：带小女孩找阳光下的空位 # (3, self.add_walker, (5, 230, 1200)), - # (3, self.control_walker, (0, False, 200, 60, 520, 0)), - # (10, self.customer_say, (0, "早上好呀，我想找个能晒太阳的地方。")), - # (10, self.customer_say, (0,"可以带我过去嘛？")), - # (20, self.control_walker, (0, False, 50, 140, 1200, 180)), # 小女孩站在了 BrightTable1 旁边就餐啦 + # (1, self.control_walker, (0, False, 200, 60, 520, 0)), + # (9, self.customer_say, (0, "早上好呀，我想找个能晒太阳的地方。")), + # (-1, self.customer_say, (0,"可以带我过去嘛？")), + # (0, self.control_walker, (0, False, 50, 140, 1200, 180)), # 小女孩站在了 BrightTable1 旁边就餐啦 # 场景2：有个胖胖男人点单交流并要咖啡，帮他送去角落的桌子 - (3, self.add_walker, (5, 230, 1200)), # 小女孩 + # (3, self.add_walker, (5, 230, 1200)), # 小女孩 # 上述准备 - (self.st2, self.add_walker, (26, -28, -150, 90)), - (self.st2, self.add_walker, (10, -70, -200, -45)), - (self.st2, self.customer_say, (1, "嘿，RoboWaiter，过来一下！")), - (self.st2+10, self.control_walkers_and_say, ([[[1, False, 100, -18, -200, -90, "你们这有什么饮料嘛？"]]])), # 20 胖胖男到了 BrightTable6 - (self.st2+15, self.customer_say, (1, "咖啡有哪些呢？")),# 10 - (self.st2+15+5, self.customer_say, (1,"来杯卡布奇诺吧。")), # 15 + # (self.st2, self.add_walker, (26, -28, -150, 90)), + # (self.st2, self.add_walker, (10, -70, -200, -45)), + # (self.st2, self.customer_say, (1, "嘿，RoboWaiter，过来一下！")), + # (self.st2+10, self.control_walkers_and_say, ([[[1, False, 100, -18, -200, -90, "你们这有什么饮料嘛？"]]])), # 20 胖胖男到了 BrightTable6 + # (self.st2+15, self.customer_say, (1, "咖啡有哪些呢？")),# 10 + # (self.st2+15+5, self.customer_say, (1,"来杯卡布奇诺吧。")), # 15 # 场景3：有位女士要杯水和冰红茶 - # (3, self.add_walker, (5, 230, 1200)), - # (3, self.add_walker, (26, -30, -200, -90)), - # (3, self.add_walker, (10, -80, -180, -45)), + (3, self.add_walker, (5, 230, 1200)), + (0, self.add_walker, (26, -30, -200, -90)), + (0, self.add_walker, (10, -80, -180, -45)), # # 上述准备 - # (self.st3, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), - # (self.st3, self.add_walker, (32, -80, 850, 135)), # 女 'BrightTable3': (-80, 850, 135), - # (self.st3, self.add_walker, (1, 60, 420, 135)), - # (self.st3, self.control_walker, (5, True, 50, 250, 1200, 180)), #设置id=4 的2小男孩随机游走红随机游走 - # (self.st3, self.add_walker, (48, 60, 520,0)), - # (self.st3 + 3, self.customer_say, (6, "哎呦，今天这么多人，还有空位吗？")),# 女士问 - # (self.st3 + 10, self.customer_say, (6, "我带着孩子呢，想要宽敞亮堂的地方。")), # 女士问 - # # 好的，我明白了，那么您可以选择我们的家庭亲子座，这样可以容纳您的孩子，并且更加宽敞舒适。 - # # 这里可以加一下自主导航和探索，找到一个位置 - # # 好的，我明白了，那么我们推荐您到大厅的桌子，那里的空间比较宽敞，环境也比较明亮，适合带着孩子一起用餐。 - # (self.st3 + 15, self.customer_say, (6, "大厅的桌子好啊，快带我去呀！")), - # (self.st3 + 25, self.control_walker, (6, False, 50,-250, 480, 0)), # #290, 400 - # (self.st3 + 25, self.customer_say, (6, "我想来杯水，帮我孩子拿个酸奶吧。")), - # (self.st3 + 60, self.customer_say, (6, "谢谢！")) + # (0, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), + # (0, self.add_walker, (32, -80, 850, 135)), # 女 'BrightTable3': (-80, 850, 135), + # (0, self.add_walker, (1, 60, 420, 135)), # 生成小男孩随机游走 + (0,self.add_walkers,([[[21, 65, 1000, -90],[32, -80, 850, 135],[1, 60, 420, 135]]])), + (0, self.control_walker, (5, True, 50, 250, 1200, 180)), #设置id=4 的2小男孩随机游走红随机游走 + # (0, self.add_walker, (48, 60, 520,0)),# 生成他妈妈 + # (0, self.add_walker, (31, 60, 600, -90)), # 女红色排队 7号 + # (0, self.add_walker, (20, 60, 680, -90)), # 大胖男色排队 8号 + # (0, self.add_walker, (9, 60, 760, -90)), # 男灰黑排队 9号 + # (0, self.add_walker, (29, -290, 400, 180)), # 青色女人占了位置 BrightTable5 + (0, self.add_walkers, ([[[48, 60, 520,0], [31, 60, 600, -90], [20, 60, 680, -90],[9, 60, 760, -90],[29, -290, 400, 180]]])), - # 场景4：三人排队点单，一个要点心，一个没座位了赠送保温杯 + (0, self.customer_say, (6, "哎呦，今天这么多人，还有空位吗？")),# 女士问 + (10, self.customer_say, (6, "我带着孩子呢，想要宽敞亮堂的地方。")), # 女士问 + # 好的，我明白了，那么您可以选择我们的家庭亲子座，这样可以容纳您的孩子，并且更加宽敞舒适。 + # 这里可以加一下自主导航和探索，找到一个位置 + # 好的，我明白了，那么我们推荐您到大厅的桌子，那里的空间比较宽敞，环境也比较明亮，适合带着孩子一起用餐。 + + + (8, self.customer_say, (6, "大厅的桌子好啊，快带我去呀！")), + (10, self.control_walker, (6, False, 50,-250, 480, 0)), # #290, 400 + # (5, self.customer_say, (6, "我想来杯水，帮我孩子拿个酸奶吧。")), + (-1, self.customer_say, (6, "谢谢！")), + + # 9号灰色男 排队排着排着，不排了 + (0, self.control_walker, (9, False, 100, 100, 760, 180)), + (0, self.control_walker, (9, True, 100, 0, 0, 180)), + # ### 增加场景，孩子说热要开空调 或者9号随机游走和说 + # (0, self.control_walker, (9, False, 100, 100, 760, 180)), + # (0, self.control_walker, (9, False, 100, 0, 0, 180)), + # (0, self.control_walkers_and_say, ([[[9, True, 100, 0, 0, 180, "好热呀！太阳也好大！"]]])), + # # (9, True, 100, 0, 0, 180)), + # # (0, self.customer_say, (1, "咖啡有哪些呢？")), + # ### 增加场景，孩子说热要开空调 或者9号随机游走和说 + + + + # 场景4：三人排队点单，女士要保温杯 # (3, self.add_walker, (5, 230, 1200)), - # (3, self.add_walker, (26, -30, -200, -90)), - # (3, self.add_walker, (10, -80, -180, -45)), - # (3, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), - # (3, self.add_walker, (32, -80, 850, 135)), - # (3, self.add_walker, (1, 60, 420, 135)), - # (3, self.add_walker, (48, 60, 520, 0)), - # # # 上述准备 - # (self.st4, self.add_walker, (31, 60, 600, -90)), # 女红色排队 - # (self.st4, self.add_walker, (9, 60, 680, -90)), # 男灰黑色排队 - # (self.st4, self.add_walker, (20, 60, 760, -90)), # 大胖男排队 + # (0, self.add_walker, (26, -30, -200, -90)), + # (0, self.add_walker, (10, -80, -180, -45)), + # (0, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), + # (0, self.add_walker, (32, -80, 850, 135)), + # (0, self.add_walker, (1, 60, 220, 135)), + # (0, self.add_walker, (48, 60, 320, 0)), + # (0, self.add_walker, (31, 60, 600, -90)), # 女红色排队 7号找保温杯的顾客 + # (0, self.add_walker, (20, 60, 680, -90)), # 大胖男排队 + # (0, self.add_walker, (9, 60, 760, -90)), # 男灰黑色排队 + # (0, self.add_walker, (29, -290, 400, 180)), # # 青色女人占了位置 BrightTable5 + # # # # 上述准备 + (0, self.control_walkers_and_say, ([[[7, False, 100, 60, 520, 0, "我昨天保温杯好像落在你们咖啡厅了，你看到了吗？"]]])), + (5, self.customer_say, (7,"你可以帮我拿来吗，我在前门的桌子前等你。")), + (1, self.control_walker,(7, False, 80, -10, 520, 90)),# 红女士在吧台前后退一步 + (1, self.control_walker, (7, False, 80, 240, 1000, -45)), # 红女士走到Table1前 + (1, self.control_walker, (8, False, 100, 60, 600, -90)), # 大胖男排队往前走一步 + (2, self.control_walker, (9, False, 100, 60, 680, -90)), # 男灰黑色排队往前走一步 + (6, self.customer_say, (7,"就是这个杯子！找到啦，好开心！")), # 红女士在Table1前 + + + + # 场景5：三人排队点单，一人要冰红茶，一个要点心，一个没座位了赠送保温杯 + # (3, self.add_walker, (5, 230, 1200)), + # (0, self.add_walker, (26, -30, -200, -90)), + # (0, self.add_walker, (10, -80, -180, -45)), + # (0, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), + # (0, self.add_walker, (32, -80, 850, 135)), + # (0, self.add_walker, (1, 60, 220, 135)), + # (0, self.add_walker, (48, 60, 320, 0)), # 生成他妈妈 + # (0, self.add_walker, (31, 280, 1200, -45)), # # 女红色排队 7号找保温杯的顾客 + # (0, self.add_walker, (20, 60, 680, -90)), # 大胖男排队 + # (0, self.add_walker, (9, 60, 760, -90)), # 男灰黑色排队 + # (0, self.add_walker, (29, -290, 400, 180)), # # 青色女人占了位置 BrightTable5 + + ## 9号灰色男 排队排着排着，不排了 + (0, self.control_walker, (9, False, 100, 100, 760, 180)), + (0, self.control_walker, (9, True, 100, 0, 0, 180)), + # 上述准备 + + (0, self.control_walker, (9, False, 100, 60, 760, -90)), + + + # 场景6：大胖男点了外卖，离开啦 + (2, self.control_walkers_and_say,([[[8, False, 100, 60, 520, 0, "给我来份午餐套餐。"]]])), + (0, self.animation_reset,()), #重置任务 + (2, self.customer_say, (8, "打包吧，快点！")), + (2, self.control_walker, (9, False, 100, 60, 620, -90)), # 男灰黑色排队往前走一步 + (-1, self.customer_say, (8, "谢啦，我赶时间！")), + (2, self.control_walker, (8, False, 250, 20, 520, -90)), # 大胖男着急得离开啦 + (2, self.control_walker, (8, False, 250, 240, -150, -90)), + (5, self.remove_walkers, ([[0,7,8]])), + # (2, self.control_walker, (6, False, 100, 60, 520, 0)), # 9号变7号 男灰黑色排队往前,轮到他 + + + # 场景7：最后排队那个随机游走 9号变为6号，随机游走。 + # 机器人自主发现任务，走一圈去擦桌子/拖地，碰到灰色男问好，灰色男说“太阳大，要关窗帘和空调调低” + # 开了空调 + (2, self.control_walker, (6, False, 100, 60, 520, 0)), # 9号变7号 男灰黑色排队往前,轮到他 + (2, self.customer_say, (6, "好热呀！太阳也好大！")) + + + + ] def _reset(self): @@ -100,7 +176,7 @@ class SceneVLM(Scene): if end[1]>=600 or end[1]<=450 or end[0]>=250: # if int(self.status.location.X)!=247 or int(self.status.location.X)!=520: self.walker_followed = True - self.control_walkers_and_say([[0,False,150,end[0],end[1],90,"wd！"]]) + self.control_walkers_and_say([[0,False,150,end[0],end[1],90,"谢谢！"]]) self.scene_flag += 1 pass diff --git a/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order_1119.py b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order_1119.py new file mode 100644 index 0000000..6d1bb40 --- /dev/null +++ b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order_1119.py @@ -0,0 +1,135 @@ +""" +视觉语言操作 +机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板 +""" + +import time +from robowaiter.scene.scene import Scene + + + +class SceneVLM(Scene): + + def __init__(self, robot): + super().__init__(robot) + # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) + + self.scene_flag = 2 + self.st1 = 3 + # self.st2 = self.st1 + 30 + # self.st3 = self.st2 + 65 + self.st2 = 3 + self.st3 = 3 + self.st4 = 3 + + self.new_event_list = [ + + # 场景1：带小女孩找阳光下的空位 + # (3, self.add_walker, (5, 230, 1200)), + # (3, self.control_walker, (0, False, 200, 60, 520, 0)), + # (12, self.customer_say, (0, "早上好呀，我想找个能晒太阳的地方。")), + # (12, self.customer_say, (0,"可以带我过去嘛？")), + # (20, self.control_walker, (0, False, 50, 140, 1200, 180)), # 小女孩站在了 BrightTable1 旁边就餐啦 + + # 场景2：有个胖胖男人点单交流并要咖啡，帮他送去角落的桌子 + # (3, self.add_walker, (5, 230, 1200)), # 小女孩 + # 上述准备 + # (self.st2, self.add_walker, (26, -28, -150, 90)), + # (self.st2, self.add_walker, (10, -70, -200, -45)), + # (self.st2, self.customer_say, (1, "嘿，RoboWaiter，过来一下！")), + # (self.st2+10, self.control_walkers_and_say, ([[[1, False, 100, -18, -200, -90, "你们这有什么饮料嘛？"]]])), # 20 胖胖男到了 BrightTable6 + # (self.st2+15, self.customer_say, (1, "咖啡有哪些呢？")),# 10 + # (self.st2+15+5, self.customer_say, (1,"来杯卡布奇诺吧。")), # 15 + + # 场景3：有位女士要杯水和冰红茶 + # (3, self.add_walker, (5, 230, 1200)), + # (3, self.add_walker, (26, -30, -200, -90)), + # (3, self.add_walker, (10, -80, -180, -45)), + # # # 上述准备 + # (self.st3, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), + # (self.st3, self.add_walker, (32, -80, 850, 135)), # 女 'BrightTable3': (-80, 850, 135), + # + # (self.st3, self.add_walker, (1, 60, 420, 135)), # 生成小男孩随机游走 + # (self.st3, self.control_walker, (5, True, 50, 250, 1200, 180)), #设置id=4 的2小男孩随机游走红随机游走 + # (self.st3, self.add_walker, (48, 60, 520,0)),# 生成他妈妈 + # + # (self.st3, self.add_walker, (31, 60, 600, -90)), # 女红色排队 7号 + # (self.st3, self.add_walker, (9, 60, 680, -90)), # 男灰黑色排队 8号 + # (self.st3, self.add_walker, (20, 60, 760, -90)), # 大胖男排队 9号 + # + # (self.st3 + 3, self.customer_say, (6, "哎呦，今天这么多人，还有空位吗？")),# 女士问 + # (self.st3 + 10, self.customer_say, (6, "我带着孩子呢，想要宽敞亮堂的地方。")), # 女士问 + # # 好的，我明白了，那么您可以选择我们的家庭亲子座，这样可以容纳您的孩子，并且更加宽敞舒适。 + # # 这里可以加一下自主导航和探索，找到一个位置 + # # 好的，我明白了，那么我们推荐您到大厅的桌子，那里的空间比较宽敞，环境也比较明亮，适合带着孩子一起用餐。 + # (self.st3 + 15, self.customer_say, (6, "大厅的桌子好啊，快带我去呀！")), + # (self.st3 + 25, self.control_walker, (6, False, 50,-250, 480, 0)), # #290, 400 + # (self.st3 + 25, self.customer_say, (6, "我想来杯水，帮我孩子拿个酸奶吧。")), + # (self.st3 + 60, self.customer_say, (6, "谢谢！")), + + + # 场景4：三人排队点单，一人要冰红茶，一个要点心，一个没座位了赠送保温杯 + (3, self.add_walker, (5, 230, 1200)), + (3, self.add_walker, (26, -30, -200, -90)), + (3, self.add_walker, (10, -80, -180, -45)), + (3, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), + (3, self.add_walker, (32, -80, 850, 135)), + (3, self.add_walker, (1, 60, 220, 135)), + (3, self.add_walker, (48, 60, 320, 0)), + (3, self.add_walker, (31, 60, 600, -90)), # 女红色排队 + (3, self.add_walker, (9, 60, 680, -90)), # 男灰黑色排队 + (3, self.add_walker, (20, 60, 760, -90)), # 大胖男排队 + + (3, self.add_walker, (6, -55, 750,180)), + + # # # 上述准备 + (self.st4+5, self.control_walkers_and_say, ([[[7, False, 100, 60, 520, 0, "我昨天保温杯好像落在你们咖啡厅了，你看到了吗？"]]])), + (self.st4 + 10, self.customer_say, (7,"你可以帮我拿来吗，我在前门的桌子前等你。")), + (self.st4 + 13, self.control_walker,(7, False, 80, -10, 520, 90)),# 红女士在吧台前后退一步 + (self.st4 + 14, self.control_walker, (7, False, 80, 240, 1000, -45)), # 红女士走到Table1前 + (self.st4 + 15, self.control_walker, (8, False, 100, 60, 600, -90)), + (self.st4 + 17, self.control_walker, (9, False, 100, 60, 680, -90)), + (self.st4 + 25, self.customer_say, (7,"就是这个杯子！找到啦，好开心！")), # 红女士在Table1前 + ] + + def _reset(self): + self.gen_obj() + # self.add_walkers([[47, 920]]) + pass + + def _run(self, op_type=10): + # 一个行人从门口走到 吧台 + # 打招呼需要什么 + # 行人说 哪里有位置，想晒个太阳 + # 带领行人去有太阳的地方 + # 行人说 有点热 + # 好的，这就去开空调 + self.walker_followed = False + pass + + def _step(self): + + + if self.scene_flag == 1: + # 如果机器人不在 吧台 + if self.walker_followed: + return + end = [self.status.location.X, self.status.location.Y] + if end[1]>=600 or end[1]<=450 or end[0]>=250: + # if int(self.status.location.X)!=247 or int(self.status.location.X)!=520: + self.walker_followed = True + self.control_walkers_and_say([[0,False,150,end[0],end[1],90,"谢谢！"]]) + self.scene_flag += 1 + + pass + +if __name__ == '__main__': + import os + from robowaiter.robot.robot import Robot + + robot = Robot() + + # create task + task = SceneVLM(robot) + task.reset() + task.run() diff --git a/robowaiter/scene/tasks/OT/Open_tasks_test.py b/robowaiter/scene/tasks/OT/Open_tasks_test.py index 8844020..895cc98 100644 --- a/robowaiter/scene/tasks/OT/Open_tasks_test.py +++ b/robowaiter/scene/tasks/OT/Open_tasks_test.py @@ -18,13 +18,17 @@ class SceneOT(Scene): super().__init__(robot) # 在这里加入场景中发生的事件 self.signal_event_list = [ - (3, self.customer_say, ("System","来一号桌")), - (-1, self.customer_say, ("System","回去吧")), + # (3, self.customer_say, ("System","来一号桌")), + # (-1, self.customer_say, ("System","回去吧")), # (5, self.set_goal("At(Robot,BrightTable4)")) ] + self.event_list = [ + (3, self.set_goal("On(VacuumCup,Bar)")) + ] def _reset(self): # self.add_walkers([[0, 880], [250, 1200]]) + self.gen_obj() pass diff --git a/robowaiter/scene/tasks/VLM/VLM.py b/robowaiter/scene/tasks/VLM/VLM.py index 2524f6c..eb42cd1 100644 --- a/robowaiter/scene/tasks/VLM/VLM.py +++ b/robowaiter/scene/tasks/VLM/VLM.py @@ -75,7 +75,7 @@ class SceneVLM(Scene): # self.move_task_area(op_type=4) pass - def _run(self, op_type=10): + def _run(self, op_type=16): # 共17个操作 # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7 # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12 @@ -83,18 +83,20 @@ class SceneVLM(Scene): # "抓握物体","放置物体" # 16-17 # self.gen_obj() - # if op_type <=15: - # self.move_task_area(op_type) - # self.op_task_execute(op_type) - # if op_type == 16: # 16: 抓操作需要传入物品id - # self.move_task_area(op_type, obj_id=0) - # self.op_task_execute(op_type, obj_id=0) - # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107] - # # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0] # 桌子2:[-55.0, 150.0, 107], - # if op_type == 17: # 17: 放操作需要传入放置位置周围的可达区域 - # pos = [240.0, 40.0, 100.0] - # self.move_task_area(op_type, release_pos=pos) - # self.op_task_execute(op_type, release_pos=pos) # [325.0, 860.0, 100] + if op_type <=15: + self.move_task_area(op_type) + self.op_task_execute(op_type) + if op_type == 16: # 16: 抓操作需要传入物品id + self.move_task_area(op_type, obj_id=1) + self.op_task_execute(op_type, obj_id=1) + # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107] + # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0] # 桌子2:[-55.0, 150.0, 107], + if op_type == 17: # 17: 放操作需要传入放置位置周围的可达区域 + pos = [240.0, 40.0, 100.0] + self.move_task_area(op_type, release_pos=pos) + self.op_task_execute(op_type, release_pos=pos) # [325.0, 860.0, 100] + + # 流程测试 # 抓握放置:抓吧台前生成的酸奶，放到抹布桌上