From 897920beb940e5baa2ba9e822b1b3dbb7f9ca629 Mon Sep 17 00:00:00 2001 From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com> Date: Mon, 20 Nov 2023 21:12:18 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=A4=8D=E6=9D=82=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=EF=BC=8C=E7=82=B9=E9=A4=90=E5=92=8C=E5=8A=A8=E4=BD=9C?= =?UTF-8?q?=E8=A7=84=E5=88=92=E3=80=82OT=5Fcoffee=5Fwater=5Fdessert.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/behavior_lib/act/GreetCustomer.py | 4 +- robowaiter/llm_client/data/fix_questions.txt | 13 +- robowaiter/scene/scene.py | 21 +++ .../scene/tasks/GQA/GQA_1_ask_order_place.py | 45 +++++++ .../scene/tasks/OT/OT_coffee_water_dessert.py | 123 +++--------------- robowaiter/scene/tasks/VLM/VLM_1_order.py | 2 +- 6 files changed, 96 insertions(+), 112 deletions(-) create mode 100644 robowaiter/scene/tasks/GQA/GQA_1_ask_order_place.py diff --git a/robowaiter/behavior_lib/act/GreetCustomer.py b/robowaiter/behavior_lib/act/GreetCustomer.py index 2fcdcd1..6353d42 100644 --- a/robowaiter/behavior_lib/act/GreetCustomer.py +++ b/robowaiter/behavior_lib/act/GreetCustomer.py @@ -28,8 +28,8 @@ class GreetCustomer(Act): self.scene.walk_to(goal[0] - 5, goal[1], 180, 180, 0) if self.scene.show_bubble: - self.scene.chat_bubble("欢迎光临！") - # self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？") + # self.scene.chat_bubble("欢迎光临！") + self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？") customer_name = self.scene.state['attention']['customer'] self.scene.state['greeted_customers'].add(customer_name) diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt index f2af3cb..6884103 100644 --- a/robowaiter/llm_client/data/fix_questions.txt +++ b/robowaiter/llm_client/data/fix_questions.txt @@ -134,6 +134,17 @@ create_sub_task 给我来杯咖啡，哦对，再倒一杯水。 -好的，请稍等。 +速速就好，请稍等！ create_sub_task {"goal":"On(Water,WaterTable),On(Coffee,WaterTable)"} + + +我想来份点心和酸奶。 +稍等片刻噢~ +create_sub_task +{"goal":"On(Yogurt,Bar),On(Dessert,Bar)"} + +酸奶。 +稍等片刻噢~ +create_sub_task +{"goal":"On(Yogurt,Bar)"} diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py index 9ae5af6..03d2b97 100644 --- a/robowaiter/scene/scene.py +++ b/robowaiter/scene/scene.py @@ -491,6 +491,27 @@ class Scene: scene = stub.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=self.sceneID)) return scene + def control_walker_ls(self, walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]]): + """pose:表示行人的终止位置姿态""" + scene = self.status + walker_loc = walker_loc + controls = [] + for walker in walker_loc: + if len(walker) == 2: + self.control_walker(walker[0], walker[1]) + elif len(walker) == 3: + self.control_walker(walker[0], walker[1], walker[2]) + elif len(walker) == 4: + self.control_walker(walker[0], walker[1], walker[2], walker[3]) + elif len(walker) == 5: + self.control_walker(walker[0], walker[1], walker[2], walker[3], walker[4]) + elif len(walker) == 6: + self.control_walker(walker[0], walker[1], walker[2], walker[3], walker[4], walker[5]) + # self.control_walker() + # scene = stub.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=self.sceneID)) + # return scene + return + def control_joints(self, angles): stub.Do( GrabSim_pb2.Action( diff --git a/robowaiter/scene/tasks/GQA/GQA_1_ask_order_place.py b/robowaiter/scene/tasks/GQA/GQA_1_ask_order_place.py new file mode 100644 index 0000000..ee9a208 --- /dev/null +++ b/robowaiter/scene/tasks/GQA/GQA_1_ask_order_place.py @@ -0,0 +1,45 @@ +""" +具身多轮对话 GQA +点餐（order）的对话，咖啡厅服务员可以为客人（NPC）完成点餐基本对话 +场景对话（GQA）结合场景：询问卫生间、附近娱乐场所（数据来源自主定义） +开始条件：顾客NPC发出点餐指令 +结束条件：顾客NPC发出指令，表示不再需要服务 +""" + +# todo: 使用大模型进行对话，获得指令信息，适时结束对话 +# order = {...} + +from robowaiter.scene.scene import Scene + +class SceneGQA(Scene): + def __init__(self, robot): + super().__init__(robot) + # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) + self.new_event_list = [ + (3, self.customer_say, ("System","哪里有蛋糕")) + ] + + def _reset(self): + self.gen_obj() + self.add_walkers([ [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]]) + self.control_walker(1, True, 100, 60, 720, 0) + self.control_walker(4, True, 100, 60, -120, 0) + self.add_walkers([[16,60, 520], [47,-40, 520]]) + pass + + + + def _run(self): + pass + + +if __name__ == '__main__': + import os + from robowaiter.robot.robot import Robot + + robot = Robot() + + # create task + task = SceneGQA(robot) + task.reset() + task.run() diff --git a/robowaiter/scene/tasks/OT/OT_coffee_water_dessert.py b/robowaiter/scene/tasks/OT/OT_coffee_water_dessert.py index ee32c70..9db7e84 100644 --- a/robowaiter/scene/tasks/OT/OT_coffee_water_dessert.py +++ b/robowaiter/scene/tasks/OT/OT_coffee_water_dessert.py @@ -11,121 +11,28 @@ class SceneVLM(Scene): super().__init__(robot) # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) self.signal_event_list = [ - (3, self.add_walker, (20,0,700)), - (1, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw] - (10, self.customer_say, (6, "给我来杯咖啡，哦对，再倒一杯水。")), - # (10, self.customer_say, (6,"给我来杯酸奶和咖啡，哦对，再倒一杯水。")), - # (-1, self.customer_say, (6, "来杯酸奶吧。")), - # (3, self.customer_say, (6, "给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。")), - (5, self.control_walker, (6, False, 100, -250, 480, 0)), #(-100,600) - - # 有人提出要开空调和关窗帘 - # bar (60, 520) - # (28, self.add_walker, (0, 0, 0)), - # (33, self.control_walker, (7, False, 100, 60, 520, 180)), - # (35, self.customer_say, (7,"好热呀！太阳也好大！")), - # (45, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])), - - # (3, self.add_walker, (0, 0, 0)), - # (1, self.control_walker, (5, False, 100, 60, 520, 180)), - # (1, self.customer_say, (5,"好热呀！太阳也好大！")), - # (-1, self.control_walkers_and_say, ([[[5, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])), + (5, self.customer_say, (5, "给我来杯咖啡，哦对，再倒一杯水。")), + (1, self.control_walker_ls,([[[5, False, 100, -250, 480, 0],[6, False, 100, 60, 520, 0]]])), + (-1, self.customer_say, (5, "感谢，这些够啦，你去忙吧。")), + (10, self.customer_say, (6, "我想来份点心和酸奶。")), + (-1, self.customer_say, (6, "真美味啊！")), ] def _reset(self): self.gen_obj() - # self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)', - # 'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)', - # 'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)', - # 'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'} - # # 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220) - self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]]) - # [3,1, 880] 1号桌旁边小女孩 - # [31,250, 1200] 最角落QuietTable1女红色 - # [6,-55, 750] 1号桌附近小男孩 - # [10,70, -200] 另一边角落 QuietTable2 男黄色 - # [27,-290, 400, 180] 中间 BrightTable4 女灰 - # [26, 60,-320,90] 另一边角落 BrightTable5 红胖男 - self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True) - - # 0-3男孩 4-7女孩 8-26男 - # 3男孩 31女红 32女灰 10黄色衣服男瘦 9男灰瘦 26红胖男 - # 17 是员工 police - # [0, -150,180] - - - # self.control_walkers(walker_loc=[[-55, 750]],is_autowalk = False) - # 在场景中随机增加一堆行人。 - # walker_loc = [[-55, 750], [70, -200], [250, 1200], [0, 880]] - # controls = [] - # for i in range(len(s.walkers)): - # loc = walker_loc[i] - # is_autowalk = False - # pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180) - # controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=200, pose=pose)) - # scene = sim_client.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=scene_id)) - - # self.gen_obj(type=5) - # self.gen_obj(type=9) - # self.op_task_execute(op_type=16, obj_id=0) - # self.move_task_area(op_type=4) + self.add_walkers([ [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]]) + self.control_walker(1, True, 100, 60, 720, 0) + self.control_walker(4, True, 100, 60, -120, 0) + self.add_walkers([[16,60, 520], [47,-40, 520]]) + self.state["condition_set"] = { 'At(Robot,Bar)', 'Is(AC,Off)', + 'Holding(Nothing)', 'Exist(Yogurt)', 'Exist(BottledDrink)', + 'On(BottledDrink,Bar)', + 'Exist(VacuumCup)', 'On(VacuumCup,Table2)', + 'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)', + 'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'} pass def _run(self, op_type=10): - # 共17个操作 - # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7 - # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12 - # "调整空调开关","调高空调温度","调低空调温度", # 13-15 - # "抓握物体","放置物体" # 16-17 - # - # self.gen_obj() - # if op_type <=15: - # self.move_task_area(op_type) - # self.op_task_execute(op_type) - # if op_type == 16: # 16: 抓操作需要传入物品id - # self.move_task_area(op_type, obj_id=0) - # self.op_task_execute(op_type, obj_id=0) - # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107] - # # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0] # 桌子2:[-55.0, 150.0, 107], - # if op_type == 17: # 17: 放操作需要传入放置位置周围的可达区域 - # pos = [240.0, 40.0, 100.0] - # self.move_task_area(op_type, release_pos=pos) - # self.op_task_execute(op_type, release_pos=pos) # [325.0, 860.0, 100] - # - # 流程测试 - # 抓握放置:抓吧台前生成的酸奶，放到抹布桌上 - # self.gen_obj() - # self.move_task_area(16, obj_id=0) - # self.op_task_execute(16, obj_id=0) - # pos = [340.0, 900.0, 99.0] - # self.move_task_area(17, release_pos=pos) - # self.op_task_execute(17, release_pos=pos) - # - # # 做咖啡:做完的咖啡放到水杯桌上 - # self.move_task_area(1) - # self.op_task_execute(1) - # - # self.find_obj("CoffeeCup") - # - # self.move_task_area(16, obj_id=275) - # self.op_task_execute(16, obj_id=275) - # pos = [-70.0, 500.0, 107] - # self.move_task_area(17, release_pos=pos) - # self.op_task_execute(17, release_pos=pos) - # - # # 倒水:倒完的水放到旁边桌子上 - # self.move_task_area(2) - # self.op_task_execute(2) - # - # - # self.move_task_area(16, obj_id=190) - # self.op_task_execute(16, obj_id=190) - # pos = [-55.0, 0.0, 107] - # self.move_task_area(17, release_pos=pos) - # self.op_task_execute(17, release_pos=pos) - # - # self.test_yaw() - pass def _step(self): diff --git a/robowaiter/scene/tasks/VLM/VLM_1_order.py b/robowaiter/scene/tasks/VLM/VLM_1_order.py index 49b9d0e..337ccac 100644 --- a/robowaiter/scene/tasks/VLM/VLM_1_order.py +++ b/robowaiter/scene/tasks/VLM/VLM_1_order.py @@ -21,7 +21,7 @@ class SceneVLM(Scene): self.gen_obj() self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]]) - self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True) + self. control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True) pass