添加了顾客提出点单和提出开关空调的场景 VLM_order

robowaiter/behavior_lib/act/GreetCustomer.py

@@ -23,7 +23,8 @@ class GreetCustomer(Act):
 
         goal = Act.place_xyz_dic['Bar']
         self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0)
-        self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？")
+        # self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？")
+        self.scene.chat_bubble("欢迎光临！")
 
         customer_name = self.scene.state['attention']['customer']
         self.scene.state['greeted_customers'].add(customer_name)

robowaiter/llm_client/data/test_questions.txt

@@ -1 +1 @@
-{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}
+{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}, "给我来杯酸奶和咖啡，哦对，再倒一杯水。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,WaterTable)\",\"On(Water,WaterTable)\"}"}, "好热呀！太阳也好大！": {"Answer": "抱歉！我这就去开空调和关窗帘！", "Goal": "{\"Is(AC,On)\",\"Is(Curtain,Off)\"}"}, "来杯酸奶吧。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Yogurt,WaterTable)\"}"}}

robowaiter/llm_client/data_raw/test_questions.csv

@@ -22,3 +22,6 @@ Question,Answer,Goal
 把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。,明白，我这就去办！,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
 下班啦！打扫卫生，关灯关空调关窗帘。,太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
 可以带我去空位上嘛？我想晒太阳。,没问题！请跟我来。,"{""At(Robot,BrightTable1)""}"
+给我来杯酸奶和咖啡，哦对，再倒一杯水。,好嘞，请稍等！,"{""On(Coffee,WaterTable)"",""On(Yogurt,WaterTable)"",""On(Water,WaterTable)""}"
+好热呀！太阳也好大！,抱歉！我这就去开空调和关窗帘！,"{""Is(AC,On)"",""Is(Curtain,Off)""}"
+来杯酸奶吧。,好嘞，请稍等！,"{""On(Yogurt,WaterTable)""}"

robowaiter/scene/scene.py

@@ -286,15 +286,15 @@ class Scene:
 
     def add_walkers(self,walker_loc=[[0, 880], [250, 1200], [-55, 750], [70, -200]]):
         print('------------------add_walkers----------------------')
-        for id,walker in enumerate(walker_loc):
+        for i,walker in enumerate(walker_loc):
             if len(walker)==2:
-                self.add_walker(id,walker[0],walker[1])
+                self.add_walker(i,walker[0],walker[1])
             elif len(walker)==3:
-                self.add_walker(id, walker[0], walker[1],walker[2])
+                self.add_walker(walker[0], walker[1], walker[2])
             elif len(walker) == 4:
-                self.add_walker(id, walker[0], walker[1], walker[2], walker[3])
+                self.add_walker(walker[0], walker[1], walker[2], walker[3])
             elif len(walker) == 5:
-                self.add_walker(id, walker[0], walker[1], walker[2], walker[3], walker[4])
+                self.add_walker(walker[0], walker[1], walker[2], walker[3], walker[4])
 
     def remove_walker(self, *args):  # take single walkerID or a list of walkerIDs
         remove_list = []
@@ -318,10 +318,15 @@ class Scene:
     def clean_walker(self):
         stub.CleanWalkers(GrabSim_pb2.SceneID(value=self.sceneID))
 
-    def control_walker(self, control_list):
+    def control_walker(self, walkerID,autowalk,speed,X,Y,Yaw=0):
-        stub.ControlWalkers(
+        pose = GrabSim_pb2.Pose(X=X, Y=Y, Yaw=Yaw)
-            GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
+        scene = stub.ControlWalkers(
+            GrabSim_pb2.WalkerControls(controls=[GrabSim_pb2.WalkerControls.WControl(id=walkerID, autowalk=autowalk, speed=speed, pose=pose)], scene=self.sceneID)
         )
+        return scene
+        # stub.ControlWalkers(
+        #     GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
+        # )
 
     def control_walkers_and_say(self, control_list_ls):
         """ 同时处理行人的行走和对话
@@ -331,13 +336,15 @@ class Scene:
         for control in control_list_ls:
             if control[-1]!= None:
                 walkerID = control[0]
-                cont = self.status.walkers[walkerID].name + ":"+control[-1]
+                # cont = self.status.walkers[walkerID].name + ":"+control[-1]
-                self.control_robot_action(control[walkerID], 3, cont)
+                # self.control_robot_action(control[walkerID], 3, cont)
+                self.customer_say(walkerID,control[-1])
             control_list.append(self.walker_control_generator(walkerID=control[0], autowalk=control[1], speed=control[2], X=control[3], Y=control[4], Yaw=control[5]))
         # 收集没有对话的统一控制
-        stub.ControlWalkers(
+        scene = stub.ControlWalkers(
             GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
         )
+        return scene
 
     def control_walkers(self,walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True):
         """pose:表示行人的终止位置姿态"""
@@ -350,7 +357,7 @@ class Scene:
             pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180)
             controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=80, pose=pose))
         scene = stub.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=self.sceneID))
-
+        return scene
 
     def control_joints(self, angles):
         stub.Do(

robowaiter/scene/tasks/VLM/VLM_order.py

@@ -0,0 +1,142 @@
+"""
+视觉语言操作
+机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板
+"""
+
+import time
+from robowaiter.scene.scene import Scene
+
+class SceneVLM(Scene):
+    def __init__(self, robot):
+        super().__init__(robot)
+        # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
+        self.new_event_list = [
+            (3, self.add_walker,  (20,0,700)),
+            (5, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw]
+            # (10, self.customer_say, (6,"给我来杯酸奶和咖啡，哦对，再倒一杯水。")),
+            (6, self.customer_say, (6, "来杯酸奶吧。")),
+            (7, self.control_walker, (6, False, 100, -250, 480, 0)),  #(-100,600)
+
+            # 有人提出要开空调和关窗帘
+            # bar (60, 520)
+            (20, self.add_walker, (0, 0, 0)),
+            (25, self.control_walker, (7, False, 100, 60, 520, 180)),
+            (28, self.customer_say, (7,"好热呀！太阳也好大！")),
+            (40, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "这下舒服了！"]]])),
+
+            # (5, self.add_walker, (0, 0, 0)),
+            # (6, self.control_walker, (7, False, 100, 60, 520, 180)),
+            # (10, self.customer_say, (7, "好热呀！太阳也好大！")),
+            # (12, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "这下舒服了！"]]])),
+        ]
+
+    def _reset(self):
+        self.gen_obj()
+        self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)',
+         'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)',
+         'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
+         'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
+        # 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220)
+        self.add_walkers([[3,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
+        # [3,1, 880] 1号桌旁边小女孩
+        # [31,250, 1200] 最角落QuietTable1女红色
+        # [6,-55, 750] 1号桌附近小男孩
+        # [10,70, -200]  另一边角落 QuietTable2 男黄色
+        # [27,-290, 400, 180] 中间 BrightTable4 女灰
+        # [26, 60,-320,90] 另一边角落 BrightTable5 红胖男
+        self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
+
+        # 0-3男孩  4-7女孩 8-26男
+        # 3男孩 31女红  32女灰  10黄色衣服男瘦  9男灰瘦 26红胖男
+        # 17 是员工 police
+        # [0, -150,180]
+
+
+        # self.control_walkers(walker_loc=[[-55, 750]],is_autowalk = False)
+        # 在场景中随机增加一堆行人。
+        # walker_loc = [[-55, 750], [70, -200], [250, 1200], [0, 880]]
+        # controls = []
+        # for i in range(len(s.walkers)):
+        #     loc = walker_loc[i]
+        #     is_autowalk = False
+        #     pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180)
+        #     controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=200, pose=pose))
+        # scene = sim_client.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=scene_id))
+
+        # self.gen_obj(type=5)
+        # self.gen_obj(type=9)
+        # self.op_task_execute(op_type=16, obj_id=0)
+        # self.move_task_area(op_type=4)
+        pass
+
+    def _run(self, op_type=10):
+        # 共17个操作
+        # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子",    # 1-7
+        # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘",            # 8-12
+        # "调整空调开关","调高空调温度","调低空调温度",                   # 13-15
+        # "抓握物体","放置物体"                                       # 16-17
+
+        # self.gen_obj()
+        # if op_type <=15:
+        #     self.move_task_area(op_type)
+        #     self.op_task_execute(op_type)
+        # if op_type == 16:   # 16: 抓操作需要传入物品id
+        #     self.move_task_area(op_type, obj_id=0)
+        #     self.op_task_execute(op_type, obj_id=0)
+        # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107]
+        # # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0]   # 桌子2:[-55.0, 150.0, 107],
+        # if op_type == 17:   # 17: 放操作需要传入放置位置周围的可达区域
+        #     pos = [240.0, 40.0, 100.0]
+        #     self.move_task_area(op_type, release_pos=pos)
+        #     self.op_task_execute(op_type, release_pos=pos)   # [325.0, 860.0, 100]
+
+        # 流程测试
+        # 抓握放置:抓吧台前生成的酸奶，放到抹布桌上
+        # self.gen_obj()
+        # self.move_task_area(16, obj_id=0)
+        # self.op_task_execute(16, obj_id=0)
+        # pos = [340.0, 900.0, 99.0]
+        # self.move_task_area(17, release_pos=pos)
+        # self.op_task_execute(17, release_pos=pos)
+        #
+        # # 做咖啡:做完的咖啡放到水杯桌上
+        # self.move_task_area(1)
+        # self.op_task_execute(1)
+        #
+        # self.find_obj("CoffeeCup")
+        #
+        # self.move_task_area(16, obj_id=275)
+        # self.op_task_execute(16, obj_id=275)
+        # pos = [-70.0, 500.0, 107]
+        # self.move_task_area(17, release_pos=pos)
+        # self.op_task_execute(17, release_pos=pos)
+        #
+        # # 倒水:倒完的水放到旁边桌子上
+        # self.move_task_area(2)
+        # self.op_task_execute(2)
+
+        #
+        # self.move_task_area(16, obj_id=190)
+        # self.op_task_execute(16, obj_id=190)
+        # pos = [-55.0, 0.0, 107]
+        # self.move_task_area(17, release_pos=pos)
+        # self.op_task_execute(17, release_pos=pos)
+
+        # self.test_yaw()
+
+        pass
+    
+    def _step(self):
+        pass
+
+
+if __name__ == '__main__':
+    import os
+    from robowaiter.robot.robot import Robot
+
+    robot = Robot()
+
+    # create task
+    task = SceneVLM(robot)
+    task.reset()
+    task.run()

robowaiter/scene/tasks/VLM/VLN_greet_lead.py

@@ -12,7 +12,7 @@ class SceneVLM(Scene):
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.new_event_list = [
             (3, self.add_walker,  (0,60,520)),
-            (5, self.customer_say, (0,"请问可以带我去空位上嘛？我想晒太阳。")),
+            (5, self.customer_say, (0,"可以带我去空位上嘛？我想晒太阳。")),
         ]
 
     def _reset(self):
@@ -43,11 +43,7 @@ class SceneVLM(Scene):
         if end[1]>=600 or end[1]<=450 or end[0]>=250:
         # if int(self.status.location.X)!=247 or  int(self.status.location.X)!=520:
             self.walker_followed = True
-            self.control_walker(
+            self.control_walkers_and_say([[0,False,300,end[0],end[1],90,"谢谢！"]])
-                    [self.walker_control_generator(walkerID=0, autowalk=False, speed=300, X=end[0], Y=end[1], Yaw=-90)])
-
-            cont = self.status.walkers[0].name+"谢谢！"
-            self.control_robot_action(0,3,cont)
 
 
 if __name__ == '__main__':