更新 VLM2 机器人带小女孩到窗边的位置

robowaiter/behavior_lib/_base/Behavior.py

@@ -30,7 +30,7 @@ class Bahavior(ptree.behaviour.Behaviour):
     # all_place=set()
 
     place_xyz_dic={
-        'Bar': (247.0, 520.0, 100.0),
+        'Bar': (247.0, 520.0, 180.0), #(247.0, 520.0, 100.0)
         'Bar2': (240.0, 40.0, 70.0),
         'WaterTable':(-70.0, 500.0, 107),
         'CoffeeTable':(250.0, 310.0, 100.0),

robowaiter/llm_client/data/test_questions.txt

@@ -1 +1 @@
-{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "请问可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}
+{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}

robowaiter/llm_client/data_raw/test_questions.csv

@@ -21,4 +21,4 @@ Question,Answer,Goal
 可以带我去吗,当然可以，前往一号桌,"{""At(Robot,Table1)""}"
 把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。,明白，我这就去办！,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
 下班啦！打扫卫生，关灯关空调关窗帘。,太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
-请问可以带我去空位上嘛？我想晒太阳。,没问题！请跟我来。,"{""At(Robot,BrightTable1)""}"
+可以带我去空位上嘛？我想晒太阳。,没问题！请跟我来。,"{""At(Robot,BrightTable1)""}"

robowaiter/llm_client/data_raw/test_questions.csv~

@@ -0,0 +1,24 @@
+Question,Answer,Goal
+做一杯咖啡,OK，我这就去做一杯咖啡,"{""On(Coffee,CoffeeTable)""}"
+做一杯咖啡放到吧台上,OK，我这就去做一杯咖啡放到吧台上,"{""On(Coffee,Bar)""}"
+做一杯咖啡放到水杯桌上，再倒一杯水,OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水,"{""On(Coffee,WaterTable)"",""On(Water,WaterTable)""}"
+前往2号桌,OK，我这前往2号桌,"{""At(Robot,Table2)""}"
+测试AEM,测试AEM,"{""EnvExplored()""}"
+倒一杯水,OK，我这就去倒一杯水,"{""On(Water,WaterTable)""}"
+开空调,OK，我这就去开空调,"{""Is(AC,On)""}"
+关空调,OK，我这就去关空调,"{""Is(AC,Off)""}"
+关大厅灯,OK，我这就去关大厅灯,"{""Is(HallLight,Off)""}"
+开大厅灯,OK，我这就去开大厅灯,"{""Is(HallLight,On)""}"
+关筒灯,OK，我这就去关筒灯,"{""Is(TubeLight,Off)""}"
+开筒灯,OK，我这就去开筒灯,"{""Is(TubeLight,On)""}"
+关窗帘,OK，我这就去关窗帘,"{""Is(Curtain,Off)""}"
+开窗帘,OK，我这就去开窗帘,"{""Is(Curtain,On)""}"
+拖地,OK，我这就去拖地,"{""Is(Floor,Clean)""}"
+擦桌子,OK，我这就去擦桌子,"{""Is(Table1,Clean)""}"
+整理椅子,OK，我这就去整理椅子,"{""Is(Chairs,Clean)""}"
+把冰红茶放到Table2,OK，我这就去把冰红茶放到Table2,"{""On(BottledDrink,Table2)""}"
+我有点热，能开个空调吗？,当然可以，我现在就开！,"{""Is(AC,On)""}"
+可以带我去吗,当然可以，前往一号桌,"{""At(Robot,Table1)""}"
+把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。,明白，我这就去办！,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
+下班啦！打扫卫生，关灯关空调关窗帘。,太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
+请问可以带我去空位上嘛？我想晒太阳。,没问题！请跟我来。,"{""At(Robot,BrightTable1)""}"

robowaiter/scene/scene.py

@@ -164,6 +164,8 @@ class Scene:
 
         return customer_say
 
+
+
     def set_goal(self,goal):
         g = eval("{'" + goal + "'}")
         def set_sub_task():
@@ -277,6 +279,21 @@ class Scene:
             GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
         )
 
+    def control_walkers_and_say(self, control_list_ls):
+        """ 同时处理行人的行走和对话
+        control_list_ls =[walkerID,autowalk,speed,X,Y,Yaw,cont]
+        """
+        control_list= []
+        for control in control_list_ls:
+            if control[-1]!= None:
+                walkerID = control[0]
+                cont = self.status.walkers[walkerID].name + ":"+control[-1]
+                self.control_robot_action(control[walkerID], 3, cont)
+            control_list.append(self.walker_control_generator(walkerID=control[0], autowalk=control[1], speed=control[2], X=control[3], Y=control[4], Yaw=control[5]))
+        # 收集没有对话的统一控制
+        stub.ControlWalkers(
+            GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
+        )
 
     def control_walkers(self,walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True):
         """pose:表示行人的终止位置姿态"""

robowaiter/scene/tasks/VLM/VLM2.py

@@ -11,7 +11,7 @@ class SceneVLM(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.event_list = [
-            (5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
+            (5, self.create_chat_event("可以带我去空位上嘛？我想晒太阳。"))
         ]
 
     def _reset(self):
@@ -26,28 +26,24 @@ class SceneVLM(Scene):
         # 带领行人去有太阳的地方
         # 行人说 有点热
         # 好的，这就去开空调
+        self.control_walker(
+            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=0)])
+        time.sleep(3)
+        self.chat_bubble("欢迎光临！请问有什么可以帮您？")
+        time.sleep(3)
+        cont = self.status.walkers[0].name+":可以带我去空位上嘛？我想晒太阳。"
+        self.control_robot_action(0,3,cont)
+        time.sleep(4)
+
+        # self.event_list.append((5, self.create_chat_event("可以带我去空位上嘛？我想晒太阳。")))
+
         pass
     
     def _step(self):
-
-        self.control_walker(
-            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
-        time.sleep(3)
-        cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
-        self.control_robot_action(0,3,cont)
-
-
         # 如果机器人不在 吧台
-        # if "At(Robot,Bar)" not in self.state['condition_set']:
+        if "At(Robot,Bar)" not in self.state['condition_set']:
-        end = [self.status.location.X, self.status.location.Y]
+            end = [self.status.location.X, self.status.location.Y]
-        # print("end:",end)
+            self.control_walkers_and_say([[0, False, 100, end[0], end[1], -90, "谢谢！"]]) # walkerID,autowalk,speed,X,Y,Yaw,cont
-        if end[1]>=600 or end[1]<=450 or end[0]>=250:
-        # if int(self.status.location.X)!=247 or  int(self.status.location.X)!=520:
-            self.control_walker(
-                    [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=-90)])
-
-            cont = self.status.walkers[0].name+"谢谢！"
-            self.control_robot_action(0,3,cont)
         pass
 
 

robowaiter/scene/tasks/VLM/VLM3.py

@@ -11,7 +11,7 @@ class SceneVLM(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.event_list = [
-            (5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
+            (5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
         ]
 
     def _reset(self):