From aed3cd9aae89abb18160173ae0ab34b2e876ce91 Mon Sep 17 00:00:00 2001
From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com>
Date: Fri, 17 Nov 2023 20:04:39 +0800
Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20VLM2=20=E6=9C=BA=E5=99=A8?=
 =?UTF-8?q?=E4=BA=BA=E5=B8=A6=E5=B0=8F=E5=A5=B3=E5=AD=A9=E5=88=B0=E7=AA=97?=
 =?UTF-8?q?=E8=BE=B9=E7=9A=84=E4=BD=8D=E7=BD=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/behavior_lib/_base/Behavior.py     |  2 +-
 robowaiter/llm_client/data/test_questions.txt |  2 +-
 .../llm_client/data_raw/test_questions.csv    |  2 +-
 .../llm_client/data_raw/test_questions.csv~   | 24 +++++++++++++
 robowaiter/scene/scene.py                     | 17 ++++++++++
 robowaiter/scene/tasks/VLM/VLM2.py            | 34 ++++++++-----------
 robowaiter/scene/tasks/VLM/VLM3.py            |  2 +-
 7 files changed, 60 insertions(+), 23 deletions(-)
 create mode 100644 robowaiter/llm_client/data_raw/test_questions.csv~

diff --git a/robowaiter/behavior_lib/_base/Behavior.py b/robowaiter/behavior_lib/_base/Behavior.py
index 8baf2ca..728e258 100644
--- a/robowaiter/behavior_lib/_base/Behavior.py
+++ b/robowaiter/behavior_lib/_base/Behavior.py
@@ -30,7 +30,7 @@ class Bahavior(ptree.behaviour.Behaviour):
     # all_place=set()
 
     place_xyz_dic={
-        'Bar': (247.0, 520.0, 100.0),
+        'Bar': (247.0, 520.0, 180.0), #(247.0, 520.0, 100.0)
         'Bar2': (240.0, 40.0, 70.0),
         'WaterTable':(-70.0, 500.0, 107),
         'CoffeeTable':(250.0, 310.0, 100.0),
diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt
index a0ba46a..83bfc38 100644
--- a/robowaiter/llm_client/data/test_questions.txt
+++ b/robowaiter/llm_client/data/test_questions.txt
@@ -1 +1 @@
-{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "请问可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}
+{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}
diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv
index 4ce218c..dc3fcd9 100644
--- a/robowaiter/llm_client/data_raw/test_questions.csv
+++ b/robowaiter/llm_client/data_raw/test_questions.csv
@@ -21,4 +21,4 @@ Question,Answer,Goal
 ���Դ���ȥ��,��Ȼ���ԣ�ǰ��һ����,"{""At(Robot,Table1)""}"
 �����̷ŵ�1����������һ�������͵�ˮ�����ϣ��ٵ�һ��ˮ��,���ף������ȥ�죡,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
 �°�������ɨ�������صƹؿյ��ش�����,̫�������°����������ȥ��ɨ�������صƹؿյ��ش�����,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
-���ʿ��Դ���ȥ��λ�������ɹ̫����,û���⣡���������,"{""At(Robot,BrightTable1)""}"
+���Դ���ȥ��λ�������ɹ̫����,û���⣡���������,"{""At(Robot,BrightTable1)""}"
diff --git a/robowaiter/llm_client/data_raw/test_questions.csv~ b/robowaiter/llm_client/data_raw/test_questions.csv~
new file mode 100644
index 0000000..4ce218c
--- /dev/null
+++ b/robowaiter/llm_client/data_raw/test_questions.csv~
@@ -0,0 +1,24 @@
+Question,Answer,Goal
+��һ������,OK�������ȥ��һ������,"{""On(Coffee,CoffeeTable)""}"
+��һ�����ȷŵ���̨��,OK�������ȥ��һ�����ȷŵ���̨��,"{""On(Coffee,Bar)""}"
+��һ�����ȷŵ�ˮ�����ϣ��ٵ�һ��ˮ,OK�������ȥ��һ�����ȷŵ�ˮ�����ϣ��ٵ�һ��ˮ,"{""On(Coffee,WaterTable)"",""On(Water,WaterTable)""}"
+ǰ��2����,OK������ǰ��2����,"{""At(Robot,Table2)""}"
+����AEM,����AEM,"{""EnvExplored()""}"
+��һ��ˮ,OK�������ȥ��һ��ˮ,"{""On(Water,WaterTable)""}"
+���յ�,OK�������ȥ���յ�,"{""Is(AC,On)""}"
+�ؿյ�,OK�������ȥ�ؿյ�,"{""Is(AC,Off)""}"
+�ش�����,OK�������ȥ�ش�����,"{""Is(HallLight,Off)""}"
+��������,OK�������ȥ��������,"{""Is(HallLight,On)""}"
+��Ͳ��,OK�������ȥ��Ͳ��,"{""Is(TubeLight,Off)""}"
+��Ͳ��,OK�������ȥ��Ͳ��,"{""Is(TubeLight,On)""}"
+�ش���,OK�������ȥ�ش���,"{""Is(Curtain,Off)""}"
+������,OK�������ȥ������,"{""Is(Curtain,On)""}"
+�ϵ�,OK�������ȥ�ϵ�,"{""Is(Floor,Clean)""}"
+������,OK�������ȥ������,"{""Is(Table1,Clean)""}"
+��������,OK�������ȥ��������,"{""Is(Chairs,Clean)""}"
+�ѱ����ŵ�Table2,OK�������ȥ�ѱ����ŵ�Table2,"{""On(BottledDrink,Table2)""}"
+���е��ȣ��ܿ����յ���,��Ȼ���ԣ������ھͿ���,"{""Is(AC,On)""}"
+���Դ���ȥ��,��Ȼ���ԣ�ǰ��һ����,"{""At(Robot,Table1)""}"
+�����̷ŵ�1����������һ�������͵�ˮ�����ϣ��ٵ�һ��ˮ��,���ף������ȥ�죡,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
+�°�������ɨ�������صƹؿյ��ش�����,̫�������°����������ȥ��ɨ�������صƹؿյ��ش�����,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
+���ʿ��Դ���ȥ��λ�������ɹ̫����,û���⣡���������,"{""At(Robot,BrightTable1)""}"
diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py
index daa4c76..70633aa 100644
--- a/robowaiter/scene/scene.py
+++ b/robowaiter/scene/scene.py
@@ -164,6 +164,8 @@ class Scene:
 
         return customer_say
 
+
+
     def set_goal(self,goal):
         g = eval("{'" + goal + "'}")
         def set_sub_task():
@@ -277,6 +279,21 @@ class Scene:
             GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
         )
 
+    def control_walkers_and_say(self, control_list_ls):
+        """ 同时处理行人的行走和对话
+        control_list_ls =[walkerID,autowalk,speed,X,Y,Yaw,cont]
+        """
+        control_list= []
+        for control in control_list_ls:
+            if control[-1]!= None:
+                walkerID = control[0]
+                cont = self.status.walkers[walkerID].name + ":"+control[-1]
+                self.control_robot_action(control[walkerID], 3, cont)
+            control_list.append(self.walker_control_generator(walkerID=control[0], autowalk=control[1], speed=control[2], X=control[3], Y=control[4], Yaw=control[5]))
+        # 收集没有对话的统一控制
+        stub.ControlWalkers(
+            GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
+        )
 
     def control_walkers(self,walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True):
         """pose:表示行人的终止位置姿态"""
diff --git a/robowaiter/scene/tasks/VLM/VLM2.py b/robowaiter/scene/tasks/VLM/VLM2.py
index 1f23690..30066ff 100644
--- a/robowaiter/scene/tasks/VLM/VLM2.py
+++ b/robowaiter/scene/tasks/VLM/VLM2.py
@@ -11,7 +11,7 @@ class SceneVLM(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.event_list = [
-            (5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
+            (5, self.create_chat_event("可以带我去空位上嘛？我想晒太阳。"))
         ]
 
     def _reset(self):
@@ -26,28 +26,24 @@ class SceneVLM(Scene):
         # 带领行人去有太阳的地方
         # 行人说 有点热
         # 好的，这就去开空调
+        self.control_walker(
+            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=0)])
+        time.sleep(3)
+        self.chat_bubble("欢迎光临！请问有什么可以帮您？")
+        time.sleep(3)
+        cont = self.status.walkers[0].name+":可以带我去空位上嘛？我想晒太阳。"
+        self.control_robot_action(0,3,cont)
+        time.sleep(4)
+
+        # self.event_list.append((5, self.create_chat_event("可以带我去空位上嘛？我想晒太阳。")))
+
         pass
     
     def _step(self):
-
-        self.control_walker(
-            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
-        time.sleep(3)
-        cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
-        self.control_robot_action(0,3,cont)
-
-
         # 如果机器人不在 吧台
-        # if "At(Robot,Bar)" not in self.state['condition_set']:
-        end = [self.status.location.X, self.status.location.Y]
-        # print("end:",end)
-        if end[1]>=600 or end[1]<=450 or end[0]>=250:
-        # if int(self.status.location.X)!=247 or  int(self.status.location.X)!=520:
-            self.control_walker(
-                    [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=-90)])
-
-            cont = self.status.walkers[0].name+"谢谢！"
-            self.control_robot_action(0,3,cont)
+        if "At(Robot,Bar)" not in self.state['condition_set']:
+            end = [self.status.location.X, self.status.location.Y]
+            self.control_walkers_and_say([[0, False, 100, end[0], end[1], -90, "谢谢！"]]) # walkerID,autowalk,speed,X,Y,Yaw,cont
         pass
 
 
diff --git a/robowaiter/scene/tasks/VLM/VLM3.py b/robowaiter/scene/tasks/VLM/VLM3.py
index f83416c..802919c 100644
--- a/robowaiter/scene/tasks/VLM/VLM3.py
+++ b/robowaiter/scene/tasks/VLM/VLM3.py
@@ -11,7 +11,7 @@ class SceneVLM(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.event_list = [
-            (5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
+            (5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
         ]
 
     def _reset(self):