From aed3cd9aae89abb18160173ae0ab34b2e876ce91 Mon Sep 17 00:00:00 2001
From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com>
Date: Fri, 17 Nov 2023 20:04:39 +0800
Subject: [PATCH 01/12] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20VLM2=20=E6=9C=BA?=
 =?UTF-8?q?=E5=99=A8=E4=BA=BA=E5=B8=A6=E5=B0=8F=E5=A5=B3=E5=AD=A9=E5=88=B0?=
 =?UTF-8?q?=E7=AA=97=E8=BE=B9=E7=9A=84=E4=BD=8D=E7=BD=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/behavior_lib/_base/Behavior.py     |  2 +-
 robowaiter/llm_client/data/test_questions.txt |  2 +-
 .../llm_client/data_raw/test_questions.csv    |  2 +-
 .../llm_client/data_raw/test_questions.csv~   | 24 +++++++++++++
 robowaiter/scene/scene.py                     | 17 ++++++++++
 robowaiter/scene/tasks/VLM/VLM2.py            | 34 ++++++++-----------
 robowaiter/scene/tasks/VLM/VLM3.py            |  2 +-
 7 files changed, 60 insertions(+), 23 deletions(-)
 create mode 100644 robowaiter/llm_client/data_raw/test_questions.csv~

diff --git a/robowaiter/behavior_lib/_base/Behavior.py b/robowaiter/behavior_lib/_base/Behavior.py
index 8baf2ca..728e258 100644
--- a/robowaiter/behavior_lib/_base/Behavior.py
+++ b/robowaiter/behavior_lib/_base/Behavior.py
@@ -30,7 +30,7 @@ class Bahavior(ptree.behaviour.Behaviour):
     # all_place=set()
 
     place_xyz_dic={
-        'Bar': (247.0, 520.0, 100.0),
+        'Bar': (247.0, 520.0, 180.0), #(247.0, 520.0, 100.0)
         'Bar2': (240.0, 40.0, 70.0),
         'WaterTable':(-70.0, 500.0, 107),
         'CoffeeTable':(250.0, 310.0, 100.0),
diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt
index a0ba46a..83bfc38 100644
--- a/robowaiter/llm_client/data/test_questions.txt
+++ b/robowaiter/llm_client/data/test_questions.txt
@@ -1 +1 @@
-{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "请问可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}
+{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}
diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv
index 4ce218c..dc3fcd9 100644
--- a/robowaiter/llm_client/data_raw/test_questions.csv
+++ b/robowaiter/llm_client/data_raw/test_questions.csv
@@ -21,4 +21,4 @@ Question,Answer,Goal
 ���Դ���ȥ��,��Ȼ���ԣ�ǰ��һ����,"{""At(Robot,Table1)""}"
 �����̷ŵ�1����������һ�������͵�ˮ�����ϣ��ٵ�һ��ˮ��,���ף������ȥ�죡,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
 �°�������ɨ�������صƹؿյ��ش�����,̫�������°����������ȥ��ɨ�������صƹؿյ��ش�����,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
-���ʿ��Դ���ȥ��λ�������ɹ̫����,û���⣡���������,"{""At(Robot,BrightTable1)""}"
+���Դ���ȥ��λ�������ɹ̫����,û���⣡���������,"{""At(Robot,BrightTable1)""}"
diff --git a/robowaiter/llm_client/data_raw/test_questions.csv~ b/robowaiter/llm_client/data_raw/test_questions.csv~
new file mode 100644
index 0000000..4ce218c
--- /dev/null
+++ b/robowaiter/llm_client/data_raw/test_questions.csv~
@@ -0,0 +1,24 @@
+Question,Answer,Goal
+��һ������,OK�������ȥ��һ������,"{""On(Coffee,CoffeeTable)""}"
+��һ�����ȷŵ���̨��,OK�������ȥ��һ�����ȷŵ���̨��,"{""On(Coffee,Bar)""}"
+��һ�����ȷŵ�ˮ�����ϣ��ٵ�һ��ˮ,OK�������ȥ��һ�����ȷŵ�ˮ�����ϣ��ٵ�һ��ˮ,"{""On(Coffee,WaterTable)"",""On(Water,WaterTable)""}"
+ǰ��2����,OK������ǰ��2����,"{""At(Robot,Table2)""}"
+����AEM,����AEM,"{""EnvExplored()""}"
+��һ��ˮ,OK�������ȥ��һ��ˮ,"{""On(Water,WaterTable)""}"
+���յ�,OK�������ȥ���յ�,"{""Is(AC,On)""}"
+�ؿյ�,OK�������ȥ�ؿյ�,"{""Is(AC,Off)""}"
+�ش�����,OK�������ȥ�ش�����,"{""Is(HallLight,Off)""}"
+��������,OK�������ȥ��������,"{""Is(HallLight,On)""}"
+��Ͳ��,OK�������ȥ��Ͳ��,"{""Is(TubeLight,Off)""}"
+��Ͳ��,OK�������ȥ��Ͳ��,"{""Is(TubeLight,On)""}"
+�ش���,OK�������ȥ�ش���,"{""Is(Curtain,Off)""}"
+������,OK�������ȥ������,"{""Is(Curtain,On)""}"
+�ϵ�,OK�������ȥ�ϵ�,"{""Is(Floor,Clean)""}"
+������,OK�������ȥ������,"{""Is(Table1,Clean)""}"
+��������,OK�������ȥ��������,"{""Is(Chairs,Clean)""}"
+�ѱ����ŵ�Table2,OK�������ȥ�ѱ����ŵ�Table2,"{""On(BottledDrink,Table2)""}"
+���е��ȣ��ܿ����յ���,��Ȼ���ԣ������ھͿ���,"{""Is(AC,On)""}"
+���Դ���ȥ��,��Ȼ���ԣ�ǰ��һ����,"{""At(Robot,Table1)""}"
+�����̷ŵ�1����������һ�������͵�ˮ�����ϣ��ٵ�һ��ˮ��,���ף������ȥ�죡,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
+�°�������ɨ�������صƹؿյ��ش�����,̫�������°����������ȥ��ɨ�������صƹؿյ��ش�����,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
+���ʿ��Դ���ȥ��λ�������ɹ̫����,û���⣡���������,"{""At(Robot,BrightTable1)""}"
diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py
index daa4c76..70633aa 100644
--- a/robowaiter/scene/scene.py
+++ b/robowaiter/scene/scene.py
@@ -164,6 +164,8 @@ class Scene:
 
         return customer_say
 
+
+
     def set_goal(self,goal):
         g = eval("{'" + goal + "'}")
         def set_sub_task():
@@ -277,6 +279,21 @@ class Scene:
             GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
         )
 
+    def control_walkers_and_say(self, control_list_ls):
+        """ 同时处理行人的行走和对话
+        control_list_ls =[walkerID,autowalk,speed,X,Y,Yaw,cont]
+        """
+        control_list= []
+        for control in control_list_ls:
+            if control[-1]!= None:
+                walkerID = control[0]
+                cont = self.status.walkers[walkerID].name + ":"+control[-1]
+                self.control_robot_action(control[walkerID], 3, cont)
+            control_list.append(self.walker_control_generator(walkerID=control[0], autowalk=control[1], speed=control[2], X=control[3], Y=control[4], Yaw=control[5]))
+        # 收集没有对话的统一控制
+        stub.ControlWalkers(
+            GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
+        )
 
     def control_walkers(self,walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True):
         """pose:表示行人的终止位置姿态"""
diff --git a/robowaiter/scene/tasks/VLM/VLM2.py b/robowaiter/scene/tasks/VLM/VLM2.py
index 1f23690..30066ff 100644
--- a/robowaiter/scene/tasks/VLM/VLM2.py
+++ b/robowaiter/scene/tasks/VLM/VLM2.py
@@ -11,7 +11,7 @@ class SceneVLM(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.event_list = [
-            (5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
+            (5, self.create_chat_event("可以带我去空位上嘛？我想晒太阳。"))
         ]
 
     def _reset(self):
@@ -26,28 +26,24 @@ class SceneVLM(Scene):
         # 带领行人去有太阳的地方
         # 行人说 有点热
         # 好的，这就去开空调
+        self.control_walker(
+            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=0)])
+        time.sleep(3)
+        self.chat_bubble("欢迎光临！请问有什么可以帮您？")
+        time.sleep(3)
+        cont = self.status.walkers[0].name+":可以带我去空位上嘛？我想晒太阳。"
+        self.control_robot_action(0,3,cont)
+        time.sleep(4)
+
+        # self.event_list.append((5, self.create_chat_event("可以带我去空位上嘛？我想晒太阳。")))
+
         pass
     
     def _step(self):
-
-        self.control_walker(
-            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
-        time.sleep(3)
-        cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
-        self.control_robot_action(0,3,cont)
-
-
         # 如果机器人不在 吧台
-        # if "At(Robot,Bar)" not in self.state['condition_set']:
-        end = [self.status.location.X, self.status.location.Y]
-        # print("end:",end)
-        if end[1]>=600 or end[1]<=450 or end[0]>=250:
-        # if int(self.status.location.X)!=247 or  int(self.status.location.X)!=520:
-            self.control_walker(
-                    [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=-90)])
-
-            cont = self.status.walkers[0].name+"谢谢！"
-            self.control_robot_action(0,3,cont)
+        if "At(Robot,Bar)" not in self.state['condition_set']:
+            end = [self.status.location.X, self.status.location.Y]
+            self.control_walkers_and_say([[0, False, 100, end[0], end[1], -90, "谢谢！"]]) # walkerID,autowalk,speed,X,Y,Yaw,cont
         pass
 
 
diff --git a/robowaiter/scene/tasks/VLM/VLM3.py b/robowaiter/scene/tasks/VLM/VLM3.py
index f83416c..802919c 100644
--- a/robowaiter/scene/tasks/VLM/VLM3.py
+++ b/robowaiter/scene/tasks/VLM/VLM3.py
@@ -11,7 +11,7 @@ class SceneVLM(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.event_list = [
-            (5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
+            (5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
         ]
 
     def _reset(self):

From 40642b86a0f2c3c575266a5000c0e16e72c604a9 Mon Sep 17 00:00:00 2001
From: ChenXL97 <908926798@qq.com>
Date: Sat, 18 Nov 2023 14:13:07 +0800
Subject: [PATCH 02/12] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E5=8D=95=E6=AC=A1?=
 =?UTF-8?q?=E5=B8=A6=E8=B7=AFVLN=5Fgreet=5Flead?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 requirements.txt                              |  4 ----
 robowaiter/behavior_lib/act/DealChat.py       | 12 ++++------
 robowaiter/behavior_lib/cond/Chatting.py      |  9 ++++++++
 robowaiter/scene/scene.py                     | 10 ++++----
 .../tasks/VLM/{VLM2.py => VLN_greet_lead.py}  | 23 +++++++++++--------
 5 files changed, 34 insertions(+), 24 deletions(-)
 rename robowaiter/scene/tasks/VLM/{VLM2.py => VLN_greet_lead.py} (70%)

diff --git a/requirements.txt b/requirements.txt
index 5dc0fee..c43674f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,10 +8,6 @@ requests~=2.31.0
 urllib3~=2.0.7
 tabulate~=0.9.0
 autopep8
-pytorch==1.11.0
-torchvision==0.12.0
-torchaudio==0.11.0
-cudatoolkit=11.3
 loguru~=0.5.3
 matplotlib~=3.8.0
 numpy~=1.26.0
diff --git a/robowaiter/behavior_lib/act/DealChat.py b/robowaiter/behavior_lib/act/DealChat.py
index bb69360..fa4cf11 100644
--- a/robowaiter/behavior_lib/act/DealChat.py
+++ b/robowaiter/behavior_lib/act/DealChat.py
@@ -10,18 +10,16 @@ class DealChat(Act):
 
     def _update(self) -> ptree.common.Status:
         # if self.scene.status?
-        name,sentence = self.scene.state['chat_list'][0]
+        name,sentence = self.scene.state['chat_list'].pop(0)
 
-
-        chat = self.scene.state['chat_list'].pop()
-        if isinstance(chat,set):
-            self.create_sub_task(chat)
+        if name == "Goal":
+            self.create_sub_task(sentence)
             return ptree.common.Status.RUNNING
 
 
-        self.chat_history += chat + '\n'
+        self.chat_history += sentence + '\n'
 
-        res_dict = ask_llm(chat)
+        res_dict = ask_llm(sentence)
         answer = res_dict["Answer"]
         self.scene.chat_bubble(answer) # 机器人输出对话
         self.chat_history += answer + '\n'
diff --git a/robowaiter/behavior_lib/cond/Chatting.py b/robowaiter/behavior_lib/cond/Chatting.py
index 6e95def..ec83aa2 100644
--- a/robowaiter/behavior_lib/cond/Chatting.py
+++ b/robowaiter/behavior_lib/cond/Chatting.py
@@ -13,5 +13,14 @@ class Chatting(Cond):
         # if self.scene.status?
         if self.scene.state['chat_list'] == []:
             return ptree.common.Status.FAILURE
+
+        name,sentence = self.scene.state['chat_list'][0]
+        if "customer" in self.scene.state["attention"]:
+            attention_customer = self.scene.state["attention"]["customer"]
+            if name == attention_customer:
+                return ptree.common.Status.SUCCESS
+            else:
+                return ptree.common.Status.FAILURE
         else:
+            self.scene.state["attention"]["customer"] = name
             return ptree.common.Status.SUCCESS
diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py
index 0cc08b7..7c2370d 100644
--- a/robowaiter/scene/scene.py
+++ b/robowaiter/scene/scene.py
@@ -9,7 +9,7 @@ from robowaiter.proto import semantic_map
 import math
 from robowaiter.proto import GrabSim_pb2
 from robowaiter.proto import GrabSim_pb2_grpc
-
+import copy
 import os
 from robowaiter.utils import get_root_path
 root_path = get_root_path()
@@ -127,7 +127,7 @@ class Scene:
         self.reset_sim()
 
         # reset state
-        self.state = self.default_state
+        self.state = copy.deepcopy(self.default_state)
 
 
 
@@ -184,13 +184,13 @@ class Scene:
     def set_goal(self,goal):
         g = eval("{'" + goal + "'}")
         def set_sub_task():
-            self.state['chat_list'].append(g)
+            self.state['chat_list'].append(("Goal",g))
 
         return set_sub_task
 
     def new_set_goal(self,goal):
         g = eval("{'" + goal + "'}")
-        self.state['chat_list'].append(g)
+        self.state['chat_list'].append(("Goal",g))
 
 
     @property
@@ -429,6 +429,8 @@ class Scene:
         self.control_robot_action(0, 3, talk_content)
 
     def customer_say(self,name,sentence,show_bubble=True):
+        if isinstance(name,int):
+            name = self.walker_index2mem(name)
         print(f'{name} say: {sentence}')
         if show_bubble:
             self.walker_bubble(name,sentence)
diff --git a/robowaiter/scene/tasks/VLM/VLM2.py b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
similarity index 70%
rename from robowaiter/scene/tasks/VLM/VLM2.py
rename to robowaiter/scene/tasks/VLM/VLN_greet_lead.py
index 1f23690..6d4c090 100644
--- a/robowaiter/scene/tasks/VLM/VLM2.py
+++ b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
@@ -10,13 +10,14 @@ class SceneVLM(Scene):
     def __init__(self, robot):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
-        self.event_list = [
-            (5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
+        self.new_event_list = [
+            (3, self.add_walker,  (0,60,520)),
+            (5, self.customer_say, (0,"请问可以带我去空位上嘛？我想晒太阳。")),
         ]
 
     def _reset(self):
         self.gen_obj()
-        self.add_walkers([[47, 920]])
+        # self.add_walkers([[47, 920]])
         pass
 
     def _run(self, op_type=10):
@@ -26,29 +27,33 @@ class SceneVLM(Scene):
         # 带领行人去有太阳的地方
         # 行人说 有点热
         # 好的，这就去开空调
+        self.walker_followed = False
         pass
     
     def _step(self):
 
-        self.control_walker(
-            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
-        time.sleep(3)
-        cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
-        self.control_robot_action(0,3,cont)
+        # self.control_walker(
+        #     [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
+        # time.sleep(3)
+        # cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
+        # self.control_robot_action(0,3,cont)
 
 
         # 如果机器人不在 吧台
         # if "At(Robot,Bar)" not in self.state['condition_set']:
+        if self.walker_followed:
+            return
+
         end = [self.status.location.X, self.status.location.Y]
         # print("end:",end)
         if end[1]>=600 or end[1]<=450 or end[0]>=250:
         # if int(self.status.location.X)!=247 or  int(self.status.location.X)!=520:
+            self.walker_followed = True
             self.control_walker(
                     [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=-90)])
 
             cont = self.status.walkers[0].name+"谢谢！"
             self.control_robot_action(0,3,cont)
-        pass
 
 
 if __name__ == '__main__':

From f637e871bad2fa5ce4207036e1f7ddffffe10e26 Mon Sep 17 00:00:00 2001
From: ChenXL97 <908926798@qq.com>
Date: Sat, 18 Nov 2023 14:51:17 +0800
Subject: [PATCH 03/12] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E9=A1=BE=E5=AE=A2?=
 =?UTF-8?q?=E9=A2=86=E8=B7=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/behavior_lib/act/DealChat.py          |  5 +++++
 robowaiter/behavior_lib/act/DelSubTree.py        |  2 ++
 robowaiter/behavior_lib/act/GreetCustomer.py     |  2 ++
 robowaiter/behavior_lib/act/ServeCustomer.py     |  5 ++---
 .../cond/{Chatting.py => CustomerChatting.py}    |  2 +-
 robowaiter/behavior_lib/cond/FocusingCustomer.py | 16 ++++++++++++++++
 .../{DetectCustomer.py => NewCustomerComing.py}  |  2 +-
 robowaiter/robot/Default.ptml                    | 11 ++++++++---
 robowaiter/robot/robot.py                        |  3 +--
 robowaiter/scene/scene.py                        |  3 ++-
 robowaiter/scene/tasks/VLM/VLN_greet_lead.py     |  8 +-------
 11 files changed, 41 insertions(+), 18 deletions(-)
 rename robowaiter/behavior_lib/cond/{Chatting.py => CustomerChatting.py} (96%)
 create mode 100644 robowaiter/behavior_lib/cond/FocusingCustomer.py
 rename robowaiter/behavior_lib/cond/{DetectCustomer.py => NewCustomerComing.py} (93%)

diff --git a/robowaiter/behavior_lib/act/DealChat.py b/robowaiter/behavior_lib/act/DealChat.py
index fa4cf11..a47ea66 100644
--- a/robowaiter/behavior_lib/act/DealChat.py
+++ b/robowaiter/behavior_lib/act/DealChat.py
@@ -16,6 +16,11 @@ class DealChat(Act):
             self.create_sub_task(sentence)
             return ptree.common.Status.RUNNING
 
+        self.scene.state["attention"]["customer"] = name
+        self.scene.state["serve_state"] = {
+            "last_chat_time": self.scene.time,
+        }
+
 
         self.chat_history += sentence + '\n'
 
diff --git a/robowaiter/behavior_lib/act/DelSubTree.py b/robowaiter/behavior_lib/act/DelSubTree.py
index 9c43c00..f139eed 100644
--- a/robowaiter/behavior_lib/act/DelSubTree.py
+++ b/robowaiter/behavior_lib/act/DelSubTree.py
@@ -9,6 +9,8 @@ class DelSubTree(Act):
         super().__init__(*args)
 
     def _update(self) -> ptree.common.Status:
+        self.scene.state["attention"] = {}
+
         sub_task_tree = self.parent
         self.scene.sub_task_seq.children.remove(sub_task_tree)
         return Status.RUNNING
\ No newline at end of file
diff --git a/robowaiter/behavior_lib/act/GreetCustomer.py b/robowaiter/behavior_lib/act/GreetCustomer.py
index 9297dd4..94ac629 100644
--- a/robowaiter/behavior_lib/act/GreetCustomer.py
+++ b/robowaiter/behavior_lib/act/GreetCustomer.py
@@ -28,4 +28,6 @@ class GreetCustomer(Act):
         customer_name = self.scene.state['attention']['customer']
         self.scene.state['greeted_customers'].add(customer_name)
 
+
+
         return ptree.common.Status.RUNNING
diff --git a/robowaiter/behavior_lib/act/ServeCustomer.py b/robowaiter/behavior_lib/act/ServeCustomer.py
index 1b4312e..0c9f888 100644
--- a/robowaiter/behavior_lib/act/ServeCustomer.py
+++ b/robowaiter/behavior_lib/act/ServeCustomer.py
@@ -3,9 +3,6 @@ from robowaiter.behavior_lib._base.Act import Act
 from robowaiter.algos.navigator.navigate import Navigator
 
 class ServeCustomer(Act):
-    can_be_expanded = False
-    num_args = 0
-    valid_args = ()
 
     def __init__(self, *args):
         super().__init__(*args)
@@ -20,6 +17,8 @@ class ServeCustomer(Act):
         return info
 
     def _update(self) -> ptree.common.Status:
+        # if self.scene.time - self.scene.state["serve_state"]["last_chat_time"] > 10:
+        #     self.chat_bubble
 
         goal = Act.place_xyz_dic['Bar']
         self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0)
diff --git a/robowaiter/behavior_lib/cond/Chatting.py b/robowaiter/behavior_lib/cond/CustomerChatting.py
similarity index 96%
rename from robowaiter/behavior_lib/cond/Chatting.py
rename to robowaiter/behavior_lib/cond/CustomerChatting.py
index ec83aa2..5d26c85 100644
--- a/robowaiter/behavior_lib/cond/Chatting.py
+++ b/robowaiter/behavior_lib/cond/CustomerChatting.py
@@ -2,7 +2,7 @@ import py_trees as ptree
 from typing import Any
 from robowaiter.behavior_lib._base.Cond import Cond
 
-class Chatting(Cond):
+class CustomerChatting(Cond):
 
 
     def __init__(self):
diff --git a/robowaiter/behavior_lib/cond/FocusingCustomer.py b/robowaiter/behavior_lib/cond/FocusingCustomer.py
new file mode 100644
index 0000000..9420c1e
--- /dev/null
+++ b/robowaiter/behavior_lib/cond/FocusingCustomer.py
@@ -0,0 +1,16 @@
+import py_trees as ptree
+from typing import Any
+from robowaiter.behavior_lib._base.Cond import Cond
+
+class FocusingCustomer(Cond):
+
+    def __init__(self):
+        super().__init__()
+
+
+    def _update(self) -> ptree.common.Status:
+        # if self.scene.status?
+        if "customer" in self.scene.state['attention']:
+            return ptree.common.Status.SUCCESS
+        else:
+            return ptree.common.Status.FAILURE
diff --git a/robowaiter/behavior_lib/cond/DetectCustomer.py b/robowaiter/behavior_lib/cond/NewCustomerComing.py
similarity index 93%
rename from robowaiter/behavior_lib/cond/DetectCustomer.py
rename to robowaiter/behavior_lib/cond/NewCustomerComing.py
index 7595f63..a77c190 100644
--- a/robowaiter/behavior_lib/cond/DetectCustomer.py
+++ b/robowaiter/behavior_lib/cond/NewCustomerComing.py
@@ -3,7 +3,7 @@ from typing import Any
 from robowaiter.behavior_lib._base.Cond import Cond
 import itertools
 
-class DetectCustomer(Cond):
+class NewCustomerComing(Cond):
     can_be_expanded = False
     num_params = 0
     valid_args = ()
diff --git a/robowaiter/robot/Default.ptml b/robowaiter/robot/Default.ptml
index e1a98fa..b498f75 100644
--- a/robowaiter/robot/Default.ptml
+++ b/robowaiter/robot/Default.ptml
@@ -2,9 +2,8 @@ selector
 {
     sequence
     {
-        cond Chatting()
+        cond CustomerChatting()
         act DealChat()
-
     }
     sequence
     {
@@ -16,7 +15,12 @@ selector
     }
     sequence
     {
-        cond DetectCustomer()
+        cond FocusingCustomer()
+        act ServeCustomer()
+    }
+    sequence
+    {
+        cond NewCustomerComing()
         selector
         {
             cond At(Robot,Bar)
@@ -24,4 +28,5 @@ selector
         }
         act GreetCustomer()
     }
+
 }
\ No newline at end of file
diff --git a/robowaiter/robot/robot.py b/robowaiter/robot/robot.py
index bc79121..40af1af 100644
--- a/robowaiter/robot/robot.py
+++ b/robowaiter/robot/robot.py
@@ -41,7 +41,6 @@ class Robot(object):
     def load_BT(self):
         self.bt = load_bt_from_ptml(self.scene, self.ptml_path,self.behavior_lib_path)
         sub_task_place_holder = find_node_by_name(self.bt.root,"SubTaskPlaceHolder()")
-        print(sub_task_place_holder)
         if sub_task_place_holder:
             sub_task_seq = sub_task_place_holder.parent
             sub_task_seq.children.pop()
@@ -53,7 +52,7 @@ class Robot(object):
 
     def expand_sub_task_tree(self,goal):
         if self.action_list is None:
-            print("\n--------------------")
+            print("\n\n--------------------")
             print(f"首次运行行为树扩展算法")
             self.action_list = self.collect_action_nodes()
             print(f"共收集到{len(self.action_list)}个实例化动作:")
diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py
index 6d15ef4..4acb354 100644
--- a/robowaiter/scene/scene.py
+++ b/robowaiter/scene/scene.py
@@ -69,7 +69,8 @@ class Scene:
         "customer_mem":{},
         "served_mem":{},
         "greeted_customers":set(),
-        "attention":{}
+        "attention":{},
+        "serve_state":{},
     }
     """
     status:
diff --git a/robowaiter/scene/tasks/VLM/VLN_greet_lead.py b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
index 6d4c090..39ac481 100644
--- a/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
+++ b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
@@ -32,12 +32,6 @@ class SceneVLM(Scene):
     
     def _step(self):
 
-        # self.control_walker(
-        #     [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
-        # time.sleep(3)
-        # cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
-        # self.control_robot_action(0,3,cont)
-
 
         # 如果机器人不在 吧台
         # if "At(Robot,Bar)" not in self.state['condition_set']:
@@ -50,7 +44,7 @@ class SceneVLM(Scene):
         # if int(self.status.location.X)!=247 or  int(self.status.location.X)!=520:
             self.walker_followed = True
             self.control_walker(
-                    [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=-90)])
+                    [self.walker_control_generator(walkerID=0, autowalk=False, speed=300, X=end[0], Y=end[1], Yaw=-90)])
 
             cont = self.status.walkers[0].name+"谢谢！"
             self.control_robot_action(0,3,cont)

From 25902b618219e0fecfb16c628935c372488f9d56 Mon Sep 17 00:00:00 2001
From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com>
Date: Sat, 18 Nov 2023 17:20:13 +0800
Subject: [PATCH 04/12] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E9=A1=BE?=
 =?UTF-8?q?=E5=AE=A2=E6=8F=90=E5=87=BA=E7=82=B9=E5=8D=95=E5=92=8C=E6=8F=90?=
 =?UTF-8?q?=E5=87=BA=E5=BC=80=E5=85=B3=E7=A9=BA=E8=B0=83=E7=9A=84=E5=9C=BA?=
 =?UTF-8?q?=E6=99=AF=20VLM=5Forder?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/behavior_lib/act/GreetCustomer.py  |   3 +-
 robowaiter/llm_client/data/test_questions.txt |   2 +-
 .../llm_client/data_raw/test_questions.csv    |   3 +
 robowaiter/scene/scene.py                     |  31 ++--
 robowaiter/scene/tasks/VLM/VLM_order.py       | 142 ++++++++++++++++++
 robowaiter/scene/tasks/VLM/VLN_greet_lead.py  |   8 +-
 6 files changed, 169 insertions(+), 20 deletions(-)
 create mode 100644 robowaiter/scene/tasks/VLM/VLM_order.py

diff --git a/robowaiter/behavior_lib/act/GreetCustomer.py b/robowaiter/behavior_lib/act/GreetCustomer.py
index 94ac629..62dae5a 100644
--- a/robowaiter/behavior_lib/act/GreetCustomer.py
+++ b/robowaiter/behavior_lib/act/GreetCustomer.py
@@ -23,7 +23,8 @@ class GreetCustomer(Act):
 
         goal = Act.place_xyz_dic['Bar']
         self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0)
-        self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？")
+        # self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？")
+        self.scene.chat_bubble("欢迎光临！")
 
         customer_name = self.scene.state['attention']['customer']
         self.scene.state['greeted_customers'].add(customer_name)
diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt
index 83bfc38..d950325 100644
--- a/robowaiter/llm_client/data/test_questions.txt
+++ b/robowaiter/llm_client/data/test_questions.txt
@@ -1 +1 @@
-{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}
+{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}, "给我来杯酸奶和咖啡，哦对，再倒一杯水。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,WaterTable)\",\"On(Water,WaterTable)\"}"}, "好热呀！太阳也好大！": {"Answer": "抱歉！我这就去开空调和关窗帘！", "Goal": "{\"Is(AC,On)\",\"Is(Curtain,Off)\"}"}, "来杯酸奶吧。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Yogurt,WaterTable)\"}"}}
diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv
index dc3fcd9..a89b902 100644
--- a/robowaiter/llm_client/data_raw/test_questions.csv
+++ b/robowaiter/llm_client/data_raw/test_questions.csv
@@ -22,3 +22,6 @@ Question,Answer,Goal
 �����̷ŵ�1����������һ�������͵�ˮ�����ϣ��ٵ�һ��ˮ��,���ף������ȥ�죡,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
 �°�������ɨ�������صƹؿյ��ش�����,̫�������°����������ȥ��ɨ�������صƹؿյ��ش�����,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
 ���Դ���ȥ��λ�������ɹ̫����,û���⣡���������,"{""At(Robot,BrightTable1)""}"
+�����������̺Ϳ��ȣ�Ŷ�ԣ��ٵ�һ��ˮ��,���ϣ����Եȣ�,"{""On(Coffee,WaterTable)"",""On(Yogurt,WaterTable)"",""On(Water,WaterTable)""}"
+����ѽ��̫��Ҳ�ô�,��Ǹ�������ȥ���յ��͹ش�����,"{""Is(AC,On)"",""Is(Curtain,Off)""}"
+�������̰ɡ�,���ϣ����Եȣ�,"{""On(Yogurt,WaterTable)""}"
diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py
index 32bf629..f92c1c1 100644
--- a/robowaiter/scene/scene.py
+++ b/robowaiter/scene/scene.py
@@ -286,15 +286,15 @@ class Scene:
 
     def add_walkers(self,walker_loc=[[0, 880], [250, 1200], [-55, 750], [70, -200]]):
         print('------------------add_walkers----------------------')
-        for id,walker in enumerate(walker_loc):
+        for i,walker in enumerate(walker_loc):
             if len(walker)==2:
-                self.add_walker(id,walker[0],walker[1])
+                self.add_walker(i,walker[0],walker[1])
             elif len(walker)==3:
-                self.add_walker(id, walker[0], walker[1],walker[2])
+                self.add_walker(walker[0], walker[1], walker[2])
             elif len(walker) == 4:
-                self.add_walker(id, walker[0], walker[1], walker[2], walker[3])
+                self.add_walker(walker[0], walker[1], walker[2], walker[3])
             elif len(walker) == 5:
-                self.add_walker(id, walker[0], walker[1], walker[2], walker[3], walker[4])
+                self.add_walker(walker[0], walker[1], walker[2], walker[3], walker[4])
 
     def remove_walker(self, *args):  # take single walkerID or a list of walkerIDs
         remove_list = []
@@ -318,10 +318,15 @@ class Scene:
     def clean_walker(self):
         stub.CleanWalkers(GrabSim_pb2.SceneID(value=self.sceneID))
 
-    def control_walker(self, control_list):
-        stub.ControlWalkers(
-            GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
+    def control_walker(self, walkerID,autowalk,speed,X,Y,Yaw=0):
+        pose = GrabSim_pb2.Pose(X=X, Y=Y, Yaw=Yaw)
+        scene = stub.ControlWalkers(
+            GrabSim_pb2.WalkerControls(controls=[GrabSim_pb2.WalkerControls.WControl(id=walkerID, autowalk=autowalk, speed=speed, pose=pose)], scene=self.sceneID)
         )
+        return scene
+        # stub.ControlWalkers(
+        #     GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
+        # )
 
     def control_walkers_and_say(self, control_list_ls):
         """ 同时处理行人的行走和对话
@@ -331,13 +336,15 @@ class Scene:
         for control in control_list_ls:
             if control[-1]!= None:
                 walkerID = control[0]
-                cont = self.status.walkers[walkerID].name + ":"+control[-1]
-                self.control_robot_action(control[walkerID], 3, cont)
+                # cont = self.status.walkers[walkerID].name + ":"+control[-1]
+                # self.control_robot_action(control[walkerID], 3, cont)
+                self.customer_say(walkerID,control[-1])
             control_list.append(self.walker_control_generator(walkerID=control[0], autowalk=control[1], speed=control[2], X=control[3], Y=control[4], Yaw=control[5]))
         # 收集没有对话的统一控制
-        stub.ControlWalkers(
+        scene = stub.ControlWalkers(
             GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
         )
+        return scene
 
     def control_walkers(self,walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True):
         """pose:表示行人的终止位置姿态"""
@@ -350,7 +357,7 @@ class Scene:
             pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180)
             controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=80, pose=pose))
         scene = stub.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=self.sceneID))
-
+        return scene
 
     def control_joints(self, angles):
         stub.Do(
diff --git a/robowaiter/scene/tasks/VLM/VLM_order.py b/robowaiter/scene/tasks/VLM/VLM_order.py
new file mode 100644
index 0000000..9e13afa
--- /dev/null
+++ b/robowaiter/scene/tasks/VLM/VLM_order.py
@@ -0,0 +1,142 @@
+"""
+视觉语言操作
+机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板
+"""
+
+import time
+from robowaiter.scene.scene import Scene
+
+class SceneVLM(Scene):
+    def __init__(self, robot):
+        super().__init__(robot)
+        # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
+        self.new_event_list = [
+            (3, self.add_walker,  (20,0,700)),
+            (5, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw]
+            # (10, self.customer_say, (6,"给我来杯酸奶和咖啡，哦对，再倒一杯水。")),
+            (6, self.customer_say, (6, "来杯酸奶吧。")),
+            (7, self.control_walker, (6, False, 100, -250, 480, 0)),  #(-100,600)
+
+            # 有人提出要开空调和关窗帘
+            # bar (60, 520)
+            (20, self.add_walker, (0, 0, 0)),
+            (25, self.control_walker, (7, False, 100, 60, 520, 180)),
+            (28, self.customer_say, (7,"好热呀！太阳也好大！")),
+            (40, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "这下舒服了！"]]])),
+
+            # (5, self.add_walker, (0, 0, 0)),
+            # (6, self.control_walker, (7, False, 100, 60, 520, 180)),
+            # (10, self.customer_say, (7, "好热呀！太阳也好大！")),
+            # (12, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "这下舒服了！"]]])),
+        ]
+
+    def _reset(self):
+        self.gen_obj()
+        self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)',
+         'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)',
+         'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
+         'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
+        # 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220)
+        self.add_walkers([[3,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
+        # [3,1, 880] 1号桌旁边小女孩
+        # [31,250, 1200] 最角落QuietTable1女红色
+        # [6,-55, 750] 1号桌附近小男孩
+        # [10,70, -200]  另一边角落 QuietTable2 男黄色
+        # [27,-290, 400, 180] 中间 BrightTable4 女灰
+        # [26, 60,-320,90] 另一边角落 BrightTable5 红胖男
+        self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
+
+        # 0-3男孩  4-7女孩 8-26男
+        # 3男孩 31女红  32女灰  10黄色衣服男瘦  9男灰瘦 26红胖男
+        # 17 是员工 police
+        # [0, -150,180]
+
+
+        # self.control_walkers(walker_loc=[[-55, 750]],is_autowalk = False)
+        # 在场景中随机增加一堆行人。
+        # walker_loc = [[-55, 750], [70, -200], [250, 1200], [0, 880]]
+        # controls = []
+        # for i in range(len(s.walkers)):
+        #     loc = walker_loc[i]
+        #     is_autowalk = False
+        #     pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180)
+        #     controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=200, pose=pose))
+        # scene = sim_client.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=scene_id))
+
+        # self.gen_obj(type=5)
+        # self.gen_obj(type=9)
+        # self.op_task_execute(op_type=16, obj_id=0)
+        # self.move_task_area(op_type=4)
+        pass
+
+    def _run(self, op_type=10):
+        # 共17个操作
+        # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子",    # 1-7
+        # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘",            # 8-12
+        # "调整空调开关","调高空调温度","调低空调温度",                   # 13-15
+        # "抓握物体","放置物体"                                       # 16-17
+
+        # self.gen_obj()
+        # if op_type <=15:
+        #     self.move_task_area(op_type)
+        #     self.op_task_execute(op_type)
+        # if op_type == 16:   # 16: 抓操作需要传入物品id
+        #     self.move_task_area(op_type, obj_id=0)
+        #     self.op_task_execute(op_type, obj_id=0)
+        # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107]
+        # # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0]   # 桌子2:[-55.0, 150.0, 107],
+        # if op_type == 17:   # 17: 放操作需要传入放置位置周围的可达区域
+        #     pos = [240.0, 40.0, 100.0]
+        #     self.move_task_area(op_type, release_pos=pos)
+        #     self.op_task_execute(op_type, release_pos=pos)   # [325.0, 860.0, 100]
+
+        # 流程测试
+        # 抓握放置:抓吧台前生成的酸奶，放到抹布桌上
+        # self.gen_obj()
+        # self.move_task_area(16, obj_id=0)
+        # self.op_task_execute(16, obj_id=0)
+        # pos = [340.0, 900.0, 99.0]
+        # self.move_task_area(17, release_pos=pos)
+        # self.op_task_execute(17, release_pos=pos)
+        #
+        # # 做咖啡:做完的咖啡放到水杯桌上
+        # self.move_task_area(1)
+        # self.op_task_execute(1)
+        #
+        # self.find_obj("CoffeeCup")
+        #
+        # self.move_task_area(16, obj_id=275)
+        # self.op_task_execute(16, obj_id=275)
+        # pos = [-70.0, 500.0, 107]
+        # self.move_task_area(17, release_pos=pos)
+        # self.op_task_execute(17, release_pos=pos)
+        #
+        # # 倒水:倒完的水放到旁边桌子上
+        # self.move_task_area(2)
+        # self.op_task_execute(2)
+
+        #
+        # self.move_task_area(16, obj_id=190)
+        # self.op_task_execute(16, obj_id=190)
+        # pos = [-55.0, 0.0, 107]
+        # self.move_task_area(17, release_pos=pos)
+        # self.op_task_execute(17, release_pos=pos)
+
+        # self.test_yaw()
+
+        pass
+    
+    def _step(self):
+        pass
+
+
+if __name__ == '__main__':
+    import os
+    from robowaiter.robot.robot import Robot
+
+    robot = Robot()
+
+    # create task
+    task = SceneVLM(robot)
+    task.reset()
+    task.run()
diff --git a/robowaiter/scene/tasks/VLM/VLN_greet_lead.py b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
index 39ac481..6eb8d3e 100644
--- a/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
+++ b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
@@ -12,7 +12,7 @@ class SceneVLM(Scene):
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.new_event_list = [
             (3, self.add_walker,  (0,60,520)),
-            (5, self.customer_say, (0,"请问可以带我去空位上嘛？我想晒太阳。")),
+            (5, self.customer_say, (0,"可以带我去空位上嘛？我想晒太阳。")),
         ]
 
     def _reset(self):
@@ -43,11 +43,7 @@ class SceneVLM(Scene):
         if end[1]>=600 or end[1]<=450 or end[0]>=250:
         # if int(self.status.location.X)!=247 or  int(self.status.location.X)!=520:
             self.walker_followed = True
-            self.control_walker(
-                    [self.walker_control_generator(walkerID=0, autowalk=False, speed=300, X=end[0], Y=end[1], Yaw=-90)])
-
-            cont = self.status.walkers[0].name+"谢谢！"
-            self.control_robot_action(0,3,cont)
+            self.control_walkers_and_say([[0,False,300,end[0],end[1],90,"谢谢！"]])
 
 
 if __name__ == '__main__':

From fdd6b18c794844a1b6ec307b655b3da7163b0048 Mon Sep 17 00:00:00 2001
From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com>
Date: Sat, 18 Nov 2023 17:44:05 +0800
Subject: [PATCH 05/12] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E8=AD=A6?=
 =?UTF-8?q?=E5=AF=9F=E9=80=9A=E7=9F=A5=E4=B8=8B=E7=8F=AD=20=E6=9C=BA?=
 =?UTF-8?q?=E5=99=A8=E4=BA=BA=E6=89=93=E6=89=AB=E5=8D=AB=E7=94=9F=E5=85=B3?=
 =?UTF-8?q?=E7=81=AF=E5=85=B3=E9=97=A8=E5=85=B3=E7=AA=97=E5=B8=98=E7=9A=84?=
 =?UTF-8?q?=E5=9C=BA=E6=99=AF=20VLM=5Fcafe=5Fshutdown?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/llm_client/data/test_questions.txt |  2 +-
 .../llm_client/data_raw/test_questions.csv    |  2 +-
 .../llm_client/data_raw/test_questions.csv~   | 24 --------
 robowaiter/robot/robot.py                     | 10 +++-
 .../scene/tasks/VLM/VLM_cafe_shutdown.py      | 59 +++++++++++++++++++
 robowaiter/scene/tasks/VLM/VLM_order.py       | 10 ++--
 6 files changed, 73 insertions(+), 34 deletions(-)
 delete mode 100644 robowaiter/llm_client/data_raw/test_questions.csv~
 create mode 100644 robowaiter/scene/tasks/VLM/VLM_cafe_shutdown.py

diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt
index d950325..57c6704 100644
--- a/robowaiter/llm_client/data/test_questions.txt
+++ b/robowaiter/llm_client/data/test_questions.txt
@@ -1 +1 @@
-{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}, "给我来杯酸奶和咖啡，哦对，再倒一杯水。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,WaterTable)\",\"On(Water,WaterTable)\"}"}, "好热呀！太阳也好大！": {"Answer": "抱歉！我这就去开空调和关窗帘！", "Goal": "{\"Is(AC,On)\",\"Is(Curtain,Off)\"}"}, "来杯酸奶吧。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Yogurt,WaterTable)\"}"}}
+{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！别忘了打扫卫生。": {"Answer": "收到！下班啦，我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}, "给我来杯酸奶和咖啡，哦对，再倒一杯水。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,WaterTable)\",\"On(Water,WaterTable)\"}"}, "好热呀！太阳也好大！": {"Answer": "抱歉！我这就去开空调和关窗帘！", "Goal": "{\"Is(AC,On)\",\"Is(Curtain,Off)\"}"}, "来杯酸奶吧。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Yogurt,WaterTable)\"}"}}
diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv
index a89b902..4098d2c 100644
--- a/robowaiter/llm_client/data_raw/test_questions.csv
+++ b/robowaiter/llm_client/data_raw/test_questions.csv
@@ -20,7 +20,7 @@ Question,Answer,Goal
 ���е��ȣ��ܿ����յ���,��Ȼ���ԣ������ھͿ���,"{""Is(AC,On)""}"
 ���Դ���ȥ��,��Ȼ���ԣ�ǰ��һ����,"{""At(Robot,Table1)""}"
 �����̷ŵ�1����������һ�������͵�ˮ�����ϣ��ٵ�һ��ˮ��,���ף������ȥ�죡,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
-�°�������ɨ�������صƹؿյ��ش�����,̫�������°����������ȥ��ɨ�������صƹؿյ��ش�����,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
+�°����������˴�ɨ������,�յ����°����������ȥ��ɨ�������صƹؿյ��ش�����,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
 ���Դ���ȥ��λ�������ɹ̫����,û���⣡���������,"{""At(Robot,BrightTable1)""}"
 �����������̺Ϳ��ȣ�Ŷ�ԣ��ٵ�һ��ˮ��,���ϣ����Եȣ�,"{""On(Coffee,WaterTable)"",""On(Yogurt,WaterTable)"",""On(Water,WaterTable)""}"
 ����ѽ��̫��Ҳ�ô�,��Ǹ�������ȥ���յ��͹ش�����,"{""Is(AC,On)"",""Is(Curtain,Off)""}"
diff --git a/robowaiter/llm_client/data_raw/test_questions.csv~ b/robowaiter/llm_client/data_raw/test_questions.csv~
deleted file mode 100644
index 4ce218c..0000000
--- a/robowaiter/llm_client/data_raw/test_questions.csv~
+++ /dev/null
@@ -1,24 +0,0 @@
-Question,Answer,Goal
-��һ������,OK�������ȥ��һ������,"{""On(Coffee,CoffeeTable)""}"
-��һ�����ȷŵ���̨��,OK�������ȥ��һ�����ȷŵ���̨��,"{""On(Coffee,Bar)""}"
-��һ�����ȷŵ�ˮ�����ϣ��ٵ�һ��ˮ,OK�������ȥ��һ�����ȷŵ�ˮ�����ϣ��ٵ�һ��ˮ,"{""On(Coffee,WaterTable)"",""On(Water,WaterTable)""}"
-ǰ��2����,OK������ǰ��2����,"{""At(Robot,Table2)""}"
-����AEM,����AEM,"{""EnvExplored()""}"
-��һ��ˮ,OK�������ȥ��һ��ˮ,"{""On(Water,WaterTable)""}"
-���յ�,OK�������ȥ���յ�,"{""Is(AC,On)""}"
-�ؿյ�,OK�������ȥ�ؿյ�,"{""Is(AC,Off)""}"
-�ش�����,OK�������ȥ�ش�����,"{""Is(HallLight,Off)""}"
-��������,OK�������ȥ��������,"{""Is(HallLight,On)""}"
-��Ͳ��,OK�������ȥ��Ͳ��,"{""Is(TubeLight,Off)""}"
-��Ͳ��,OK�������ȥ��Ͳ��,"{""Is(TubeLight,On)""}"
-�ش���,OK�������ȥ�ش���,"{""Is(Curtain,Off)""}"
-������,OK�������ȥ������,"{""Is(Curtain,On)""}"
-�ϵ�,OK�������ȥ�ϵ�,"{""Is(Floor,Clean)""}"
-������,OK�������ȥ������,"{""Is(Table1,Clean)""}"
-��������,OK�������ȥ��������,"{""Is(Chairs,Clean)""}"
-�ѱ����ŵ�Table2,OK�������ȥ�ѱ����ŵ�Table2,"{""On(BottledDrink,Table2)""}"
-���е��ȣ��ܿ����յ���,��Ȼ���ԣ������ھͿ���,"{""Is(AC,On)""}"
-���Դ���ȥ��,��Ȼ���ԣ�ǰ��һ����,"{""At(Robot,Table1)""}"
-�����̷ŵ�1����������һ�������͵�ˮ�����ϣ��ٵ�һ��ˮ��,���ף������ȥ�죡,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
-�°�������ɨ�������صƹؿյ��ش�����,̫�������°����������ȥ��ɨ�������صƹؿյ��ش�����,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
-���ʿ��Դ���ȥ��λ�������ɹ̫����,û���⣡���������,"{""At(Robot,BrightTable1)""}"
diff --git a/robowaiter/robot/robot.py b/robowaiter/robot/robot.py
index 40af1af..46abd5c 100644
--- a/robowaiter/robot/robot.py
+++ b/robowaiter/robot/robot.py
@@ -1,3 +1,4 @@
+import copy
 import io
 import contextlib
 import os
@@ -62,9 +63,12 @@ class Robot(object):
             print("--------------------\n")
 
         # 如果目标是下班，规划的时候就直接快捷导入？
-        # end_goal = {"Is(Floor,Clean)","Is(Table1,Clean)","Is(Chairs,Clean)","Is(AC,Off)","Is(HallLight,Off)","Is(TubeLight,Off)","Is(Curtain,Off)"}
-        # if goal & end_goal == goal
-        # else:
+        end_goal = {"Is(Floor,Clean)","Is(Table1,Clean)","Is(Chairs,Clean)","Is(AC,Off)","Is(HallLight,Off)","Is(TubeLight,Off)","Is(Curtain,Off)"}
+        if goal & end_goal == goal:
+            tmp_list = copy.deepcopy(self.action_list)
+            self.action_list=[]
+            self.action_list = [action for action in tmp_list if "Turn" in action.name or "Clean" in action.name]
+
         algo = BTOptExpInterface(self.action_list,self.scene)
 
         ptml_string = algo.process(goal)
diff --git a/robowaiter/scene/tasks/VLM/VLM_cafe_shutdown.py b/robowaiter/scene/tasks/VLM/VLM_cafe_shutdown.py
new file mode 100644
index 0000000..7e19b3f
--- /dev/null
+++ b/robowaiter/scene/tasks/VLM/VLM_cafe_shutdown.py
@@ -0,0 +1,59 @@
+"""
+视觉语言操作
+机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板
+"""
+
+import time
+from robowaiter.scene.scene import Scene
+
+class SceneVLM(Scene):
+    def __init__(self, robot):
+        super().__init__(robot)
+        # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
+        # 出现员工和机器人说关门了(机器人可能手上还拿着东西)
+        self.new_event_list = [
+            (3, self.add_walker, (17, 60, 1000)),
+            (6, self.control_walkers_and_say, ([[[6, False, 500, 60, 520, 0, "下班啦！别忘了打扫卫生。"]]])),
+        ]
+
+    def _reset(self):
+        self.gen_obj()
+        self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)',
+         'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)',
+         'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
+         'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
+        # 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220)
+        self.add_walkers([[3,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
+        # [3,1, 880] 1号桌旁边小女孩
+        # [31,250, 1200] 最角落QuietTable1女红色
+        # [6,-55, 750] 1号桌附近小男孩
+        # [10,70, -200]  另一边角落 QuietTable2 男黄色
+        # [27,-290, 400, 180] 中间 BrightTable4 女灰
+        # [26, 60,-320,90] 另一边角落 BrightTable5 红胖男
+        self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
+
+        # 0-3男孩  4-7女孩 8-26男
+        # 3男孩 31女红  32女灰  10黄色衣服男瘦  9男灰瘦 26红胖男
+        # 17 是员工 police
+        # [0, -150,180]
+
+        pass
+
+    def _run(self, op_type=10):
+
+        pass
+    
+    def _step(self):
+        pass
+
+
+if __name__ == '__main__':
+    import os
+    from robowaiter.robot.robot import Robot
+
+    robot = Robot()
+
+    # create task
+    task = SceneVLM(robot)
+    task.reset()
+    task.run()
diff --git a/robowaiter/scene/tasks/VLM/VLM_order.py b/robowaiter/scene/tasks/VLM/VLM_order.py
index 9e13afa..b3fbe75 100644
--- a/robowaiter/scene/tasks/VLM/VLM_order.py
+++ b/robowaiter/scene/tasks/VLM/VLM_order.py
@@ -14,7 +14,7 @@ class SceneVLM(Scene):
             (3, self.add_walker,  (20,0,700)),
             (5, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw]
             # (10, self.customer_say, (6,"给我来杯酸奶和咖啡，哦对，再倒一杯水。")),
-            (6, self.customer_say, (6, "来杯酸奶吧。")),
+            (8, self.customer_say, (6, "来杯酸奶吧。")),
             (7, self.control_walker, (6, False, 100, -250, 480, 0)),  #(-100,600)
 
             # 有人提出要开空调和关窗帘
@@ -75,7 +75,7 @@ class SceneVLM(Scene):
         # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘",            # 8-12
         # "调整空调开关","调高空调温度","调低空调温度",                   # 13-15
         # "抓握物体","放置物体"                                       # 16-17
-
+        #
         # self.gen_obj()
         # if op_type <=15:
         #     self.move_task_area(op_type)
@@ -89,7 +89,7 @@ class SceneVLM(Scene):
         #     pos = [240.0, 40.0, 100.0]
         #     self.move_task_area(op_type, release_pos=pos)
         #     self.op_task_execute(op_type, release_pos=pos)   # [325.0, 860.0, 100]
-
+        #
         # 流程测试
         # 抓握放置:抓吧台前生成的酸奶，放到抹布桌上
         # self.gen_obj()
@@ -114,14 +114,14 @@ class SceneVLM(Scene):
         # # 倒水:倒完的水放到旁边桌子上
         # self.move_task_area(2)
         # self.op_task_execute(2)
-
+        #
         #
         # self.move_task_area(16, obj_id=190)
         # self.op_task_execute(16, obj_id=190)
         # pos = [-55.0, 0.0, 107]
         # self.move_task_area(17, release_pos=pos)
         # self.op_task_execute(17, release_pos=pos)
-
+        #
         # self.test_yaw()
 
         pass

From f8e4f0053cb30b18569e37edcb7c0e45203e2811 Mon Sep 17 00:00:00 2001
From: ChenXL97 <908926798@qq.com>
Date: Sat, 18 Nov 2023 17:56:48 +0800
Subject: [PATCH 06/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=9B=BA=E5=AE=9A?=
 =?UTF-8?q?=E7=BB=84=E5=90=88?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/behavior_lib/act/DealChat.py       |  52 +++---
 robowaiter/llm_client/data/fix_questions.txt  |  17 ++
 robowaiter/llm_client/multi_rounds.py         | 173 ++++++++++++++++--
 robowaiter/llm_client/single_round.py         |   4 +-
 robowaiter/llm_client/tool_register.py        |  11 ++
 .../interact/{non_customer.py => system.py}   |   0
 robowaiter/scene/scene.py                     |   3 +-
 robowaiter/scene/tasks/OT/Open_tasks_test.py  |  18 +-
 .../scene/tasks/OT/Open_tasks_walkers.py      |  57 ++++++
 9 files changed, 272 insertions(+), 63 deletions(-)
 create mode 100644 robowaiter/llm_client/data/fix_questions.txt
 rename robowaiter/scene/interact/{non_customer.py => system.py} (100%)
 create mode 100644 robowaiter/scene/tasks/OT/Open_tasks_walkers.py

diff --git a/robowaiter/behavior_lib/act/DealChat.py b/robowaiter/behavior_lib/act/DealChat.py
index a47ea66..4e87a1f 100644
--- a/robowaiter/behavior_lib/act/DealChat.py
+++ b/robowaiter/behavior_lib/act/DealChat.py
@@ -1,51 +1,55 @@
 import py_trees as ptree
 from robowaiter.behavior_lib._base.Act import Act
-from robowaiter.llm_client.ask_llm import ask_llm
 
+from robowaiter.llm_client.multi_rounds import ask_llm,new_history
 
 class DealChat(Act):
     def __init__(self):
         super().__init__()
         self.chat_history = ""
+        self.function_success = False
+        self.func_map = {
+            "create_sub_task": self.create_sub_task
+        }
 
     def _update(self) -> ptree.common.Status:
         # if self.scene.status?
         name,sentence = self.scene.state['chat_list'].pop(0)
 
         if name == "Goal":
-            self.create_sub_task(sentence)
+            self.create_sub_task(goal=sentence)
             return ptree.common.Status.RUNNING
 
+        if name not in self.scene.state["chat_history"]:
+            self.scene.state["chat_history"][name] = new_history()
+
+        history = self.scene.state["chat_history"][name]
         self.scene.state["attention"]["customer"] = name
         self.scene.state["serve_state"] = {
             "last_chat_time": self.scene.time,
         }
 
+        function_call, response = ask_llm(sentence,history,func_map=self.func_map)
 
-        self.chat_history += sentence + '\n'
 
-        res_dict = ask_llm(sentence)
-        answer = res_dict["Answer"]
-        self.scene.chat_bubble(answer) # 机器人输出对话
-        self.chat_history += answer + '\n'
-
-        goal = res_dict["Goal"]
-        if goal:
-            if "{" not in goal:
-                goal = {str(goal)}
-            else:
-                goal=eval(goal)
-
-        if goal is not None:
-            print(f'goal：{goal}')
-
-            self.create_sub_task(goal)
-
-        if self.scene.show_bubble:
-            self.scene.chat_bubble(f"{answer}")
+        self.scene.chat_bubble(response) # 机器人输出对话
 
         return ptree.common.Status.RUNNING
 
 
-    def create_sub_task(self,goal):
-        self.scene.robot.expand_sub_task_tree(goal)
+    def create_sub_task(self,**args):
+        try:
+            goal = args['goal']
+
+            w = goal.split(")")
+            goal_set = set()
+            goal_set.add(w[0] + ")")
+            if len(w)>1:
+                for x in w[1:]:
+                    if x != "":
+                        goal_set.add(x[0] + ")")
+            self.function_success = True
+        except:
+            print("参数解析错误")
+
+        self.scene.robot.expand_sub_task_tree(goal_set)
diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt
new file mode 100644
index 0000000..d6361b6
--- /dev/null
+++ b/robowaiter/llm_client/data/fix_questions.txt
@@ -0,0 +1,17 @@
+你好
+您好，我是这家咖啡厅的服务员，请问您要点什么？
+
+做一杯咖啡
+好的，我马上做咖啡
+create_sub_task
+{"goal":"On(Coffee,CoffeeTable)"}
+
+不用了
+好的，您有需要再跟我说
+stop_serve
+{}
+
+来一号桌
+好的，我马上来一号桌
+create_sub_task
+{"goal":"At(Robot,Table1)"}
diff --git a/robowaiter/llm_client/multi_rounds.py b/robowaiter/llm_client/multi_rounds.py
index ca3e09c..6997d56 100644
--- a/robowaiter/llm_client/multi_rounds.py
+++ b/robowaiter/llm_client/multi_rounds.py
@@ -1,29 +1,162 @@
+import json
+
+import openai
+from colorama import init, Fore
+from loguru import logger
+import json
+from robowaiter.llm_client.tool_register import get_tools, dispatch_tool
 import requests
+import json
+from collections import deque
+
 import urllib3
-from robowaiter.llm_client.tool_api import run_conversation
+import copy
+init(autoreset=True)
+from robowaiter.utils import get_root_path
+import os
+import re
+from robowaiter.llm_client.single_round import single_round
 ########################################
-#   该文件实现了与大模型的简单通信、多轮对话，输入end表示对话结束
+#   该文件实现了与大模型的通信以及工具调用
 ########################################
 
 # 忽略https的安全性警告
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
-url = "https://45.125.46.134:25344/v1/chat/completions"
-headers = {"Content-Type": "application/json"}
+base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址
 
-#在这里输入你的问题
-k=input()
-data_memory=[]
-n=1
-while k!='end':
-	question_now=k
-	user_dict={"role": "user","content":question_now}
-	data_memory.append(user_dict)
-	#print(data_memory)
-	response = run_conversation(str(data_memory))
-	answer=str(response)
-	print(answer)
-	assistant_dict={"role": "assistant","content":answer}
-	data_memory.append(assistant_dict)
-	n=n+2
-	k=input()
+root_path = get_root_path()
+# load test questions
+file_path = os.path.join(root_path,"robowaiter/llm_client/data/fix_questions.txt")
+
+fix_questions_dict = {}
+with open(file_path,'r',encoding="utf-8") as f:
+     #读取所有行
+    lines = f.read().strip()
+    sections = re.split(r'\n\s*\n', lines)
+    for s in sections:
+        x = s.split()
+        fix_questions_dict[x[0]] = x[1:]
+
+functions = get_tools()
+role_system = [{
+        "role": "system",
+        "content": "你是RoboWaiter,一个由HPCL团队开发的机器人服务员，你在咖啡厅工作。接受顾客的指令并调用工具函数来完成各种服务任务。如果顾客问你们这里有什么，或者想要点单，你说我们咖啡厅提供咖啡，水，点心，酸奶等食物。如果顾客不需要你了，你就回到吧台招待。如果顾客叫你去做某事，你回复：好的，我马上去做这件事。",
+    }]
+
+def new_history(max_length=7):
+    history = deque(maxlen=max_length)
+
+    return history
+
+def new_response():
+    return {'choices': [{'index': 0, 'message':{} }]}
+
+def parse_fix_question(question):
+    response = new_response()
+    fix_ans = fix_questions_dict[question]
+    if len(fix_ans)<=1: #简单对话
+        message = {'role': 'assistant', 'content': fix_ans, 'name': None,
+         'function_call': None}
+    else:
+        reply, func,args = fix_ans
+        # tool_response = dispatch_tool(function_call["name"], json.loads(args))
+        # logger.info(f"Tool Call Response: {tool_response}")
+        message = {'role': 'assistant',
+         'content': f"\n <|assistant|>  {func}({args})\n ```python\ntool_call(goal={args})\n```",
+         'name': None,
+         'function_call': {'name': func, 'arguments': args}}
+
+    response["choices"][0]["message"] = message
+    return response
+
+def get_response(sentence, history, allow_function_call = True):
+    if sentence:
+        history.append({"role": "user", "content": sentence})
+
+    if sentence in fix_questions_dict:
+        return parse_fix_question(sentence)
+
+    params = dict(model="RoboWaiter")
+    params['messages'] = role_system + list(history)
+    if allow_function_call:
+        params["functions"] = functions
+
+
+    response = requests.post(f"{base_url}/v1/chat/completions", json=params, stream=False, verify=False)
+    decoded_line = response.json()
+    return decoded_line
+
+def deal_response(response, history, func_map=None ):
+    if response["choices"][0]["message"].get("function_call"):
+        function_call = response["choices"][0]["message"]["function_call"]
+        logger.info(f"Function Call Response: {function_call}")
+
+        function_name = function_call["name"]
+        function_args = json.loads(function_call["arguments"])
+        if func_map:
+            tool_response = func_map[function_name](**function_args)
+        else:
+            try:
+                tool_response = dispatch_tool(function_call["name"], function_args)
+                logger.info(f"Tool Call Response: {tool_response}")
+            except:
+                logger.info(f"重试工具调用")
+        # tool_response = dispatch_tool(function_call["name"], function_args)
+            return function_name,None
+
+        return_message = response["choices"][0]["message"]
+
+        history.append(return_message)
+        t = {
+            "role": "function",
+            "name": function_call["name"],
+            "content": str(tool_response),  # 调用函数返回结果
+        }
+
+        history.append(t)
+        return function_call["name"], return_message
+
+    else:
+        return_message = response["choices"][0]["message"]
+        reply = return_message["content"]
+
+        history.append(return_message)
+        logger.info(f"Final Reply: \n{reply}")
+
+        return False, reply
+
+
+def ask_llm(question,history, func_map=None, retry=3):
+    response = get_response(question, history)
+
+    function_call,result = deal_response(response, history, func_map)
+    if function_call:
+        if question in fix_questions_dict:
+            reply = fix_questions_dict[question][0]
+            result = single_round(reply,"你是机器人服务员，请把以下句子换一种表述方式对顾客说，但是意思不变，尽量简短：\n")
+            message = {'role': 'assistant', 'content': result, 'name': None,
+                       'function_call': None}
+            history.append(message)
+
+        else:
+            response = get_response(None, history,allow_function_call=False)
+            _,result = deal_response(response, history, func_map)
+
+
+    print(f'{len(history)}条历史记录:')
+    for x in history:
+        print(x)
+    return function_call, result
+
+if __name__ == "__main__":
+    question = input("\n顾客：")
+    history = new_history()
+    n = 1
+    max_retry = 2
+
+    while question != 'end':
+        function_call, return_message = ask_llm(question,history)
+
+
+        question = input("\n顾客：")
diff --git a/robowaiter/llm_client/single_round.py b/robowaiter/llm_client/single_round.py
index 34c9e47..d0c88c6 100644
--- a/robowaiter/llm_client/single_round.py
+++ b/robowaiter/llm_client/single_round.py
@@ -9,7 +9,7 @@ import urllib3
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
-def single_round(question):
+def single_round(question,prefix=""):
     url = "https://45.125.46.134:25344/v1/chat/completions"
     headers = {"Content-Type": "application/json"}
     data = {
@@ -21,7 +21,7 @@ def single_round(question):
           },
           {
             "role": "user",
-            "content": question
+            "content": prefix + question
           }
         ]
       }
diff --git a/robowaiter/llm_client/tool_register.py b/robowaiter/llm_client/tool_register.py
index 36f204b..1e4b42e 100644
--- a/robowaiter/llm_client/tool_register.py
+++ b/robowaiter/llm_client/tool_register.py
@@ -147,6 +147,17 @@ def create_sub_task(
 
     return goal
 
+
+@register_tool
+def stop_serve(
+) -> bool:
+    """
+    当顾客通过任何形式表示不再需要服务时，调用该函数
+    """
+    return True
+
+
+
 # @register_tool
 # def get_object_info(
 #         obj: Annotated[str, '需要获取信息的物体名称', True]
diff --git a/robowaiter/scene/interact/non_customer.py b/robowaiter/scene/interact/system.py
similarity index 100%
rename from robowaiter/scene/interact/non_customer.py
rename to robowaiter/scene/interact/system.py
diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py
index 32bf629..ea376ea 100644
--- a/robowaiter/scene/scene.py
+++ b/robowaiter/scene/scene.py
@@ -71,6 +71,7 @@ class Scene:
         "greeted_customers":set(),
         "attention":{},
         "serve_state":{},
+        "chat_history":{}
     }
     """
     status:
@@ -439,7 +440,7 @@ class Scene:
     def chat_bubble(self, message):
         stub.ControlRobot(
             GrabSim_pb2.ControlInfo(
-                scene=self.sceneID, type=0, action=1, content=message
+                scene=self.sceneID, type=0, action=1, content=message.strip()
             )
         )
 
diff --git a/robowaiter/scene/tasks/OT/Open_tasks_test.py b/robowaiter/scene/tasks/OT/Open_tasks_test.py
index 95408bf..d39aae6 100644
--- a/robowaiter/scene/tasks/OT/Open_tasks_test.py
+++ b/robowaiter/scene/tasks/OT/Open_tasks_test.py
@@ -18,28 +18,14 @@ class SceneOT(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件
         self.new_event_list = [
-            # (9,self.add_walkers,([[0, 880]],)),
-            # (10,self.walker_walk_to,(2,50,500))
-            # (5, self.set_goal("On(Yogurt,Table4)"))
+            (3, self.customer_say, ("System","来一号桌"))
             # (5, self.set_goal("At(Robot,BrightTable4)"))
         ]
 
     def _reset(self):
         # self.add_walkers([[0, 880], [250, 1200]])
+        pass
 
-        # 展示顾客，前8个id是小孩，后面都是大人
-        for i in range(4):
-            self.add_walker(i,50,300 + i * 50)
-        name1 = self.walker_index2mem(1)
-        name2 = self.walker_index2mem(3)
-
-        self.remove_walker(0,2)
-
-        index1 = self.state["customer_mem"][name1]
-        index2 = self.state["customer_mem"][name2]
-
-        self.walker_bubble(name1,f"我是第{index1}个")
-        self.walker_bubble(name2,f"我是第{index2}个")
 
     def _run(self):
         pass
diff --git a/robowaiter/scene/tasks/OT/Open_tasks_walkers.py b/robowaiter/scene/tasks/OT/Open_tasks_walkers.py
new file mode 100644
index 0000000..95408bf
--- /dev/null
+++ b/robowaiter/scene/tasks/OT/Open_tasks_walkers.py
@@ -0,0 +1,57 @@
+"""
+人提出请求，机器人完成任务
+1. 做咖啡（固定动画）：接收到做咖啡指令、走到咖啡机、拿杯子、操作咖啡机、取杯子、送到客人桌子上
+2. 倒水
+3. 夹点心
+
+具体描述：设计一套点单规则（如菜单包含咖啡、水、点心等），按照规则拟造随机的订单。在收到订单后，通过大模型让机器人输出合理的备餐计划，并尝试在模拟环境中按照这个规划实现任务。
+
+"""
+
+# todo: 接收点单信息，大模型生成任务规划
+
+from robowaiter.scene.scene import Scene
+
+class SceneOT(Scene):
+
+    def __init__(self, robot):
+        super().__init__(robot)
+        # 在这里加入场景中发生的事件
+        self.new_event_list = [
+            # (9,self.add_walkers,([[0, 880]],)),
+            # (10,self.walker_walk_to,(2,50,500))
+            # (5, self.set_goal("On(Yogurt,Table4)"))
+            # (5, self.set_goal("At(Robot,BrightTable4)"))
+        ]
+
+    def _reset(self):
+        # self.add_walkers([[0, 880], [250, 1200]])
+
+        # 展示顾客，前8个id是小孩，后面都是大人
+        for i in range(4):
+            self.add_walker(i,50,300 + i * 50)
+        name1 = self.walker_index2mem(1)
+        name2 = self.walker_index2mem(3)
+
+        self.remove_walker(0,2)
+
+        index1 = self.state["customer_mem"][name1]
+        index2 = self.state["customer_mem"][name2]
+
+        self.walker_bubble(name1,f"我是第{index1}个")
+        self.walker_bubble(name2,f"我是第{index2}个")
+
+    def _run(self):
+        pass
+
+
+if __name__ == '__main__':
+    import os
+    from robowaiter.robot.robot import Robot
+
+    robot = Robot()
+
+    # create task
+    task = SceneOT(robot)
+    task.reset()
+    task.run()

From 3684d43f5e08f1b9994392f4252631af1cea5cf7 Mon Sep 17 00:00:00 2001
From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com>
Date: Sat, 18 Nov 2023 21:02:14 +0800
Subject: [PATCH 07/12] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E5=9C=BA?=
 =?UTF-8?q?=E6=99=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/llm_client/data/fix_questions.txt | 10 ++++++++++
 robowaiter/scene/tasks/VLM/VLM_order.py      | 18 +++++++++---------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt
index d6361b6..c500aca 100644
--- a/robowaiter/llm_client/data/fix_questions.txt
+++ b/robowaiter/llm_client/data/fix_questions.txt
@@ -15,3 +15,13 @@ stop_serve
 好的，我马上来一号桌
 create_sub_task
 {"goal":"At(Robot,Table1)"}
+
+来杯酸奶吧。
+好的没问题，请稍等！
+create_sub_task
+{"goal":"On(Yogurt,WaterTable)"}
+
+好热呀！太阳也好大！
+抱歉，我这就去开空调和关窗帘。
+create_sub_task
+{"goal":"Is(AC,On),Is(Curtain,Off)"}
diff --git a/robowaiter/scene/tasks/VLM/VLM_order.py b/robowaiter/scene/tasks/VLM/VLM_order.py
index b3fbe75..09768ce 100644
--- a/robowaiter/scene/tasks/VLM/VLM_order.py
+++ b/robowaiter/scene/tasks/VLM/VLM_order.py
@@ -12,17 +12,17 @@ class SceneVLM(Scene):
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.new_event_list = [
             (3, self.add_walker,  (20,0,700)),
-            (5, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw]
-            # (10, self.customer_say, (6,"给我来杯酸奶和咖啡，哦对，再倒一杯水。")),
-            (8, self.customer_say, (6, "来杯酸奶吧。")),
-            (7, self.control_walker, (6, False, 100, -250, 480, 0)),  #(-100,600)
+            # (5, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw]
+            # # (10, self.customer_say, (6,"给我来杯酸奶和咖啡，哦对，再倒一杯水。")),
+            # (8, self.customer_say, (6, "来杯酸奶吧。")),
+            # (7, self.control_walker, (6, False, 100, -250, 480, 0)),  #(-100,600)
 
             # 有人提出要开空调和关窗帘
             # bar (60, 520)
-            (20, self.add_walker, (0, 0, 0)),
-            (25, self.control_walker, (7, False, 100, 60, 520, 180)),
-            (28, self.customer_say, (7,"好热呀！太阳也好大！")),
-            (40, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "这下舒服了！"]]])),
+            (28, self.add_walker, (0, 0, 0)),
+            (33, self.control_walker, (7, False, 100, 60, 520, 180)),
+            (35, self.customer_say, (7,"好热呀！太阳也好大！")),
+            (45, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])),
 
             # (5, self.add_walker, (0, 0, 0)),
             # (6, self.control_walker, (7, False, 100, 60, 520, 180)),
@@ -37,7 +37,7 @@ class SceneVLM(Scene):
          'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
          'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
         # 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220)
-        self.add_walkers([[3,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
+        self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
         # [3,1, 880] 1号桌旁边小女孩
         # [31,250, 1200] 最角落QuietTable1女红色
         # [6,-55, 750] 1号桌附近小男孩

From 6f3a01eb3f4a4f67e94213ba22a3bfcefed2ddcd Mon Sep 17 00:00:00 2001
From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com>
Date: Sat, 18 Nov 2023 21:03:10 +0800
Subject: [PATCH 08/12] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BA=86=E5=9C=BA?=
 =?UTF-8?q?=E6=99=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/scene/tasks/VLM/VLM_order.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/robowaiter/scene/tasks/VLM/VLM_order.py b/robowaiter/scene/tasks/VLM/VLM_order.py
index 09768ce..1d773e4 100644
--- a/robowaiter/scene/tasks/VLM/VLM_order.py
+++ b/robowaiter/scene/tasks/VLM/VLM_order.py
@@ -19,15 +19,15 @@ class SceneVLM(Scene):
 
             # 有人提出要开空调和关窗帘
             # bar (60, 520)
-            (28, self.add_walker, (0, 0, 0)),
-            (33, self.control_walker, (7, False, 100, 60, 520, 180)),
-            (35, self.customer_say, (7,"好热呀！太阳也好大！")),
-            (45, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])),
+            # (28, self.add_walker, (0, 0, 0)),
+            # (33, self.control_walker, (7, False, 100, 60, 520, 180)),
+            # (35, self.customer_say, (7,"好热呀！太阳也好大！")),
+            # (45, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])),
 
-            # (5, self.add_walker, (0, 0, 0)),
-            # (6, self.control_walker, (7, False, 100, 60, 520, 180)),
-            # (10, self.customer_say, (7, "好热呀！太阳也好大！")),
-            # (12, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "这下舒服了！"]]])),
+            (5, self.add_walker, (0, 0, 0)),
+            (6, self.control_walker, (7, False, 100, 60, 520, 180)),
+            (7, self.customer_say, (7,"好热呀！太阳也好大！")),
+            (8, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])),
         ]
 
     def _reset(self):

From 45d1874867968e4b2f00f4cbc85623b055a5bf02 Mon Sep 17 00:00:00 2001
From: ChenXL97 <908926798@qq.com>
Date: Sat, 18 Nov 2023 21:09:14 +0800
Subject: [PATCH 09/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=A4=84=E7=90=86?=
 =?UTF-8?q?=E4=B8=A4=E4=B8=AA=E7=9B=AE=E6=A0=87=E7=9A=84bug?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/behavior_lib/act/DealChat.py | 2 +-
 robowaiter/scene/scene.py               | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/robowaiter/behavior_lib/act/DealChat.py b/robowaiter/behavior_lib/act/DealChat.py
index 4e87a1f..0bdcf70 100644
--- a/robowaiter/behavior_lib/act/DealChat.py
+++ b/robowaiter/behavior_lib/act/DealChat.py
@@ -47,7 +47,7 @@ class DealChat(Act):
             if len(w)>1:
                 for x in w[1:]:
                     if x != "":
-                        goal_set.add(x[0] + ")")
+                        goal_set.add(x[1:] + ")")
             self.function_success = True
         except:
             print("参数解析错误")
diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py
index 6898cf0..a36ded3 100644
--- a/robowaiter/scene/scene.py
+++ b/robowaiter/scene/scene.py
@@ -458,6 +458,7 @@ class Scene:
     def customer_say(self,name,sentence,show_bubble=True):
         if isinstance(name,int):
             name = self.walker_index2mem(name)
+
         print(f'{name} say: {sentence}')
         if show_bubble:
             self.walker_bubble(name,sentence)

From 65fadd68fb475dcb0ef9291e8bf3811e9f837c1f Mon Sep 17 00:00:00 2001
From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com>
Date: Sat, 18 Nov 2023 21:24:11 +0800
Subject: [PATCH 10/12] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E5=9C=BA?=
 =?UTF-8?q?=E6=99=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/llm_client/data/fix_questions.txt | 11 +++++++++++
 robowaiter/scene/tasks/VLM/VLM.py            | 18 +++++++++++++++---
 robowaiter/scene/tasks/VLM/VLN_greet_lead.py |  5 +++--
 3 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt
index c500aca..0fa6d5f 100644
--- a/robowaiter/llm_client/data/fix_questions.txt
+++ b/robowaiter/llm_client/data/fix_questions.txt
@@ -16,6 +16,17 @@ stop_serve
 create_sub_task
 {"goal":"At(Robot,Table1)"}
 
+
+早上好呀，我想找个能晒太阳的地方。
+没问题，您右手边就有能晒太阳的位置呢。
+
+
+可以带我过去嘛？
+当然，请跟我来！
+create_sub_task
+{"goal":"At(Robot,BrightTable1)"}
+
+
 来杯酸奶吧。
 好的没问题，请稍等！
 create_sub_task
diff --git a/robowaiter/scene/tasks/VLM/VLM.py b/robowaiter/scene/tasks/VLM/VLM.py
index d0ba4a5..2524f6c 100644
--- a/robowaiter/scene/tasks/VLM/VLM.py
+++ b/robowaiter/scene/tasks/VLM/VLM.py
@@ -11,7 +11,7 @@ class SceneVLM(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.event_list = [
-            (5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
+            # (5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
             # (10, self.create_chat_event("开空调")),
 
             # (15, self.create_chat_event("下班啦！打扫卫生，关灯关空调关窗帘。")),
@@ -37,13 +37,25 @@ class SceneVLM(Scene):
 
     def _reset(self):
         self.gen_obj()
+
+
         self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)',
          'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)',
          'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
          'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
+
+        self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
+        # [3,1, 880] 1号桌旁边小女孩
+        # [31,250, 1200] 最角落QuietTable1女红色
+        # [6,-55, 750] 1号桌附近小男孩
+        # [10,70, -200]  另一边角落 QuietTable2 男黄色
+        # [27,-290, 400, 180] 中间 BrightTable4 女灰
+        # [26, 60,-320,90] 另一边角落 BrightTable5 红胖男
+        # self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
+
         # 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220)
-        self.add_walkers([[0, 880], [250, 1200], [-55, 750], [70, -200],[-290, 400, 0],[20, -150,180]])
-        self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
+        # self.add_walkers([[0, 880], [250, 1200], [-55, 750], [70, -200],[-290, 400, 0],[20, -150,180]])
+        # self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
 
 
         # self.control_walkers(walker_loc=[[-55, 750]],is_autowalk = False)
diff --git a/robowaiter/scene/tasks/VLM/VLN_greet_lead.py b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
index 6eb8d3e..251d302 100644
--- a/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
+++ b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py
@@ -11,8 +11,9 @@ class SceneVLM(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.new_event_list = [
-            (3, self.add_walker,  (0,60,520)),
-            (5, self.customer_say, (0,"可以带我去空位上嘛？我想晒太阳。")),
+            (3, self.add_walker, (5, 230, 1200)),
+            (5, self.control_walkers_and_say, ([[[0, False, 200, 60, 520, 0, "早上好呀，我想找个能晒太阳的地方。"]]])),# (0, 60, 520)),
+            (6, self.customer_say, (0,"可以带我过去嘛？")),
         ]
 
     def _reset(self):

From 1033b9578157d422b3205142826b9ae80ca6ac50 Mon Sep 17 00:00:00 2001
From: ChenXL97 <908926798@qq.com>
Date: Sat, 18 Nov 2023 21:33:20 +0800
Subject: [PATCH 11/12] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=AE=80=E5=8D=95?=
 =?UTF-8?q?=E5=AF=B9=E8=AF=9D=E7=9A=84bug?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/llm_client/multi_rounds.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/robowaiter/llm_client/multi_rounds.py b/robowaiter/llm_client/multi_rounds.py
index 6997d56..0726576 100644
--- a/robowaiter/llm_client/multi_rounds.py
+++ b/robowaiter/llm_client/multi_rounds.py
@@ -56,7 +56,7 @@ def parse_fix_question(question):
     response = new_response()
     fix_ans = fix_questions_dict[question]
     if len(fix_ans)<=1: #简单对话
-        message = {'role': 'assistant', 'content': fix_ans, 'name': None,
+        message = {'role': 'assistant', 'content': fix_ans[0], 'name': None,
          'function_call': None}
     else:
         reply, func,args = fix_ans

From 8bdf5a96b44540fec9def6f3add800e7e80240d4 Mon Sep 17 00:00:00 2001
From: ChenXL97 <908926798@qq.com>
Date: Sat, 18 Nov 2023 22:30:14 +0800
Subject: [PATCH 12/12] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=87=BD=E6=95=B0?=
 =?UTF-8?q?=E8=9E=8D=E5=85=A5=E5=9C=BA=E6=99=AF=E4=B8=AD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 robowaiter/behavior_lib/act/DealChat.py      | 19 +++++++-
 robowaiter/llm_client/data/fix_questions.txt |  7 +++
 robowaiter/llm_client/multi_rounds.py        | 37 +++++++++++----
 robowaiter/llm_client/tool_register.py       | 50 ++++++++++----------
 robowaiter/scene/tasks/{ => GQA}/GQA.py      |  5 +-
 robowaiter/scene/tasks/GQA/__init__.py       |  0
 6 files changed, 81 insertions(+), 37 deletions(-)
 rename robowaiter/scene/tasks/{ => GQA}/GQA.py (88%)
 create mode 100644 robowaiter/scene/tasks/GQA/__init__.py

diff --git a/robowaiter/behavior_lib/act/DealChat.py b/robowaiter/behavior_lib/act/DealChat.py
index 0bdcf70..e66ed1c 100644
--- a/robowaiter/behavior_lib/act/DealChat.py
+++ b/robowaiter/behavior_lib/act/DealChat.py
@@ -9,7 +9,8 @@ class DealChat(Act):
         self.chat_history = ""
         self.function_success = False
         self.func_map = {
-            "create_sub_task": self.create_sub_task
+            "create_sub_task": self.create_sub_task,
+            "get_object_info": self.get_object_info
         }
 
     def _update(self) -> ptree.common.Status:
@@ -53,3 +54,19 @@ class DealChat(Act):
             print("参数解析错误")
 
         self.scene.robot.expand_sub_task_tree(goal_set)
+
+
+    def get_object_info(self,**args):
+        try:
+            obj = args['obj']
+
+            self.function_success = True
+        except:
+            obj = None
+            print("参数解析错误")
+
+        near_object = "None"
+        if obj == "洗手间":
+            near_object = "大门"
+
+        return near_object
\ No newline at end of file
diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt
index 0fa6d5f..3ca9173 100644
--- a/robowaiter/llm_client/data/fix_questions.txt
+++ b/robowaiter/llm_client/data/fix_questions.txt
@@ -36,3 +36,10 @@ create_sub_task
 抱歉，我这就去开空调和关窗帘。
 create_sub_task
 {"goal":"Is(AC,On),Is(Curtain,Off)"}
+
+
+
+请问洗手间在哪里？
+洗手间在这附近
+get_object_info
+{"obj":"洗手间"}
diff --git a/robowaiter/llm_client/multi_rounds.py b/robowaiter/llm_client/multi_rounds.py
index 0726576..81ca2d8 100644
--- a/robowaiter/llm_client/multi_rounds.py
+++ b/robowaiter/llm_client/multi_rounds.py
@@ -29,16 +29,30 @@ root_path = get_root_path()
 # load test questions
 file_path = os.path.join(root_path,"robowaiter/llm_client/data/fix_questions.txt")
 
+functions = get_tools()
+
 fix_questions_dict = {}
+no_reply_functions = ["create_sub_task"]
+
 with open(file_path,'r',encoding="utf-8") as f:
      #读取所有行
     lines = f.read().strip()
     sections = re.split(r'\n\s*\n', lines)
     for s in sections:
         x = s.split()
-        fix_questions_dict[x[0]] = x[1:]
+        if len(x) == 2:
+            fix_questions_dict[x[0]] = {
+                "answer": x[1],
+                "function": None
+            }
+        else:
+            fix_questions_dict[x[0]] = {
+                "answer": x[1],
+                "function": x[2],
+                "args": x[3]
+            }
+
 
-functions = get_tools()
 role_system = [{
         "role": "system",
         "content": "你是RoboWaiter,一个由HPCL团队开发的机器人服务员，你在咖啡厅工作。接受顾客的指令并调用工具函数来完成各种服务任务。如果顾客问你们这里有什么，或者想要点单，你说我们咖啡厅提供咖啡，水，点心，酸奶等食物。如果顾客不需要你了，你就回到吧台招待。如果顾客叫你去做某事，你回复：好的，我马上去做这件事。",
@@ -55,11 +69,12 @@ def new_response():
 def parse_fix_question(question):
     response = new_response()
     fix_ans = fix_questions_dict[question]
-    if len(fix_ans)<=1: #简单对话
-        message = {'role': 'assistant', 'content': fix_ans[0], 'name': None,
+    if not fix_ans['function']: #简单对话
+        message = {'role': 'assistant', 'content': fix_ans["answer"], 'name': None,
          'function_call': None}
     else:
-        reply, func,args = fix_ans
+        func = fix_ans["function"]
+        args = fix_ans["args"]
         # tool_response = dispatch_tool(function_call["name"], json.loads(args))
         # logger.info(f"Tool Call Response: {tool_response}")
         message = {'role': 'assistant',
@@ -115,7 +130,7 @@ def deal_response(response, history, func_map=None ):
         }
 
         history.append(t)
-        return function_call["name"], return_message
+        return function_call["name"], tool_response
 
     else:
         return_message = response["choices"][0]["message"]
@@ -133,8 +148,14 @@ def ask_llm(question,history, func_map=None, retry=3):
     function_call,result = deal_response(response, history, func_map)
     if function_call:
         if question in fix_questions_dict:
-            reply = fix_questions_dict[question][0]
-            result = single_round(reply,"你是机器人服务员，请把以下句子换一种表述方式对顾客说，但是意思不变，尽量简短：\n")
+            if fix_questions_dict[question]['function'] in  no_reply_functions:
+                reply = fix_questions_dict[question]["answer"]
+                result = single_round(reply,
+                                      "你是机器人服务员，请把以下句子换一种表述方式对顾客说，但是意思不变，尽量简短：\n")
+            else:
+                reply = fix_questions_dict[question]["answer"]
+                result = single_round(f"你是机器人服务员，顾客想知道{question}, 你的具身场景查询返回的是{result},请把按照以下句子对顾客说，{reply}, 尽量简短。\n")
+
             message = {'role': 'assistant', 'content': result, 'name': None,
                        'function_call': None}
             history.append(message)
diff --git a/robowaiter/llm_client/tool_register.py b/robowaiter/llm_client/tool_register.py
index 1e4b42e..f15a927 100644
--- a/robowaiter/llm_client/tool_register.py
+++ b/robowaiter/llm_client/tool_register.py
@@ -158,31 +158,31 @@ def stop_serve(
 
 
 
-# @register_tool
-# def get_object_info(
-#         obj: Annotated[str, '需要获取信息的物体名称', True]
-# ) -> str:
-#     """
-#     获取场景中指定物体 `object` 在哪里，不涉及到具体的执行任务
-#     如果`object` 是一个地点，例如洗手间，则输出大门。
-#     如果`object`是咖啡，则输出桌子，咖啡在桌子上。
-#     如果`object` 是空桌子，则输出一号桌
-#     """
-#     near_object = None
-#     # if obj == "Table":
-#     #     near_object = "Bar"
-#     # if obj == "洗手间":
-#     #     near_object = "大门"
-#     # if obj == "空桌子":
-#     #     near_object = "一号桌"
-#     if obj in find_obj_utils.all_loc:   # object是一个地点
-#         mp = list(find_obj_utils.loc_map[obj])
-#         # near_object = random.choice(mp)
-#         near_object = mp
-#     if obj in find_obj_utils.all_obj:   # object是一个物品
-#         near_ls = find_obj_utils.all_loc + find_obj_utils.all_obj
-#         near_object = random.choices(near_ls,k=5)
-#     return near_object
+@register_tool
+def get_object_info(
+        obj: Annotated[str, '需要获取信息的物体名称', True]
+) -> str:
+    """
+    获取场景中指定物体 `object` 在哪里，不涉及到具体的执行任务
+    如果`object` 是一个地点，例如洗手间，则输出大门。
+    如果`object`是咖啡，则输出桌子，咖啡在桌子上。
+    如果`object` 是空桌子，则输出一号桌
+    """
+    near_object = None
+    # if obj == "Table":
+    #     near_object = "Bar"
+    if obj == "洗手间":
+        near_object = "大门"
+    # if obj == "空桌子":
+    #     near_object = "一号桌"
+    if obj in find_obj_utils.all_loc:   # object是一个地点
+        mp = list(find_obj_utils.loc_map[obj])
+        # near_object = random.choice(mp)
+        near_object = mp
+    if obj in find_obj_utils.all_obj:   # object是一个物品
+        near_ls = find_obj_utils.all_loc + find_obj_utils.all_obj
+        near_object = random.choices(near_ls,k=5)
+    return near_object
 
 # @register_tool
 # def find_location(
diff --git a/robowaiter/scene/tasks/GQA.py b/robowaiter/scene/tasks/GQA/GQA.py
similarity index 88%
rename from robowaiter/scene/tasks/GQA.py
rename to robowaiter/scene/tasks/GQA/GQA.py
index 577d9b6..54b9063 100644
--- a/robowaiter/scene/tasks/GQA.py
+++ b/robowaiter/scene/tasks/GQA/GQA.py
@@ -15,9 +15,8 @@ class SceneGQA(Scene):
     def __init__(self, robot):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
-        self.event_list = [
-            (5, self.create_chat_event("哪里有空桌子")),
-            (12, self.create_chat_event("可以带我去吗")),
+        self.new_event_list = [
+            (3, self.customer_say, ("System","请问洗手间在哪里？"))
         ]
 
     def _reset(self):
diff --git a/robowaiter/scene/tasks/GQA/__init__.py b/robowaiter/scene/tasks/GQA/__init__.py
new file mode 100644
index 0000000..e69de29