加入多轮对话

robowaiter/behavior_lib/act/DealChat.py

@@ -13,13 +13,16 @@ class DealChat(Act):
         chat = self.scene.state['chat_list'].pop()
         self.chat_history += chat + '\n'
 
-        res_dict = ask_llm(self.chat_history)
+        res_dict = ask_llm(chat)
         answer = res_dict["Answer"]
         self.chat_history += answer + '\n'
 
         goal = res_dict["Goal"]
-        if goal and "{" not in goal:
-            goal = {str(goal)}
+        if goal:
+            if "{" not in goal:
+                goal = {str(goal)}
+            else:
+                goal=eval(goal)
 
         if goal is not None:
             print(f'goal：{goal}')

robowaiter/llm_client/data/test_questions.txt

@@ -1 +1 @@
-{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "测试VLM：做一杯咖啡放到吧台上": {"Answer": "测试VLM：做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "测试VLN：前往2号桌": {"Answer": "测试VLN：前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关空调": {"Answer": "测试VLM：关空调", "Goal": "{\"Is(AC,Off)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：开大厅灯": {"Answer": "测试VLM：开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：开筒灯": {"Answer": "测试VLM：开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：开窗帘": {"Answer": "测试VLM：开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：整理椅子": {"Answer": "测试VLM：整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}}
+{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "测试VLM：做一杯咖啡放到吧台上": {"Answer": "测试VLM：做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "测试VLM：做一杯咖啡放到水杯桌上并倒水": {"Answer": "测试VLM：做一杯咖啡放到水杯桌上并倒水", "Goal": "{\"On(Coffee,WaterTable)\"}"}, "测试VLN：前往2号桌": {"Answer": "测试VLN：前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关空调": {"Answer": "测试VLM：关空调", "Goal": "{\"Is(AC,Off)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：开大厅灯": {"Answer": "测试VLM：开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：开筒灯": {"Answer": "测试VLM：开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：开窗帘": {"Answer": "测试VLM：开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：整理椅子": {"Answer": "测试VLM：整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}}

robowaiter/llm_client/data_raw/test_questions.csv

@@ -18,3 +18,4 @@ Question,Answer,Goal
 测试VLM：整理椅子,测试VLM：整理椅子,"{""Is(Chairs,Clean)""}"
 测试VLM：把冰红茶放到Table2,测试VLM：把冰红茶放到Table2,"{""On(BottledDrink,Table2)""}"
 我有点热，能开个空调吗？,当然可以，我现在就开！,"{""Is(AC,On)""}"
+可以带我去吗,当然可以，前往一号桌,"{""At(Robot,Table1)""}"

robowaiter/llm_client/data_tools/csv2dict.py

@@ -24,3 +24,4 @@ with open(csv_file_path, mode='r', encoding='gbk') as csv_file, \
     json_str = json.dumps(output_dict, ensure_ascii=False)
     # 将JSON字符串写入JSONL文件，并添加换行符
     jsonl_file.write(json_str + '\n')
+s

robowaiter/scene/scene.py

@@ -221,7 +221,7 @@ class Scene:
         print('------------------add_walkers----------------------')
         walker_list = []
         for i in range(len(walker_loc)):
-            loc = walker_loc[i] + [0, 0, 100]
+            loc = walker_loc[i] + [0,0, 100]
             action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=loc)
             scene = stub.Do(action)
             print(scene.info)

robowaiter/scene/tasks/AEM.py

@@ -4,7 +4,6 @@
 """
 
 from robowaiter.scene.scene import Scene
-from robowaiter.algos.explore.semantic_map import impo
 class SceneAEM(Scene):
     def __init__(self, robot):
         super().__init__(robot)

robowaiter/scene/tasks/GQA.py

@@ -22,8 +22,9 @@ class SceneGQA(Scene):
 
     def _reset(self):
         # self.clean_walker()
+        self.add_walkers()
+
 
-        self.add_walker(50, 500, 0)
         # self.walker_bubble("洗手间在哪里")
         # self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)])