简单多轮对话

robowaiter/behavior_lib/act/DealChat.py

@@ -6,14 +6,20 @@ from robowaiter.llm_client.ask_llm import ask_llm
 class DealChat(Act):
     def __init__(self):
         super().__init__()
+        self.chat_history = ""
 
     def _update(self) -> ptree.common.Status:
         # if self.scene.status?
         chat = self.scene.state['chat_list'].pop()
+        self.chat_history += chat + '\n'
 
-        res_dict = ask_llm(chat)
+        res_dict = ask_llm(self.chat_history)
         answer = res_dict["Answer"]
-        goal = eval(res_dict["Goal"])
+        self.chat_history += answer + '\n'
+
+        goal = res_dict["Goal"]
+        if goal and "{" not in goal:
+            goal = {str(goal)}
 
         if goal is not None:
             print(f'goal：{goal}')

robowaiter/llm_client/ask_llm.py

@@ -4,6 +4,8 @@ import requests
 import urllib3
 from robowaiter.utils import get_root_path
 from robowaiter.llm_client.single_round import single_round
+from robowaiter.llm_client.tool_api import run_conversation
+
 ########################################
 #   该文件实现了与大模型的简单通信
 ########################################
@@ -22,7 +24,9 @@ def ask_llm(question):
     if question in test_questions_dict:
         ans = test_questions_dict[question]
     else:
-        ans = single_round(question)
+        ans = run_conversation(question, stream=False)
+
+        # ans = single_round(question)
     print(f"大模型输出： {ans}")
     return ans
 

robowaiter/llm_client/multi_rounds.py

@@ -1,6 +1,6 @@
 import requests
 import urllib3
-
+from robowaiter.llm_client.tool_api import run_conversation
 ########################################
 #   该文件实现了与大模型的简单通信、多轮对话，输入end表示对话结束
 ########################################
@@ -20,8 +20,8 @@ while k!='end':
 	user_dict={"role": "user","content":question_now}
 	data_memory.append(user_dict)
 	#print(data_memory)
-	response = requests.post(url, headers=headers, json={"messages":data_memory, "repetition_penalty": 1.0}, verify=False)
-	answer=response.json()['choices'][n]['message']['content']
+	response = run_conversation(str(data_memory))
+	answer=str(response)
 	print(answer)
 	assistant_dict={"role": "assistant","content":answer}
 	data_memory.append(assistant_dict)

robowaiter/llm_client/tool_api.py

@@ -3,8 +3,8 @@ import json
 import openai
 from colorama import init, Fore
 from loguru import logger
-
-from tool_register import get_tools, dispatch_tool
+import json
+from robowaiter.llm_client.tool_register import get_tools, dispatch_tool
 import requests
 import json
 
@@ -29,22 +29,21 @@ def get_response(**kwargs):
 
 functions = get_tools()
 
-def run_conversation(query: str, stream=False, functions=None, max_retry=5):
+def run_conversation(query: str, stream=False,  max_retry=5):
     params = dict(model="chatglm3", messages=[{"role": "user", "content": query}], stream=stream)
-    if functions:
-        params["functions"] = functions
+    params["functions"] = functions
     response = get_response(**params)
 
     for _ in range(max_retry):
         if response["choices"][0]["message"].get("function_call"):
             function_call = response["choices"][0]["message"]["function_call"]
+            logger.info(f"Function Call Response: {function_call}")
             if "sub_task" in function_call["name"]:
                 return {
                     "Answer": "好的",
-                    "Goal": function_call["arguments"]
+                    "Goal": json.loads(function_call["arguments"])["goal"]
                 }
 
-            logger.info(f"Function Call Response: {function_call}")
             function_args = json.loads(function_call["arguments"])
             tool_response = dispatch_tool(function_call["name"], function_args)
             logger.info(f"Tool Call Response: {tool_response}")
@@ -71,5 +70,5 @@ def run_conversation(query: str, stream=False, functions=None, max_retry=5):
 
 
 if __name__ == "__main__":
-    query = "关掉空调"
-    print(run_conversation(query, functions=functions, stream=False))
+    query = "可以带我去吗"
+    print(run_conversation(query, stream=False))

robowaiter/llm_client/tool_register.py

@@ -40,7 +40,7 @@ def register_tool(func: callable):
         "params": tool_params
     }
 
-    print("[registered tool] " + pformat(tool_def))
+    # print("[registered tool] " + pformat(tool_def))
     _TOOL_HOOKS[tool_name] = func
     _TOOL_DESCRIPTIONS[tool_name] = tool_def
 
@@ -126,13 +126,14 @@ def get_tools() -> dict:
 
 @register_tool
 def create_sub_task(
-        goal: Annotated[str, '子任务需要达到的目标条件集合', True]
+        goal: Annotated[str, '子任务需要达到的目标条件集合，例如{On(Coffee,Bar)}，{At(Robot,Table1)}，{Is(AC,Off)}', True]
 ) -> str:
     """
     当需要完成具身任务（如做咖啡，拿放物体，扫地，前往某位置）时，调用该函数，根据用户的提示进行意图理解，生成子任务的目标状态集合 `goal`（以一阶逻辑的形式表示），用户意图
-    做一杯咖啡,`goal`={On(Coffee,Bar)},
-    前往一号桌,`goal`={At(Robot,Table1)},
-    打开空调,`goal`={Is(AC,On)},。
+    做一杯咖啡,则该函数的参数为 "On(Coffee,Bar)",
+    前往一号桌,则该函数的参数为 "At(Robot,Table1)",
+    打开空调,则该函数的参数为 "Is(AC,On)",。
+    关空调,则该函数的参数为 "Is(AC,Off)",。
     """
 
     return goal
@@ -142,7 +143,7 @@ def get_object_info(
         obj: Annotated[str, '需要获取信息的物体名称', True]
 ) -> str:
     """
-    获取场景中指定物体 `object` 的信息
+    获取场景中指定物体 `object` 的信息，如果`object` 是一个地点，例如洗手间，地方，则输出。如果`object`是一个咖啡，则输出。
     """
     near_object = None
     if obj == "Table":

robowaiter/scene/scene.py

@@ -332,6 +332,23 @@ class Scene:
             )
         )
 
+    # def walker_bubble(self, message):
+    #     status = self.status
+    #     walker_name = status.walkers[0].name
+    #     talk_content = walker_name + ":" + message
+    #     self.control_robot_action(0, 0, 3, talk_content)
+
+    # def control_robot_action(self, scene_id=0, type=0, action=0, message="你好"):
+    #     print('------------------control_robot_action----------------------')
+    #     scene = stub.ControlRobot(
+    #         GrabSim_pb2.ControlInfo(scene=scene_id, type=type, action=action, content=message))
+    #     if (str(scene.info).find("Action Success") > -1):
+    #         print(scene.info)
+    #         return True
+    #     else:
+    #         print(scene.info)
+    #         return False
+
     def animation_control(self, animation_type):
         # animation_type: 1:make coffee 2: pour water 3: grab food 4: mop floor 5: clean table
         scene = stub.ControlRobot(

robowaiter/scene/tasks/GQA.py

@@ -16,15 +16,30 @@ class SceneGQA(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
         self.event_list = [
-            (5, self.create_chat_event("给我一杯咖啡")),
-            (20, self.create_chat_event("我要拿铁")),
-            (40, self.create_chat_event("再来一杯")),
+            (5, self.create_chat_event("洗手间在哪里")),
+            (12, self.create_chat_event("可以带我去吗")),
         ]
 
     def _reset(self):
-        self.add_walker(1085, 2630, 220)
-        self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)])
+        self.clean_walker()
+
+
+        self.add_walker(50, 500, 0)
+        self.walker_bubble("洗手间在哪里")
+        # self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)])
 
 
     def _run(self):
         pass
+
+
+if __name__ == '__main__':
+    import os
+    from robowaiter.robot.robot import Robot
+
+    robot = Robot()
+
+    # create task
+    task = SceneGQA(robot)
+    task.reset()
+    task.run()

robowaiter/scene/tasks/Open_tasks.py

@@ -18,11 +18,12 @@ class SceneOT(Scene):
         super().__init__(robot)
         # 在这里加入场景中发生的事件
         self.event_list = [
-            (5,self.create_chat_event("给我一杯咖啡"))  # (事件发生的时间，事件函数)
+            (5,self.create_chat_event("来二号桌")),
+            # (5,self.create_chat_event("感觉有点冷，可以关一下空调吗")),
         ]
 
     def _reset(self):
-        pass
+        self.add_walker(50, 300, 0)
         # self.add_walker(1085, 2630, 220)
         # self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)])
 
@@ -31,3 +32,13 @@ class SceneOT(Scene):
         pass
 
 
+if __name__ == '__main__':
+    import os
+    from robowaiter.robot.robot import Robot
+
+    robot = Robot()
+
+    # create task
+    task = SceneOT(robot)
+    task.reset()
+    task.run()

robowaiter/scene/tasks/Open_tasks_test.py

@@ -0,0 +1,44 @@
+"""
+人提出请求，机器人完成任务
+1. 做咖啡（固定动画）：接收到做咖啡指令、走到咖啡机、拿杯子、操作咖啡机、取杯子、送到客人桌子上
+2. 倒水
+3. 夹点心
+
+具体描述：设计一套点单规则（如菜单包含咖啡、水、点心等），按照规则拟造随机的订单。在收到订单后，通过大模型让机器人输出合理的备餐计划，并尝试在模拟环境中按照这个规划实现任务。
+
+"""
+
+# todo: 接收点单信息，大模型生成任务规划
+
+from robowaiter.scene.scene import Scene
+
+class SceneOT(Scene):
+
+    def __init__(self, robot):
+        super().__init__(robot)
+        # 在这里加入场景中发生的事件
+        self.event_list = [
+            # (5,self.create_chat_event("做一杯咖啡")),
+            (5,self.create_chat_event("感觉有点冷，可以关一下空调吗")),
+        ]
+
+    def _reset(self):
+        self.add_walker(50, 300, 0)
+        # self.add_walker(1085, 2630, 220)
+        # self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)])
+
+
+    def _run(self):
+        pass
+
+
+if __name__ == '__main__':
+    import os
+    from robowaiter.robot.robot import Robot
+
+    robot = Robot()
+
+    # create task
+    task = SceneOT(robot)
+    task.reset()
+    task.run()