From 8bdf5a96b44540fec9def6f3add800e7e80240d4 Mon Sep 17 00:00:00 2001 From: ChenXL97 <908926798@qq.com> Date: Sat, 18 Nov 2023 22:30:14 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=87=BD=E6=95=B0=E8=9E=8D?= =?UTF-8?q?=E5=85=A5=E5=9C=BA=E6=99=AF=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/behavior_lib/act/DealChat.py | 19 +++++++- robowaiter/llm_client/data/fix_questions.txt | 7 +++ robowaiter/llm_client/multi_rounds.py | 37 +++++++++++---- robowaiter/llm_client/tool_register.py | 50 ++++++++++---------- robowaiter/scene/tasks/{ => GQA}/GQA.py | 5 +- robowaiter/scene/tasks/GQA/__init__.py | 0 6 files changed, 81 insertions(+), 37 deletions(-) rename robowaiter/scene/tasks/{ => GQA}/GQA.py (88%) create mode 100644 robowaiter/scene/tasks/GQA/__init__.py diff --git a/robowaiter/behavior_lib/act/DealChat.py b/robowaiter/behavior_lib/act/DealChat.py index 0bdcf70..e66ed1c 100644 --- a/robowaiter/behavior_lib/act/DealChat.py +++ b/robowaiter/behavior_lib/act/DealChat.py @@ -9,7 +9,8 @@ class DealChat(Act): self.chat_history = "" self.function_success = False self.func_map = { - "create_sub_task": self.create_sub_task + "create_sub_task": self.create_sub_task, + "get_object_info": self.get_object_info } def _update(self) -> ptree.common.Status: @@ -53,3 +54,19 @@ class DealChat(Act): print("参数解析错误") self.scene.robot.expand_sub_task_tree(goal_set) + + + def get_object_info(self,**args): + try: + obj = args['obj'] + + self.function_success = True + except: + obj = None + print("参数解析错误") + + near_object = "None" + if obj == "洗手间": + near_object = "大门" + + return near_object \ No newline at end of file diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt index 0fa6d5f..3ca9173 100644 --- a/robowaiter/llm_client/data/fix_questions.txt +++ b/robowaiter/llm_client/data/fix_questions.txt @@ -36,3 +36,10 @@ create_sub_task 抱歉，我这就去开空调和关窗帘。 create_sub_task {"goal":"Is(AC,On),Is(Curtain,Off)"} + + + +请问洗手间在哪里？ +洗手间在这附近 +get_object_info +{"obj":"洗手间"} diff --git a/robowaiter/llm_client/multi_rounds.py b/robowaiter/llm_client/multi_rounds.py index 0726576..81ca2d8 100644 --- a/robowaiter/llm_client/multi_rounds.py +++ b/robowaiter/llm_client/multi_rounds.py @@ -29,16 +29,30 @@ root_path = get_root_path() # load test questions file_path = os.path.join(root_path,"robowaiter/llm_client/data/fix_questions.txt") +functions = get_tools() + fix_questions_dict = {} +no_reply_functions = ["create_sub_task"] + with open(file_path,'r',encoding="utf-8") as f: #读取所有行 lines = f.read().strip() sections = re.split(r'\n\s*\n', lines) for s in sections: x = s.split() - fix_questions_dict[x[0]] = x[1:] + if len(x) == 2: + fix_questions_dict[x[0]] = { + "answer": x[1], + "function": None + } + else: + fix_questions_dict[x[0]] = { + "answer": x[1], + "function": x[2], + "args": x[3] + } + -functions = get_tools() role_system = [{ "role": "system", "content": "你是RoboWaiter,一个由HPCL团队开发的机器人服务员，你在咖啡厅工作。接受顾客的指令并调用工具函数来完成各种服务任务。如果顾客问你们这里有什么，或者想要点单，你说我们咖啡厅提供咖啡，水，点心，酸奶等食物。如果顾客不需要你了，你就回到吧台招待。如果顾客叫你去做某事，你回复：好的，我马上去做这件事。", @@ -55,11 +69,12 @@ def new_response(): def parse_fix_question(question): response = new_response() fix_ans = fix_questions_dict[question] - if len(fix_ans)<=1: #简单对话 - message = {'role': 'assistant', 'content': fix_ans[0], 'name': None, + if not fix_ans['function']: #简单对话 + message = {'role': 'assistant', 'content': fix_ans["answer"], 'name': None, 'function_call': None} else: - reply, func,args = fix_ans + func = fix_ans["function"] + args = fix_ans["args"] # tool_response = dispatch_tool(function_call["name"], json.loads(args)) # logger.info(f"Tool Call Response: {tool_response}") message = {'role': 'assistant', @@ -115,7 +130,7 @@ def deal_response(response, history, func_map=None ): } history.append(t) - return function_call["name"], return_message + return function_call["name"], tool_response else: return_message = response["choices"][0]["message"] @@ -133,8 +148,14 @@ def ask_llm(question,history, func_map=None, retry=3): function_call,result = deal_response(response, history, func_map) if function_call: if question in fix_questions_dict: - reply = fix_questions_dict[question][0] - result = single_round(reply,"你是机器人服务员，请把以下句子换一种表述方式对顾客说，但是意思不变，尽量简短：\n") + if fix_questions_dict[question]['function'] in no_reply_functions: + reply = fix_questions_dict[question]["answer"] + result = single_round(reply, + "你是机器人服务员，请把以下句子换一种表述方式对顾客说，但是意思不变，尽量简短：\n") + else: + reply = fix_questions_dict[question]["answer"] + result = single_round(f"你是机器人服务员，顾客想知道{question}, 你的具身场景查询返回的是{result},请把按照以下句子对顾客说，{reply}, 尽量简短。\n") + message = {'role': 'assistant', 'content': result, 'name': None, 'function_call': None} history.append(message) diff --git a/robowaiter/llm_client/tool_register.py b/robowaiter/llm_client/tool_register.py index 1e4b42e..f15a927 100644 --- a/robowaiter/llm_client/tool_register.py +++ b/robowaiter/llm_client/tool_register.py @@ -158,31 +158,31 @@ def stop_serve( -# @register_tool -# def get_object_info( -# obj: Annotated[str, '需要获取信息的物体名称', True] -# ) -> str: -# """ -# 获取场景中指定物体 `object` 在哪里，不涉及到具体的执行任务 -# 如果`object` 是一个地点，例如洗手间，则输出大门。 -# 如果`object`是咖啡，则输出桌子，咖啡在桌子上。 -# 如果`object` 是空桌子，则输出一号桌 -# """ -# near_object = None -# # if obj == "Table": -# # near_object = "Bar" -# # if obj == "洗手间": -# # near_object = "大门" -# # if obj == "空桌子": -# # near_object = "一号桌" -# if obj in find_obj_utils.all_loc: # object是一个地点 -# mp = list(find_obj_utils.loc_map[obj]) -# # near_object = random.choice(mp) -# near_object = mp -# if obj in find_obj_utils.all_obj: # object是一个物品 -# near_ls = find_obj_utils.all_loc + find_obj_utils.all_obj -# near_object = random.choices(near_ls,k=5) -# return near_object +@register_tool +def get_object_info( + obj: Annotated[str, '需要获取信息的物体名称', True] +) -> str: + """ + 获取场景中指定物体 `object` 在哪里，不涉及到具体的执行任务 + 如果`object` 是一个地点，例如洗手间，则输出大门。 + 如果`object`是咖啡，则输出桌子，咖啡在桌子上。 + 如果`object` 是空桌子，则输出一号桌 + """ + near_object = None + # if obj == "Table": + # near_object = "Bar" + if obj == "洗手间": + near_object = "大门" + # if obj == "空桌子": + # near_object = "一号桌" + if obj in find_obj_utils.all_loc: # object是一个地点 + mp = list(find_obj_utils.loc_map[obj]) + # near_object = random.choice(mp) + near_object = mp + if obj in find_obj_utils.all_obj: # object是一个物品 + near_ls = find_obj_utils.all_loc + find_obj_utils.all_obj + near_object = random.choices(near_ls,k=5) + return near_object # @register_tool # def find_location( diff --git a/robowaiter/scene/tasks/GQA.py b/robowaiter/scene/tasks/GQA/GQA.py similarity index 88% rename from robowaiter/scene/tasks/GQA.py rename to robowaiter/scene/tasks/GQA/GQA.py index 577d9b6..54b9063 100644 --- a/robowaiter/scene/tasks/GQA.py +++ b/robowaiter/scene/tasks/GQA/GQA.py @@ -15,9 +15,8 @@ class SceneGQA(Scene): def __init__(self, robot): super().__init__(robot) # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) - self.event_list = [ - (5, self.create_chat_event("哪里有空桌子")), - (12, self.create_chat_event("可以带我去吗")), + self.new_event_list = [ + (3, self.customer_say, ("System","请问洗手间在哪里？")) ] def _reset(self): diff --git a/robowaiter/scene/tasks/GQA/__init__.py b/robowaiter/scene/tasks/GQA/__init__.py new file mode 100644 index 0000000..e69de29