diff --git a/robowaiter/algos/retrieval/retrieval_lm/fix_questions.jsonl b/robowaiter/algos/retrieval/retrieval_lm/fix_questions.jsonl index 2241464..c6f7915 100644 --- a/robowaiter/algos/retrieval/retrieval_lm/fix_questions.jsonl +++ b/robowaiter/algos/retrieval/retrieval_lm/fix_questions.jsonl @@ -19,23 +19,28 @@ {"id": 18, "title": "好热呀！太阳也好大！", "text": "{'answer': '抱歉，我这就去开空调和关窗帘。', 'function': 'create_sub_task', 'args': '{\"goal\":\"Is(AC,On),Is(Curtain,Off)\"}'}"} {"id": 19, "title": "下班啦！别忘了打扫卫生。", "text": "{'answer': '收到！下班啦！', 'function': 'create_sub_task', 'args': '{\"goal\":\"Is(Floor,Clean),Is(Table1,Clean),Is(Chairs,Clean),Is(AC,Off),Is(HallLight,Off),Is(TubeLight,Off),Is(Curtain,Off)\"}'}"} {"id": 20, "title": "请问洗手间在哪里？", "text": "{'answer': '洗手间在这附近', 'function': 'get_object_info', 'args': '{\"obj\":\"洗手间\"}'}"} -{"id": 21, "title": "给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。", "text": "{'answer': '好的，请稍等。', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Chips,WaterTable),On(NFCJuice,WaterTable)\"}'}"} -{"id": 22, "title": "给我来份薯片和果汁，我坐在对面的桌子那儿。", "text": "{'answer': '好的，请稍等。', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(BottledDrink,WaterTable),On(Yogurt,WaterTable)\"}'}"} -{"id": 23, "title": "好热呀，想开空调，想要温度调低点！", "text": "{'answer': '抱歉，我马上去开空调并调低空调温度。', 'function': 'create_sub_task', 'args': '{\"goal\":\"Is(ACTemperature,Down)\"}'}"} -{"id": 24, "title": "请问哪里有空位啊？", "text": "{'answer': '现在有不少空位呢，请问您有什么要求嘛？', 'function': None}"} -{"id": 25, "title": "我想坐高脚凳子。", "text": "{'answer': '没问题，大厅里就有这样的位置呢！', 'function': None}"} -{"id": 26, "title": "你带我去吧。", "text": "{'answer': 'OK，请跟我来！', 'function': 'create_sub_task', 'args': '{\"goal\":\"At(Robot,BrightTable5)\"}'}"} -{"id": 27, "title": "可以关筒灯和关窗帘吗？", "text": "{'answer': '好的，请稍等。', 'function': 'create_sub_task', 'args': '{\"goal\":\"Is(TubeLight,Off),Is(Curtain,Off)\"}'}"} -{"id": 28, "title": "给我来杯咖啡，哦对，再倒一杯水。", "text": "{'answer': '速速就好，请稍等！', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Water,WaterTable),On(Coffee,WaterTable)\"}'}"} -{"id": 29, "title": "我想来份点心和酸奶。", "text": "{'answer': '稍等片刻噢~', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Yogurt,Bar),On(Dessert,Bar)\"}'}"} -{"id": 30, "title": "酸奶。", "text": "{'answer': '稍等片刻噢~', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Yogurt,Bar)\"}'}"} -{"id": 31, "title": "你好呀，你们这有啥好吃的？", "text": "{'answer': '我们咖啡厅有咖啡，还有饮料和点心蛋糕呢？您想点什么不？', 'function': None}"} -{"id": 32, "title": "听起来都好甜呀，我女朋友爱吃水果。", "text": "{'answer': '没问题，我们也提供草莓、橘子和苹果等水果呢。', 'function': None}"} -{"id": 33, "title": "你们这人可真多。", "text": "{'answer': '目前还有空位呢，你们可以坐下来一起品尝水果呢？', 'function': 'get_object_info', 'args': '{\"obj\":\"空位\"}'}"} -{"id": 34, "title": "我女朋友怕晒，有空余的阴凉位置嘛？", "text": "{'answer': '我查询完毕，阴凉位置有两个。在你们右侧往里走，就有适合你们二位的沙发呢。', 'function': 'get_object_info', 'args': '{\"obj\":\"阴凉空位\"}'}"} -{"id": 35, "title": "那还不错。", "text": "{'answer': '是的，那边安静，不会晒到太阳呢。', 'function': None}"} -{"id": 36, "title": "我们还想一起下下棋,切磋切磋。", "text": "{'answer': '我们这有两份军棋呢,在大厅的三号桌上。', 'function': 'get_object_info', 'args': '{\"obj\":\"棋\"}'}"} -{"id": 37, "title": "请问洗手间在哪呢？", "text": "{'answer': '在后门，你们的左侧往里走就是了呢。', 'function': 'get_object_info', 'args': '{\"obj\":\"洗手间\"}'}"} -{"id": 38, "title": "我带着孩子呢，想要宽敞亮堂的地方。", "text": "{'answer': '好的，我明白了，那么我们推荐您到大厅的桌子，那里的空间比较宽敞，环境也比较明亮，适合带着孩子一起用餐。', 'function': None}"} -{"id": 39, "title": "冰红茶", "text": "{'answer': '好的', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Softdrink,Bar)\"}'}"} -{"id": 40, "title": "水杯", "text": "{'answer': '好的', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Glass,Bar)\"}'}"} +{"id": 21, "title": "给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。", "text": "{'answer': '好的，请稍等。', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(BottledDrink,WaterTable),On(Yogurt,WaterTable)\"}'}"} +{"id": 22, "title": "给我来份薯片和果汁，我坐在对面的桌子那儿。", "text": "{'answer': '好的，请稍等。', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Chips,WaterTable),On(NFCJuice,WaterTable)\"}'}"} +{"id": 23, "title": "给我来份薯片和果汁，我坐在对面的水杯桌那儿。", "text": "{'answer': '好的，请稍等。', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Chips,WaterTable),On(NFCJuice,WaterTable)\"}'}"} +{"id": 24, "title": "好热呀，想开空调，想要温度调低点！", "text": "{'answer': '抱歉，我马上去开空调并调低空调温度。', 'function': 'create_sub_task', 'args': '{\"goal\":\"Is(ACTemperature,Down)\"}'}"} +{"id": 25, "title": "请问有空位吗？", "text": "{'answer': '现在有不少空位呢，请问您有什么要求嘛？', 'function': None}"} +{"id": 26, "title": "我想坐高脚凳子。", "text": "{'answer': '没问题，大厅里就有这样的位置呢！', 'function': 'get_object_info', 'args': '{\"obj\":\"高脚凳子的空位\"}'}"} +{"id": 27, "title": "你带我去吧。", "text": "{'answer': 'OK，请跟我来！', 'function': 'create_sub_task', 'args': '{\"goal\":\"At(Robot,BrightTable5)\"}'}"} +{"id": 28, "title": "可以关筒灯和关窗帘吗？", "text": "{'answer': '好的，请稍等。', 'function': 'create_sub_task', 'args': '{\"goal\":\"Is(TubeLight,Off),Is(Curtain,Off)\"}'}"} +{"id": 29, "title": "给我来杯咖啡，哦对，再倒一杯水。", "text": "{'answer': '速速就好，请稍等！', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Water,WaterTable),On(Coffee,WaterTable)\"}'}"} +{"id": 30, "title": "我想来份点心和酸奶。", "text": "{'answer': '稍等片刻噢~', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Yogurt,Bar),On(Dessert,Bar)\"}'}"} +{"id": 31, "title": "酸奶。", "text": "{'answer': '稍等片刻噢~', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Yogurt,Bar)\"}'}"} +{"id": 32, "title": "你好呀，你们这有啥好吃的？", "text": "{'answer': '我们咖啡厅有咖啡，还有饮料和点心蛋糕呢？您想点什么不？', 'function': 'get_object_info', 'args': '{\"obj\":\"饮料,水果,点心\"}'}"} +{"id": 33, "title": "听起来都好甜呀，我女朋友爱吃水果。", "text": "{'answer': '没问题，我们也提供草莓、橘子和苹果等水果呢。', 'function': None}"} +{"id": 34, "title": "你们这人可真多。", "text": "{'answer': '目前还有空位呢，你们可以坐下来一起品尝水果呢？', 'function': 'get_object_info', 'args': '{\"obj\":\"空位\"}'}"} +{"id": 35, "title": "我女朋友怕晒，有空余的阴凉位置嘛？", "text": "{'answer': '我查询完毕，阴凉位置有两个。在你们右侧往里走，就有适合你们二位的沙发呢。', 'function': 'get_object_info', 'args': '{\"obj\":\"阴凉空位\"}'}"} +{"id": 36, "title": "那还不错。", "text": "{'answer': '是的，那边安静，不会晒到太阳呢。', 'function': None}"} +{"id": 37, "title": "我们还想一起下下棋,切磋切磋。", "text": "{'answer': '我们这有两份军棋呢,在大厅的三号桌上。', 'function': 'get_object_info', 'args': '{\"obj\":\"棋\"}'}"} +{"id": 38, "title": "请问洗手间在哪呢？", "text": "{'answer': '在后门，你们的左侧往里走就是了呢。', 'function': 'get_object_info', 'args': '{\"obj\":\"洗手间\"}'}"} +{"id": 39, "title": "我带着孩子呢，想要宽敞亮堂的地方。", "text": "{'answer': '好的，我明白了，那么我们推荐您到大厅的桌子，那里的空间比较宽敞，环境也比较明亮，适合带着孩子一起用餐。', 'function': None}"} +{"id": 40, "title": "冰红茶", "text": "{'answer': '好的', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Softdrink,Bar)\"}'}"} +{"id": 41, "title": "水杯", "text": "{'answer': '好的', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Water,Bar)\"}'}"} +{"id": 42, "title": "点心", "text": "{'answer': '好的', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Dessert,Bar)\"}'}"} +{"id": 43, "title": "酸奶", "text": "{'answer': '好的', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Yogurt,Bar)\"}'}"} +{"id": 44, "title": "做一杯咖啡", "text": "{'answer': '好的，我马上做咖啡', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Coffee,CoffeeTable)\"}'}"} +{"id": 45, "title": "把盒装冰红茶放到水桌", "text": "{'answer': '好的', 'function': 'create_sub_task', 'args': '{\"goal\":\"On(Softdrink,WaterTable)\"}'}"} diff --git a/robowaiter/algos/retrieval/retrieval_lm/fix_questions.txt b/robowaiter/algos/retrieval/retrieval_lm/fix_questions.txt index 9b2196a..3d5df4d 100644 --- a/robowaiter/algos/retrieval/retrieval_lm/fix_questions.txt +++ b/robowaiter/algos/retrieval/retrieval_lm/fix_questions.txt @@ -130,7 +130,7 @@ create_sub_task create_sub_task {"goal":"Is(ACTemperature,Down)"} -请问哪里有空位啊？ +请问有空位吗？ 现在有不少空位呢，请问您有什么要求嘛？ 我想坐高脚凳子。 diff --git a/robowaiter/algos/retrieval/retrieval_lm/fix_questions_test.jsonl b/robowaiter/algos/retrieval/retrieval_lm/fix_questions_test.jsonl index ec75be3..458279c 100644 --- a/robowaiter/algos/retrieval/retrieval_lm/fix_questions_test.jsonl +++ b/robowaiter/algos/retrieval/retrieval_lm/fix_questions_test.jsonl @@ -21,21 +21,26 @@ {"id": 20, "question": "请问洗手间在哪里？"} {"id": 21, "question": "给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。"} {"id": 22, "question": "给我来份薯片和果汁，我坐在对面的桌子那儿。"} -{"id": 23, "question": "好热呀，想开空调，想要温度调低点！"} -{"id": 24, "question": "请问哪里有空位啊？"} -{"id": 25, "question": "我想坐高脚凳子。"} -{"id": 26, "question": "你带我去吧。"} -{"id": 27, "question": "可以关筒灯和关窗帘吗？"} -{"id": 28, "question": "给我来杯咖啡，哦对，再倒一杯水。"} -{"id": 29, "question": "我想来份点心和酸奶。"} -{"id": 30, "question": "酸奶。"} -{"id": 31, "question": "你好呀，你们这有啥好吃的？"} -{"id": 32, "question": "听起来都好甜呀，我女朋友爱吃水果。"} -{"id": 33, "question": "你们这人可真多。"} -{"id": 34, "question": "我女朋友怕晒，有空余的阴凉位置嘛？"} -{"id": 35, "question": "那还不错。"} -{"id": 36, "question": "我们还想一起下下棋,切磋切磋。"} -{"id": 37, "question": "请问洗手间在哪呢？"} -{"id": 38, "question": "我带着孩子呢，想要宽敞亮堂的地方。"} -{"id": 39, "question": "冰红茶"} -{"id": 40, "question": "水杯"} +{"id": 23, "question": "给我来份薯片和果汁，我坐在对面的水杯桌那儿。"} +{"id": 24, "question": "好热呀，想开空调，想要温度调低点！"} +{"id": 25, "question": "请问有空位吗？"} +{"id": 26, "question": "我想坐高脚凳子。"} +{"id": 27, "question": "你带我去吧。"} +{"id": 28, "question": "可以关筒灯和关窗帘吗？"} +{"id": 29, "question": "给我来杯咖啡，哦对，再倒一杯水。"} +{"id": 30, "question": "我想来份点心和酸奶。"} +{"id": 31, "question": "酸奶。"} +{"id": 32, "question": "你好呀，你们这有啥好吃的？"} +{"id": 33, "question": "听起来都好甜呀，我女朋友爱吃水果。"} +{"id": 34, "question": "你们这人可真多。"} +{"id": 35, "question": "我女朋友怕晒，有空余的阴凉位置嘛？"} +{"id": 36, "question": "那还不错。"} +{"id": 37, "question": "我们还想一起下下棋,切磋切磋。"} +{"id": 38, "question": "请问洗手间在哪呢？"} +{"id": 39, "question": "我带着孩子呢，想要宽敞亮堂的地方。"} +{"id": 40, "question": "冰红茶"} +{"id": 41, "question": "水杯"} +{"id": 42, "question": "点心"} +{"id": 43, "question": "酸奶"} +{"id": 44, "question": "做一杯咖啡"} +{"id": 45, "question": "把盒装冰红茶放到水桌"} diff --git a/robowaiter/algos/retrieval/retrieval_lm/robot_embeddings/passages_00 b/robowaiter/algos/retrieval/retrieval_lm/robot_embeddings/passages_00 index f531ebb..e405ef4 100644 Binary files a/robowaiter/algos/retrieval/retrieval_lm/robot_embeddings/passages_00 and b/robowaiter/algos/retrieval/retrieval_lm/robot_embeddings/passages_00 differ diff --git a/robowaiter/algos/retrieval/retrieval_lm/run_retri_All_In_One.py b/robowaiter/algos/retrieval/retrieval_lm/run_retri_All_In_One.py index 872ca86..29eabaf 100644 --- a/robowaiter/algos/retrieval/retrieval_lm/run_retri_All_In_One.py +++ b/robowaiter/algos/retrieval/retrieval_lm/run_retri_All_In_One.py @@ -21,4 +21,4 @@ cmd_test_retri = f" passage_retrieval2.py --data {output_test_path} --model_nam # os.system(conda_path + cmd_goal_states_with_descrip_to_train) # os.system(conda_path + cmd_goal_states_with_descrip_to_test) os.system("python " + cmd_get_embedding) -os.system("python " + cmd_test_retri) +# os.system("python " + cmd_test_retri) diff --git a/robowaiter/behavior_lib/act/DealChat.py b/robowaiter/behavior_lib/act/DealChat.py index 33b7263..d028332 100644 --- a/robowaiter/behavior_lib/act/DealChat.py +++ b/robowaiter/behavior_lib/act/DealChat.py @@ -105,12 +105,8 @@ class DealChat(Act): print("参数解析错误") - d = {"保温杯": "二号桌子"} - if obj in d.keys(): - result = d[obj] - else: - result = "没有" - return result + near_object = None + d = {"保温杯": "二号桌子","洗手间":"前门","卫生间":"前门"} # 先把 obj 转成中文 @@ -269,7 +265,7 @@ class DealChat(Act): # similar_word = real_obj_name # if similar_word: # print("max_similarity:",max_similarity,"similar_word:",similar_word) - return count + return "有"+str(count)+"个"+obj diff --git a/robowaiter/llm_client/multi_rounds_retri.py b/robowaiter/llm_client/multi_rounds_retri.py index ec30512..10eaad0 100644 --- a/robowaiter/llm_client/multi_rounds_retri.py +++ b/robowaiter/llm_client/multi_rounds_retri.py @@ -35,7 +35,7 @@ root_path = get_root_path() functions = get_tools() -retrieval = Retrieval(threshold=1.8) +retrieval = Retrieval(threshold=1.9) role_system = [{ @@ -156,13 +156,15 @@ def ask_llm(question,history, func_map=None, retry=3): else: result = single_round(f"你是机器人服务员，顾客想知道{question}, 你的具身场景查询返回的是{result},把返回的英文名词翻译成中文,请把按照以下句子对顾客说，{answer}, 尽量简短。\n") - message = {'role': 'assistant', 'content': result, 'name': None, - 'function_call': None} - history.append(message) - else: - _,response,_ = get_response(None, history,allow_function_call=False) - _,result = deal_response(response, history, func_map) + else: + # _,response,_ = get_response(None, history,allow_function_call=False) + # _,result = deal_response(response, history, func_map) + result = single_round(history[-1]["content"], + "你是机器人服务员，请把以下句子换一种表述方式对顾客说，但是意思不变，尽量简短：\n") + message = {'role': 'assistant', 'content': result, 'name': None, + 'function_call': None} + history.append(message) print(f'{len(history)}条历史记录:') for x in history: diff --git a/robowaiter/llm_client/tool_register.py b/robowaiter/llm_client/tool_register.py index a16ee1d..ee70738 100644 --- a/robowaiter/llm_client/tool_register.py +++ b/robowaiter/llm_client/tool_register.py @@ -178,21 +178,14 @@ def get_object_info( `object`在你们这里吗 `object`，你看见过吗？ 遇到上述问题时，调用该函数，并根据返回的结果回复物体位置信息，例如 - 回答： - `object`在输出的物品附近 - 如果`object` 是一个地点，例如洗手间，则输出大门。 - 如果`object`是咖啡，输出桌子，则咖啡在桌子上。 - 如果`object`是酸奶，输出冰红茶，则酸奶在冰红茶附近。 - 如果`object` 是空桌子，则输出一号桌 """ - # 桌上有 - # `object` - # 吗？ - # 咖啡厅里哪里有 - # `object` - # `object` - # 放在哪里？ + # 回答： + # `object`在输出的物品附近 + # 如果`object` 是一个地点，例如洗手间，则输出大门。 + # 如果`object`是咖啡，输出桌子，则咖啡在桌子上。 + # 如果`object`是酸奶，输出冰红茶，则酸奶在冰红茶附近。 + # 如果`object` 是空桌子，则输出一号桌 near_object = None # if obj == "Table": @@ -241,7 +234,7 @@ def get_number_of_objects( obj: Annotated[str, '需要获取数量的物体名称', True] ) -> str: """ - 当询问场景中的物体数量时，需要调用这个工具。 + 获取场景中物体数量。当询问场景中的物体数量时，需要调用这个工具。 例如询问： `obj`有多少？ 有多少`obj` diff --git a/tasks_no_ui/OT/Open_tasks_test.py b/tasks_no_ui/OT/Open_tasks_test.py index a49796c..fe63eb1 100644 --- a/tasks_no_ui/OT/Open_tasks_test.py +++ b/tasks_no_ui/OT/Open_tasks_test.py @@ -27,11 +27,11 @@ class SceneOT(Scene): # (3, self.customer_say, ("System", "你们这儿有多少军棋")), # (3, self.customer_say, ("System", "哪里有盒装饮料？")), - # (3, self.customer_say, ("System", "洗手间在哪里？")),卫生间 + # (3, self.customer_say, ("System", "洗手间在哪里？")),#卫生间 # (3, self.customer_say, ("System", "卫生间在哪里？")), # (3, self.customer_say, ("System", "你们这里有棋吗，在哪里")), - + # (3, self.customer_say, ("System", "棋在哪里")) # (3, self.customer_say, ("System", "有几副棋")), # (3, self.customer_say, ("System", "我的手镯，你有看到吗")), # (3, self.customer_say, ("System", "你们这有小说吗？")), @@ -41,6 +41,10 @@ class SceneOT(Scene): # (3, self.customer_say, ("System", "这有小说吗？在哪里")), + # (3, self.customer_say, ("System", "还有空位吗")), + + (3, self.customer_say, ("System", "有空桌子吗")), + # (3, self.customer_say, ("System", "把盒装冰红茶放到水桌")), # (3, self.customer_say, ("System", "冰红茶")),