diff --git a/robowaiter/behavior_lib/_base/Act.py b/robowaiter/behavior_lib/_base/Act.py index b402ed9..244a060 100644 --- a/robowaiter/behavior_lib/_base/Act.py +++ b/robowaiter/behavior_lib/_base/Act.py @@ -8,7 +8,7 @@ class Act(Bahavior): super().__init__(*args) def get_conds(self): - pre = set() - add = set() - de = set() - return pre, add, de \ No newline at end of file + self.pre = set() + self.add = set() + self.de = set() + return self.pre, self.add, self.de \ No newline at end of file diff --git a/robowaiter/behavior_lib/act/MakeCoffee.py b/robowaiter/behavior_lib/act/MakeCoffee.py index d4903fe..b14727f 100644 --- a/robowaiter/behavior_lib/act/MakeCoffee.py +++ b/robowaiter/behavior_lib/act/MakeCoffee.py @@ -10,7 +10,7 @@ class MakeCoffee(Act): @property def cond_sets(self): - pre = {"At(Robot,Bar)"} + pre = {"At(Robot,CoffeeMachine)","NotHolding"} add = {"At(Coffee,Bar)"} de = {} return pre,add,de @@ -19,5 +19,5 @@ class MakeCoffee(Act): op_type = 1 self.scene.move_task_area(op_type) self.scene.op_task_execute(op_type) - self.scene.state["condition_set"].add("At(Coffee,Bar)") + self.scene.state["condition_set"].add(self.add) return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/act/PourWater.py b/robowaiter/behavior_lib/act/PourWater.py new file mode 100644 index 0000000..23db1fe --- /dev/null +++ b/robowaiter/behavior_lib/act/PourWater.py @@ -0,0 +1,23 @@ +import py_trees as ptree +from typing import Any +from robowaiter.behavior_lib._base.Act import Act +from robowaiter.behavior_lib._base.Behavior import Status + +class PourWater(Act): + + def __init__(self, *args): + super().__init__(*args) + + @property + def cond_sets(self): + self.pre = {"At(Robot,WaterTable)","NotHolding"} + self.add = {"On(Water,WaterTable)"} + self.de = {} + return self.pre,self.add,self.de + + def _update(self) -> ptree.common.Status: + op_type = 2 + self.scene.move_task_area(op_type) + self.scene.op_task_execute(op_type) + self.scene.state["condition_set"].update(self.add) + return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_tree/obtea/OptimalBTExpansionAlgorithm.py b/robowaiter/behavior_tree/obtea/OptimalBTExpansionAlgorithm.py index 7f0010f..333a644 100644 --- a/robowaiter/behavior_tree/obtea/OptimalBTExpansionAlgorithm.py +++ b/robowaiter/behavior_tree/obtea/OptimalBTExpansionAlgorithm.py @@ -94,10 +94,13 @@ class OptBTExpAlgorithm: # Mount the action node and extend BT. T = Eapand(T,c,A(c)) if c!=goal: - sequence_structure = ControlBT(type='>') - sequence_structure.add_child( - [copy.deepcopy(pair_node.cond_leaf), copy.deepcopy(pair_node.act_leaf)]) - subtree.add_child([copy.deepcopy(sequence_structure)]) # subtree 是回不断变化的，它的父亲是self.bt + if c!=set(): + sequence_structure = ControlBT(type='>') + sequence_structure.add_child( + [copy.deepcopy(pair_node.cond_leaf), copy.deepcopy(pair_node.act_leaf)]) + subtree.add_child([copy.deepcopy(sequence_structure)]) # subtree 是回不断变化的，它的父亲是self.bt + else: + subtree.add_child([copy.deepcopy(pair_node.act_leaf)]) if self.verbose: print("完成扩展 a_node= %s,对应的新条件 c_attr= %s,mincost=%d" \ % (cond_anc_pair.act_leaf.content.name, cond_anc_pair.cond_leaf.content, @@ -183,7 +186,10 @@ class OptBTExpAlgorithm: c_set_str = ', '.join(map(str, child.content)) + "\n" self.ptml_string += c_set_str elif child.type == 'act': - self.ptml_string += 'act '+child.content.name+"\n" + if '(' not in child.content.name: + self.ptml_string += 'act '+child.content.name+"()\n" + else: + self.ptml_string += 'act ' + child.content.name + "\n" elif isinstance(child, ControlBT): if child.type == '?': self.ptml_string += "selector{\n" diff --git a/robowaiter/llm_client/MemGPT_simple/.env b/robowaiter/llm_client/MemGPT_simple/.env index 1ce720e..1c9626b 100644 --- a/robowaiter/llm_client/MemGPT_simple/.env +++ b/robowaiter/llm_client/MemGPT_simple/.env @@ -1,2 +1,4 @@ -OPENAI_API_BASE= +#BACKEND_TYPE=webui +#OPENAI_API_BASE=https://45.125.46.134:25344/v1/chat/completions +OPENAI_API_BASE=https://45.125.46.134:25344/v1/chat/completions OPENAI_API_KEY= \ No newline at end of file diff --git a/robowaiter/llm_client/MemGPT_simple/agent.py b/robowaiter/llm_client/MemGPT_simple/agent.py index e49ab4e..c46758f 100644 --- a/robowaiter/llm_client/MemGPT_simple/agent.py +++ b/robowaiter/llm_client/MemGPT_simple/agent.py @@ -170,7 +170,8 @@ class Agent: ) else: # Standard non-function reply - print("### Internal monologue: " + (response_message.content if response_message.content else "")) + # print("### Internal monologue: " + (response_message.content if response_message.content else "")) + print("### Internal monologue: " + (response_message['content'] if response_message['content'] else "")) messages.append(response_message) function_failed = None @@ -178,13 +179,37 @@ class Agent: def step(self, user_message): input_message_sequence = self.messages + [{"role": "user", "content": user_message}] - response = openai.ChatCompletion.create(model=self.model, messages=input_message_sequence, - functions=self.functions_description, function_call="auto") - response_message = response.choices[0].message - response_message_copy = response_message.copy() + # 原来的通信方式 + # response = openai.ChatCompletion.create(model=self.model, messages=input_message_sequence, + # functions=self.functions_description, function_call="auto") + # + # response_message = response.choices[0].message + # response_message_copy = response_message.copy() + + # ===我们的通信方式 "tools": self.functions_description 不起作用=== + import requests + url = "https://45.125.46.134:25344/v1/chat/completions" + headers = {"Content-Type": "application/json"} + data = { + "model": "RoboWaiter", + "messages": input_message_sequence, + # "functions":self.functions_description, + # "function_call":"auto" + # "function_call":self.functions_description + "tools": self.functions_description + } + response = requests.post(url, headers=headers, json=data, verify=False) + if response.status_code == 200: + result = response.json() + response_message = result['choices'][0]['message'] + else: + response_message = "大模型请求失败:"+ str(response.status_code) + response_message_copy = response_message + # ===我们的通信方式 "tools": self.functions_description 不起作用=== + + all_response_messages, function_failed = self.handle_ai_response(response_message) - assert "api_response" not in all_response_messages[0], f"api_response already in {all_response_messages[0]}" all_response_messages[0]["api_response"] = response_message_copy assert "api_args" not in all_response_messages[0], f"api_args already in {all_response_messages[0]}" diff --git a/robowaiter/llm_client/MemGPT_simple/functions_zh.py b/robowaiter/llm_client/MemGPT_simple/functions_zh.py new file mode 100644 index 0000000..2e06f63 --- /dev/null +++ b/robowaiter/llm_client/MemGPT_simple/functions_zh.py @@ -0,0 +1,17 @@ +FUNCTIONS = [ + { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string"}, + }, + "required": ["location"], + }, + } +] diff --git a/robowaiter/llm_client/MemGPT_simple/main.py b/robowaiter/llm_client/MemGPT_simple/main.py index 1b84d03..86fc9f1 100644 --- a/robowaiter/llm_client/MemGPT_simple/main.py +++ b/robowaiter/llm_client/MemGPT_simple/main.py @@ -2,10 +2,14 @@ from dotenv import load_dotenv load_dotenv() import utils -from functions import FUNCTIONS +# from functions import FUNCTIONS +from functions_zh import FUNCTIONS from agent import Agent +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + def run_agent_loop(agent): while True: user_input = input("You: ") @@ -20,6 +24,11 @@ if __name__ == "__main__": persona = utils.get_persona_text() human = utils.get_human_text() system = utils.get_system_text() + + print("system:",system) + print("persona:", persona) + print("human:", human) + agent = Agent(model="gpt-3.5-turbo-16k-0613", system=system, functions_description=FUNCTIONS, persona_notes=persona, human_notes=human) run_agent_loop(agent) diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt index 823fa74..aa9f62c 100644 --- a/robowaiter/llm_client/data/test_questions.txt +++ b/robowaiter/llm_client/data/test_questions.txt @@ -1 +1,2 @@ -{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLN：前往桌子": {"Answer": "测试VLN：前往桌子", "Goal": "{\"At(Robot,Table)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}} +{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLN：前往桌子": {"Answer": "测试VLN：前往桌子", "Goal": "{\"At(Robot,Table)\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"At(Water,WaterTable)\"}"}} + diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv index 7523658..de611cc 100644 --- a/robowaiter/llm_client/data_raw/test_questions.csv +++ b/robowaiter/llm_client/data_raw/test_questions.csv @@ -2,3 +2,4 @@ Question,Answer,Goal VLMһ,VLMһ,"{""At(Coffee,Bar)""}" VLNǰ,VLNǰ,"{""At(Robot,Table)""}" AEM,AEM,"{""EnvExplored()""}" +VLMһˮ,VLMһˮ,"{""At(Water,WaterTable)""}" diff --git a/robowaiter/robot/robot.py b/robowaiter/robot/robot.py index 2e1253d..313e562 100644 --- a/robowaiter/robot/robot.py +++ b/robowaiter/robot/robot.py @@ -69,10 +69,14 @@ class Robot(object): action_list = [ Action(name='MakeCoffee()', pre={'At(Robot,CoffeeMachine)'}, add={'At(Coffee,Bar)'}, del_set=set(), cost=1), - Action(name='MoveTo(Table)', pre={'At(Robot,Bar)'}, + Action(name='MoveTo(Table)', pre={''}, add={'At(Robot,Table)'}, del_set=set(), cost=1), + Action(name='MoveTo(WaterTable)', pre={''}, + add={'At(Robot,WaterTable)'}, del_set=set(), cost=1), Action(name='ExploreEnv()', pre={'At(Robot,Bar)'}, add={'EnvExplored()'}, del_set=set(), cost=1), + Action(name='PourWater()', pre={'At(Robot,WaterTable)'}, + add={'At(Water,WaterTable)'}, del_set=set(), cost=1), ] return action_list diff --git a/robowaiter/scene/tasks/VLM.py b/robowaiter/scene/tasks/VLM.py index 89fd7fc..9b4c5cb 100644 --- a/robowaiter/scene/tasks/VLM.py +++ b/robowaiter/scene/tasks/VLM.py @@ -11,7 +11,8 @@ class SceneVLM(Scene): super().__init__(robot) # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) self.event_list = [ - (5, self.create_chat_event("测试VLM：做一杯咖啡")), + # (5, self.create_chat_event("测试VLM：做一杯咖啡")), + (5, self.create_chat_event("测试VLM：倒一杯水")), ] def _reset(self): diff --git a/sub_task.ptml b/sub_task.ptml index adcea35..f534388 100644 --- a/sub_task.ptml +++ b/sub_task.ptml @@ -1,7 +1,11 @@ selector{ -cond At(Coffee,Bar) +cond At(Water,WaterTable) sequence{ -cond At(Robot,CoffeeMachine) -act MakeCoffee() +cond At(Robot,WaterTable) +act PourWater() +} +sequence{ +cond +act MoveTo(WaterTable) } }