diff --git a/requirements.txt b/requirements.txt index 3c8b9f6..7659ed1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ pytorch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0 cudatoolkit=11.3 +loguru \ No newline at end of file diff --git a/robowaiter/behavior_lib/_base/Act.py b/robowaiter/behavior_lib/_base/Act.py index d2b14fb..c18677a 100644 --- a/robowaiter/behavior_lib/_base/Act.py +++ b/robowaiter/behavior_lib/_base/Act.py @@ -3,18 +3,7 @@ from robowaiter.behavior_lib._base.Behavior import Bahavior class Act(Bahavior): print_name_prefix = "act " type = 'Act' - all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'} - all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk', - 'VacuumCup'} - place_xyz_dic={ - 'Bar': (247.0, 520.0, 100.0), - 'Bar2': (240.0, 40.0, 70.0), - 'WaterTable':(-70.0, 500.0, 107), - 'CoffeeTable':(247.0, 520.0, 100.0), # 位置需要更改！！！ - 'Table1': (247.0, 520.0, 100.0),# 位置需要更改！！！ - 'Table2': (-55.0, 0.0, 107), - 'Table3':(-55.0, 150.0, 107) - } + def __init__(self,*args): super().__init__(*args) self.info = self.get_info(*args) diff --git a/robowaiter/behavior_lib/_base/Behavior.py b/robowaiter/behavior_lib/_base/Behavior.py index b80c634..83eb4ab 100644 --- a/robowaiter/behavior_lib/_base/Behavior.py +++ b/robowaiter/behavior_lib/_base/Behavior.py @@ -13,7 +13,19 @@ class Bahavior(ptree.behaviour.Behaviour): ''' scene = None print_name_prefix = "" - + all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'} + all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk', + 'VacuumCup'} + place_xyz_dic={ + 'Bar': (247.0, 520.0, 100.0), + 'Bar2': (240.0, 40.0, 70.0), + 'WaterTable':(-70.0, 500.0, 107), + 'CoffeeTable':(247.0, 520.0, 100.0), # 位置需要更改！！！ + 'Table1': (247.0, 520.0, 100.0),# 位置需要更改！！！ + 'Table2': (-55.0, 0.0, 107), + 'Table3':(-55.0, 150.0, 107) + } + @classmethod def get_ins_name(cls,*args): name = cls.__name__ @@ -24,10 +36,7 @@ class Bahavior(ptree.behaviour.Behaviour): return ins_name def __init__(self,*args): - name = self.__class__.__name__ - if len(args)>0: - name = f'{name}({",".join(list(args))})' - self.name = name + self.name = Bahavior.get_ins_name(*args) #get valid args # self.valid_arg_list = [] # lines = self.valid_params.strip().splitlines() diff --git a/robowaiter/behavior_lib/act/Clean.py b/robowaiter/behavior_lib/act/Clean.py index 81c7af1..cb7db73 100644 --- a/robowaiter/behavior_lib/act/Clean.py +++ b/robowaiter/behavior_lib/act/Clean.py @@ -28,13 +28,13 @@ class Clean(Act): info["pre"]= {f'Holding(Nothing)'} if arg == "Table1": info["add"]= {f'Is(Table1,Clean)'} - info["del"] = {f'Is(Table1,Dirty)'} + info["del_set"] = {f'Is(Table1,Dirty)'} elif arg == "Floor": info["add"] = {f'Is(Floor,Clean)'} - info["del"] = {f'Is(Floor,Dirty)'} + info["del_set"] = {f'Is(Floor,Dirty)'} elif arg == "Chairs": info["add"] = {f'Is(Chairs,Clean)'} - info["del"] = {f'Is(Chairs,Dirty)'} + info["del_set"] = {f'Is(Chairs,Dirty)'} return info def _update(self) -> ptree.common.Status: @@ -43,5 +43,5 @@ class Clean(Act): self.scene.op_task_execute(self.op_type) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/act/Make.py b/robowaiter/behavior_lib/act/Make.py index ef97eb9..86dea7f 100644 --- a/robowaiter/behavior_lib/act/Make.py +++ b/robowaiter/behavior_lib/act/Make.py @@ -26,7 +26,7 @@ class Make(Act): def get_info(cls,arg): info = {} info["pre"]= {f'Holding(Nothing)'} - info['del'] = set() + info['del_set'] = set() if arg == "Coffee": info["add"]= {f'On(Coffee,CoffeeTable)'} elif arg == "Water": @@ -41,5 +41,5 @@ class Make(Act): self.scene.op_task_execute(self.op_type) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/act/MoveTo.py b/robowaiter/behavior_lib/act/MoveTo.py index f04ad19..8b24e72 100644 --- a/robowaiter/behavior_lib/act/MoveTo.py +++ b/robowaiter/behavior_lib/act/MoveTo.py @@ -14,23 +14,39 @@ class MoveTo(Act): @classmethod - def get_info(self,arg): + def get_info(cls,arg): info = {} + info['pre'] = set() info["add"] = {f'At(Robot,{arg})'} - info["del"] = {f'At(Robot,{place})' for place in self.valid_args if place != arg} + info["del_set"] = {f'At(Robot,{place})' for place in cls.valid_args if place != arg} return info def _update(self) -> ptree.common.Status: # self.scene.test_move() - navigator = Navigator(scene=self.scene, area_range=[-350, 600, -400, 1450], map=self.scene.state["map"]["2d"]) - goal = self.scene.state['map']['obj_pos'][self.args[0]] - navigator.navigate(goal, animation=False) + # navigator = Navigator(scene=self.scene, area_range=[-350, 600, -400, 1450], map=self.scene.state["map"]["2d"]) + # goal = self.scene.state['map']['obj_pos'][self.args[0]] + # navigator.navigate(goal, animation=False) - self.scene.state['condition_set'].add('At(Robot,Table)') + if self.target_place in Act.place_xyz_dic: + goal = Act.place_xyz_dic[self.target_place] + self.scene.walk_to(goal[0],goal[1]) + else: + # 获取obj_id + for id,obj in enumerate(self.scene.objects): + if obj.name == self.target_place: + obj_id = id + break + + obj_info = self.scene.objects[obj_id] + obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z + self.scene.walk_to(obj_x,obj_y) # goal = self.scene.state['map']['obj_pos'][self.args[0]] - # self.scene.walk_to(goal[0],goal[1]) + # self.scene.walk_to(goal[0],goal[1]) # X, Y, Yaw=None, velocity=200, dis_limit=0 + + self.scene.state["condition_set"] |= (self.info["add"]) + self.scene.state["condition_set"] -= self.info["del_set"] return ptree.common.Status.RUNNING diff --git a/robowaiter/behavior_lib/act/PickUp.py b/robowaiter/behavior_lib/act/PickUp.py index 2e3f0f1..b2a2363 100644 --- a/robowaiter/behavior_lib/act/PickUp.py +++ b/robowaiter/behavior_lib/act/PickUp.py @@ -6,18 +6,18 @@ from robowaiter.behavior_lib._base.Behavior import Status class PickUp(Act): can_be_expanded = True num_args = 1 - + valid_args = Act.all_object def __init__(self, *args): super().__init__(*args) self.target_obj = self.args[0] @classmethod - def get_info(self,arg): + def get_info(cls,arg): info = {} info["pre"] = {f'At(Robot,{arg})','Holding(Nothing)'} info["add"] = {f'Holding({arg})'} - info["del"] = {f'Holding(Nothing)'} + info["del_set"] = {f'Holding(Nothing)'} return info @@ -30,5 +30,5 @@ class PickUp(Act): self.scene.op_task_execute(op_type, obj_id=obj_id) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING diff --git a/robowaiter/behavior_lib/act/PutDown.py b/robowaiter/behavior_lib/act/PutDown.py index 0fe363c..30a6476 100644 --- a/robowaiter/behavior_lib/act/PutDown.py +++ b/robowaiter/behavior_lib/act/PutDown.py @@ -2,10 +2,13 @@ import py_trees as ptree from typing import Any from robowaiter.behavior_lib._base.Act import Act from robowaiter.behavior_lib._base.Behavior import Status +import itertools class PutDown(Act): can_be_expanded = True - num_args = 1 + num_args = 2 + + valid_args = tuple(itertools.product(Act.all_object, Act.all_place)) def __init__(self, *args): super().__init__(*args) @@ -14,11 +17,11 @@ class PutDown(Act): @classmethod - def get_info(self,arg): + def get_info(cls,*arg): info = {} info["pre"] = {f'Holding({arg[0]})',f'At(Robot,{arg[1]})'} info["add"] = {f'Holding(Nothing)',f'At({arg[0]},{arg[1]})'} - info["del"] = {f'Holding(Nothing)'} + info["del_set"] = {f'Holding(Nothing)'} return info @@ -31,5 +34,5 @@ class PutDown(Act): self.scene.op_task_execute(op_type, release_pos=release_pos) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING diff --git a/robowaiter/behavior_lib/act/Turn.py b/robowaiter/behavior_lib/act/Turn.py index ed47bde..73d0551 100644 --- a/robowaiter/behavior_lib/act/Turn.py +++ b/robowaiter/behavior_lib/act/Turn.py @@ -2,12 +2,19 @@ import py_trees as ptree from typing import Any from robowaiter.behavior_lib._base.Act import Act from robowaiter.behavior_lib._base.Behavior import Status +import itertools -class Clean(Act): +class Turn(Act): can_be_expanded = True - num_args = 1 - valid_args = [('AC','ACTemperature','TubeLight','HallLight','Curtain'), - ('Off','On','Up','Down','Clean','Dirty')] + num_args = 2 + valid_args = [('AC','TubeLight','HallLight','Curtain'), + ('On','Off')] + + valid_args = list(itertools.product(valid_args[0], valid_args[1])) + valid_args.extend([('ACTemperature','Up'),('ACTemperature','Down')]) + valid_args = tuple(valid_args) + + def __init__(self, *args): super().__init__(*args) @@ -39,19 +46,26 @@ class Clean(Act): self.op_type = 12 @classmethod - def get_info(cls,arg): + def get_info(cls,*arg): info = {} - # 明天写 - # info["pre"]= {f'Holding(Nothing)'} - # if arg == "Table1": - # info["add"]= {f'Is(Table1,Clean)'} - # info["del"] = {f'Is(Table1,Dirty)'} - # elif arg == "Floor": - # info["add"] = {f'Is(Floor,Clean)'} - # info["del"] = {f'Is(Floor,Dirty)'} - # elif arg == "Chairs": - # info["add"] = {f'Is(Chairs,Clean)'} - # info["del"] = {f'Is(Chairs,Dirty)'} + if arg[0]=="TubeLight" or arg[0]=="HallLight" or arg[0]=="Curtain" or arg[0]=='AC': + if arg[1]=="On": + info["pre"] = {f'Is({arg[0]},Off)'} + info["add"] = {f'Is({arg[0]},On)'} + info["del_set"] = {f'Is({arg[0]},Off)'} + elif arg[1]=="Off": + info["pre"] = {f'Is({arg[0]},On)'} + info["add"] = {f'Is({arg[0]},Off)'} + info["del_set"] = {f'Is({arg[0]},On)'} + elif arg[0]=='ACTemperature': + if arg[1]=="Up": + info["pre"] = {f'Is({arg[0]},Down)'} + info["add"] = {f'Is({arg[0]},Up)'} + info["del_set"] = {f'Is({arg[0]},Down)'} + elif arg[1]=="Donw": + info["pre"] = {f'Is({arg[0]},Up)'} + info["add"] = {f'Is({arg[0]},Down)'} + info["del_set"] = {f'Is({arg[0]},Up)'} return info def _update(self) -> ptree.common.Status: @@ -60,5 +74,5 @@ class Clean(Act): self.scene.op_task_execute(self.op_type) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/cond/At.py b/robowaiter/behavior_lib/cond/At.py index 1c88605..4797557 100644 --- a/robowaiter/behavior_lib/cond/At.py +++ b/robowaiter/behavior_lib/cond/At.py @@ -1,13 +1,15 @@ import py_trees as ptree from typing import Any from robowaiter.behavior_lib._base.Cond import Cond +import itertools class At(Cond): can_be_expanded = True num_params = 2 - valid_params = ''' - Robot, Bar - ''' + + valid_args = list(itertools.product(('Robot','Customer'), tuple(Cond.all_object | Cond.all_place | {'Customer'}))) + valid_args.remove(('Customer','Customer')) + valid_args = tuple(valid_args) def __init__(self,*args): super().__init__(*args) @@ -15,9 +17,8 @@ class At(Cond): def _update(self) -> ptree.common.Status: # if self.scene.status? - arg_str = self.arg_str - if f'At({arg_str})' in self.scene.state["condition_set"]: + if self.name in self.scene.state["condition_set"]: return ptree.common.Status.SUCCESS else: return ptree.common.Status.FAILURE diff --git a/robowaiter/behavior_lib/cond/Holding.py b/robowaiter/behavior_lib/cond/Holding.py new file mode 100644 index 0000000..0e6a8d4 --- /dev/null +++ b/robowaiter/behavior_lib/cond/Holding.py @@ -0,0 +1,25 @@ +import py_trees as ptree +from typing import Any +from robowaiter.behavior_lib._base.Cond import Cond + +class Holding(Cond): + can_be_expanded = True + num_params = 2 + valid_args = [tuple(Cond.all_object|{'Nothing'})] + + def __init__(self,*args): + super().__init__(*args) + + + def _update(self) -> ptree.common.Status: + # if self.scene.status? + + if self.name in self.scene.state["condition_set"]: + return ptree.common.Status.SUCCESS + else: + return ptree.common.Status.FAILURE + + # if self.scene.state['chat_list'] == []: + # return ptree.common.Status.FAILURE + # else: + # return ptree.common.Status.SUCCESS diff --git a/robowaiter/behavior_lib/cond/Is.py b/robowaiter/behavior_lib/cond/Is.py new file mode 100644 index 0000000..1f95a6d --- /dev/null +++ b/robowaiter/behavior_lib/cond/Is.py @@ -0,0 +1,36 @@ +import py_trees as ptree +from typing import Any +from robowaiter.behavior_lib._base.Cond import Cond +import itertools + +class Is(Cond): + can_be_expanded = True + num_params = 2 + valid_params1 = [('AC','TubeLight','HallLight','Curtain'), + ('On','Off')] + valid_params2 = [('Table1','Floor','Chairs'), + ('Clean','Dirty')] + valid_params3 = [('ACTemperature'), + ('Up','Down')] + + valid_args = list(itertools.product(valid_params1[0], valid_params1[1])) + valid_args.extend(list(itertools.product(valid_params2[0], valid_params2[1]))) + valid_args.extend(list(itertools.product(valid_params3[0], valid_params3[1]))) + valid_args = tuple(valid_args) + + def __init__(self,*args): + super().__init__(*args) + + + def _update(self) -> ptree.common.Status: + # if self.scene.status? + + if self.name in self.scene.state["condition_set"]: + return ptree.common.Status.SUCCESS + else: + return ptree.common.Status.FAILURE + + # if self.scene.state['chat_list'] == []: + # return ptree.common.Status.FAILURE + # else: + # return ptree.common.Status.SUCCESS diff --git a/robowaiter/behavior_lib/cond/On.py b/robowaiter/behavior_lib/cond/On.py index e679cfd..ea3ff50 100644 --- a/robowaiter/behavior_lib/cond/On.py +++ b/robowaiter/behavior_lib/cond/On.py @@ -5,9 +5,9 @@ from robowaiter.behavior_lib._base.Cond import Cond class On(Cond): can_be_expanded = True num_params = 2 - valid_params = ''' - Robot, Bar - ''' + valid_params = [tuple(Cond.all_object), + tuple(Cond.all_place)] + def __init__(self,*args): super().__init__(*args) @@ -15,9 +15,8 @@ class On(Cond): def _update(self) -> ptree.common.Status: # if self.scene.status? - arg_str = self.arg_str - if f'At({arg_str})' in self.scene.state["condition_set"]: + if self.name in self.scene.state["condition_set"]: return ptree.common.Status.SUCCESS else: return ptree.common.Status.FAILURE diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt index aa9f62c..42f5756 100644 --- a/robowaiter/llm_client/data/test_questions.txt +++ b/robowaiter/llm_client/data/test_questions.txt @@ -1,2 +1 @@ -{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLN：前往桌子": {"Answer": "测试VLN：前往桌子", "Goal": "{\"At(Robot,Table)\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"At(Water,WaterTable)\"}"}} - +{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM：前往桌子2": {"Answer": " 测试VLM：前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}} diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv index de611cc..fda2291 100644 --- a/robowaiter/llm_client/data_raw/test_questions.csv +++ b/robowaiter/llm_client/data_raw/test_questions.csv @@ -1,5 +1,12 @@ Question,Answer,Goal VLMһ,VLMһ,"{""At(Coffee,Bar)""}" -VLNǰ,VLNǰ,"{""At(Robot,Table)""}" +VLMǰ2, VLMǰ2,"{""At(Robot,Table2)""}" AEM,AEM,"{""EnvExplored()""}" -VLMһˮ,VLMһˮ,"{""At(Water,WaterTable)""}" +VLMһˮ,VLMһˮ,"{""On(Water,WaterTable)""}" +VLMյ,VLMյ,"{""Is(AC,On)""}" +VLMش,VLMش,"{""Is(HallLight,Off)""}" +VLMͲ,VLMͲ,"{""Is(TubeLight,Off)""}" +VLMش,VLMش,"{""Is(Curtain,Off)""}" +VLMϵ,VLMϵ,"{""Is(Floor,Clean)""}" +VLM,VLM,"{""Is(Table1,Clean)""}" +VLMѱŵTable2,VLMѱŵTable2,"{""On(BottledDrink,Table2)""}" diff --git a/robowaiter/llm_client/openai_api_request.py b/robowaiter/llm_client/openai_api_request.py new file mode 100644 index 0000000..ebaa273 --- /dev/null +++ b/robowaiter/llm_client/openai_api_request.py @@ -0,0 +1,67 @@ +# 使用curl命令测试返回 +# curl -X POST "http://127.0.0.1:8000/v1/chat/completions" \ +# -H "Content-Type: application/json" \ +# -d "{\"model\": \"chatglm3-6b\", \"messages\": [{\"role\": \"system\", \"content\": \"You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.\"}, {\"role\": \"user\", \"content\": \"你好，给我讲一个故事，大概100字\"}], \"stream\": false, \"max_tokens\": 100, \"temperature\": 0.8, \"top_p\": 0.8}" + +# 使用Python代码测返回 +import requests +import json + +import urllib3 +######################################## +# 该文件实现了与大模型的简单通信 +######################################## + +# 忽略https的安全性警告 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址 + +def create_chat_completion(model, messages, use_stream=False): + data = { + "model": model, # 模型名称 + "messages": messages, # 会话历史 + "stream": use_stream, # 是否流式响应 + "max_tokens": 100, # 最多生成字数 + "temperature": 0.8, # 温度 + "top_p": 0.8, # 采样概率 + } + + response = requests.post(f"{base_url}/v1/chat/completions", json=data, stream=use_stream, verify=False) + if response.status_code == 200: + if use_stream: + # 处理流式响应 + for line in response.iter_lines(): + if line: + decoded_line = line.decode('utf-8')[6:] + try: + response_json = json.loads(decoded_line) + content = response_json.get("choices", [{}])[0].get("delta", {}).get("content", "") + print(content) + except: + print("Special Token:", decoded_line) + else: + # 处理非流式响应 + decoded_line = response.json() + print(decoded_line) + content = decoded_line.get("choices", [{}])[0].get("message", "").get("content", "") + print(content) + else: + print("Error:", response.status_code) + return None + + +if __name__ == "__main__": + chat_messages = [ + { + "role": "system", + "content": "You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.", + }, + { + "role": "user", + "content": "你好，给我讲一个故事，大概100字" + } + ] + create_chat_completion("chatglm3-6b", chat_messages, use_stream=False) + + diff --git a/robowaiter/llm_client/tool_api.py b/robowaiter/llm_client/tool_api.py new file mode 100644 index 0000000..03a9fb3 --- /dev/null +++ b/robowaiter/llm_client/tool_api.py @@ -0,0 +1,84 @@ +import json + +import openai +from colorama import init, Fore +from loguru import logger + +from tool_register import get_tools, dispatch_tool +import requests +import json + +import urllib3 +init(autoreset=True) + +######################################## +# 该文件实现了与大模型的通信以及工具调用 +######################################## + +# 忽略https的安全性警告 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址 + +def get_response(**kwargs): + data = kwargs + + response = requests.post(f"{base_url}/v1/chat/completions", json=data, stream=data["stream"], verify=False) + decoded_line = response.json() + return decoded_line + +functions = get_tools() + +def run_conversation(query: str, stream=False, functions=None, max_retry=5): + params = dict(model="chatglm3", messages=[{"role": "user", "content": query}], stream=stream) + if functions: + params["functions"] = functions + response = get_response(**params) + + for _ in range(max_retry): + if response["choices"][0]["message"].get("function_call"): + function_call = response["choices"][0]["message"]["function_call"] + logger.info(f"Function Call Response: {function_call}") + function_args = json.loads(function_call["arguments"]) + tool_response = dispatch_tool(function_call["name"], function_args) + logger.info(f"Tool Call Response: {tool_response}") + + params["messages"].append(response["choices"][0]["message"]) + params["messages"].append( + { + "role": "function", + "name": function_call["name"], + "content": tool_response, # 调用函数返回结果 + } + ) + else: + reply = response["choices"][0]["message"]["content"] + logger.info(f"Final Reply: \n{reply}") + return + + response = get_response(**params) + + +if __name__ == "__main__": + + # chat_messages = [ + # { + # "role": "system", + # "content": "You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.", + # }, + # { + # "role": "user", + # "content": "你好，给我讲一个故事，大概100字" + # } + # ] + # create_chat_completion("chatglm3-6b", chat_messages, use_stream=False) + + + + # query = "你是谁" + # run_conversation(query, stream=False) + # + # logger.info("\n=========== next conversation ===========") + + query = "洗手间在哪儿" + run_conversation(query, functions=functions, stream=False) diff --git a/robowaiter/llm_client/tool_register.py b/robowaiter/llm_client/tool_register.py new file mode 100644 index 0000000..ac04c32 --- /dev/null +++ b/robowaiter/llm_client/tool_register.py @@ -0,0 +1,156 @@ +import inspect +import traceback +from copy import deepcopy +from pprint import pformat +from types import GenericAlias +from typing import get_origin, Annotated + +_TOOL_HOOKS = {} +_TOOL_DESCRIPTIONS = {} + + +def register_tool(func: callable): + tool_name = func.__name__ + tool_description = inspect.getdoc(func).strip() + python_params = inspect.signature(func).parameters + tool_params = [] + for name, param in python_params.items(): + annotation = param.annotation + if annotation is inspect.Parameter.empty: + raise TypeError(f"Parameter `{name}` missing type annotation") + if get_origin(annotation) != Annotated: + raise TypeError(f"Annotation type for `{name}` must be typing.Annotated") + + typ, (description, required) = annotation.__origin__, annotation.__metadata__ + typ: str = str(typ) if isinstance(typ, GenericAlias) else typ.__name__ + if not isinstance(description, str): + raise TypeError(f"Description for `{name}` must be a string") + if not isinstance(required, bool): + raise TypeError(f"Required for `{name}` must be a bool") + + tool_params.append({ + "name": name, + "description": description, + "type": typ, + "required": required + }) + tool_def = { + "name": tool_name, + "description": tool_description, + "params": tool_params + } + + print("[registered tool] " + pformat(tool_def)) + _TOOL_HOOKS[tool_name] = func + _TOOL_DESCRIPTIONS[tool_name] = tool_def + + return func + + +def dispatch_tool(tool_name: str, tool_params: dict) -> str: + if tool_name not in _TOOL_HOOKS: + return f"Tool `{tool_name}` not found. Please use a provided tool." + tool_call = _TOOL_HOOKS[tool_name] + try: + ret = tool_call(**tool_params) + except: + ret = traceback.format_exc() + return str(ret) + + +def get_tools() -> dict: + return deepcopy(_TOOL_DESCRIPTIONS) + + +# Tool Definitions + +# @register_tool +# def random_number_generator( +# seed: Annotated[int, 'The random seed used by the generator', True], +# range: Annotated[tuple[int, int], 'The range of the generated numbers', True], +# ) -> int: +# """ +# Generates a random number x, s.t. range[0] <= x < range[1] +# """ +# if not isinstance(seed, int): +# raise TypeError("Seed must be an integer") +# if not isinstance(range, tuple): +# raise TypeError("Range must be a tuple") +# if not isinstance(range[0], int) or not isinstance(range[1], int): +# raise TypeError("Range must be a tuple of integers") +# +# import random +# return random.Random(seed).randint(*range) + + +# @register_tool +# def get_weather( +# city_name: Annotated[str, 'The name of the city to be queried', True], +# ) -> str: +# """ +# Get the current weather for `city_name` +# """ +# +# if not isinstance(city_name, str): +# raise TypeError("City name must be a string") +# +# key_selection = { +# "current_condition": ["temp_C", "FeelsLikeC", "humidity", "weatherDesc", "observation_time"], +# } +# import requests +# try: +# resp = requests.get(f"https://wttr.in/{city_name}?format=j1") +# resp.raise_for_status() +# resp = resp.json() +# ret = {k: {_v: resp[k][0][_v] for _v in v} for k, v in key_selection.items()} +# except: +# import traceback +# ret = "Error encountered while fetching weather data!\n" + traceback.format_exc() +# +# return str(ret) + + +# @register_tool +# def add( +# a: Annotated[int, '需要相加的第1个数', True], +# b: Annotated[int, '需要相加的第2个数', True] +# ) -> int: +# """ +# 获取 `a` + `b` 的值 +# """ +# +# if (not isinstance(a, int)) or (not isinstance(b, int)): +# raise TypeError("相加的数必须为整数") +# +# return int(a+b) + +@register_tool +def create_sub_task( + goal: Annotated[str, '用于子任务的目标状态集合', True] +) -> str: + """ + 当需要完成具身任务（如做咖啡，拿放物体，扫地，前往某位置）时，调用该函数，根据用户的提示进行意图理解，生成子任务的目标状态集合，以一阶逻辑的形式来表示，例如：前往桌子的目标状态为{At(Robot,Table)}，做咖啡的目标状态为{On(Coffee,Bar)}等 + """ + + return goal + +@register_tool +def find_near_object( + object: Annotated[str, '需要判断所在位置的物体', True] +) -> str: + """ + 在场景中找到相邻的物体，并说出 `object` 在输出物体的附近 + """ + near_object = None + if object == "Table": + near_object = "Bar" + if object == "洗手间": + near_object = "大门" + + return near_object + + + +if __name__ == "__main__": + print(dispatch_tool("get_weather", {"city_name": "beijing"})) + print(get_tools()) diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py index 00887c8..17a0ccd 100644 --- a/robowaiter/scene/scene.py +++ b/robowaiter/scene/scene.py @@ -418,10 +418,12 @@ class Scene: def gen_obj(self,h=100): # 4;冰红(盒) 5;酸奶 7:保温杯 9;冰红(瓶) 13:代语词典 14:cake 61:甜牛奶 - type= 9 #9 scene = stub.Observe(GrabSim_pb2.SceneID(value=self.sceneID)) ginger_loc = [scene.location.X, scene.location.Y, scene.location.Z] - obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z = h, roll=0, pitch=0, yaw=0, type=type)] + obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=9), + # GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9), + GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=7), + ] scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID)) time.sleep(1.0) @@ -434,11 +436,17 @@ class Scene: # Robot obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z walk_v = [obj_x+50, obj_y] + [180, 180, 0] + if obj_y>=820 and obj_y<= 1200 and obj_x>=240 and obj_x<= 500: # 物品位于斜的抹布桌上 ([240,500],[820,1200]) + walk_v = [obj_x+40, obj_y-35, 130, 180, 0] + obj_x += 3 + obj_y += 2.5 + # walk_v = [obj_x,obj_y-30,130, 180, 0] action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v) scene = stub.Do(action) time.sleep(1.0) + # Finger - self.ik_control_joints(2, obj_x-9, obj_y+0.5, obj_z) # -10, 0, 0 + self.ik_control_joints(2, obj_x-9, obj_y, obj_z) # -10, 0, 0 time.sleep(3.0) # Grasp Obj print('------------------grasp_obj----------------------') @@ -461,9 +469,6 @@ class Scene: angle[0] = 15 angle[19] = -15 angle[20] = -30 - for i in range(18,21): - print("name:",scene.joints[i].name,"angle:",scene.joints[i].angle) - # print("angle:",angle) action = GrabSim_pb2.Action(scene=self.sceneID,action=GrabSim_pb2.Action.ActionType.RotateJoints, # 弯腰 values=angle) scene = stub.Do(action) @@ -471,13 +476,19 @@ class Scene: def release_obj(self,release_pos): print("------------------Move to Realese Position----------------------") - walk_v = [release_pos[i] for i in range(2)] - action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v + [180,180,0]) + walk_v = [release_pos[i] for i in range(2)] + [180,180,0] + if release_pos==[340.0, 900.0, 99.0]: + walk_v[2] = 130 + action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v) scene = stub.Do(action) print("------------------release_obj----------------------") - self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2]) - time.sleep(2.0) - self.robo_stoop_parallel() + if release_pos==[340.0, 900.0, 99.0]: + self.ik_control_joints(2, 300.0, 935, release_pos[2]) + time.sleep(2.0) + else: + self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2]) + time.sleep(2.0) + self.robo_stoop_parallel() action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Release, values=[1]) scene = stub.Do(action) @@ -487,7 +498,7 @@ class Scene: return True # 执行过程：输出"开始(任务名)" -> 按步骤数执行任务 -> Robot输出成功或失败的对话 - def op_task_execute(self,op_type,obj_id=0,yaw=180,release_pos=[240,-140]): + def op_task_execute(self,op_type,obj_id=0,release_pos=[240,-140]): self.control_robot_action(0, 1, "开始"+self.op_dialog[op_type]) # 开始制作咖啡 if op_type in [13,14,15]: # 调整空调:13代表按开关,14升温,15降温 result = self.adjust_kongtiao(op_type) diff --git a/robowaiter/scene/tasks/VLM.py b/robowaiter/scene/tasks/VLM.py index d8e646f..85052d5 100644 --- a/robowaiter/scene/tasks/VLM.py +++ b/robowaiter/scene/tasks/VLM.py @@ -11,11 +11,13 @@ class SceneVLM(Scene): super().__init__(robot) # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) self.event_list = [ - (5, self.create_chat_event("测试VLM：做一杯咖啡")), + # (5, self.create_chat_event("测试VLM：做一杯咖啡")), # (5, self.create_chat_event("测试VLM：倒一杯水")), + (5, self.create_chat_event("测试VLM：开空调")), ] def _reset(self): + self.state["condition_set"] = {'At(Robot,Bar)','Holding(Nothing)','Is(AC,Off)'} pass def _run(self, op_type=7): @@ -30,9 +32,9 @@ class SceneVLM(Scene): # self.gen_obj() # self.op_task_execute(op_type, obj_id=0) # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 70.0], 水杯桌:[-70.0, 500.0, 107] - # # 桌子1:[-55.0, 0.0, 107],桌子2:[-55.0, 150.0, 107] - # elif op_type == 17: self.op_task_execute(op_type, release_pos=[247.0, 520.0, 100.0])#[-55.0, 150.0, 107] - # else: + # # 桌子1:[-55.0, 0.0, 107],桌子2:[-55.0, 150.0, 107], 抹布桌:[340.0, 900.0, 98.0] + # if op_type == 17: self.op_task_execute(op_type, release_pos=[340.0, 900.0, 99.0]) #[325.0, 860.0, 100] + # if op_type not in [16,17]: # self.move_task_area(op_type) # self.op_task_execute(op_type) pass diff --git a/run_robowaiter.py b/run_robowaiter.py index 8592471..1c11b81 100644 --- a/run_robowaiter.py +++ b/run_robowaiter.py @@ -1,7 +1,7 @@ import os from robowaiter import Robot, task_map -TASK_NAME = 'VLN' +TASK_NAME = 'VLM' # create robot project_path = "./robowaiter" diff --git a/sub_task.ptml b/sub_task.ptml index e0a32df..e2a81fa 100644 --- a/sub_task.ptml +++ b/sub_task.ptml @@ -1,4 +1,7 @@ selector{ -cond At(Robot,Table) -act MoveTo(Table) +cond Is(AC,On) +sequence{ +cond Is(AC,Off) +act Turn(AC,On) +} }