From da224f745d03903ec104a21a3b8fca23c4926e90 Mon Sep 17 00:00:00 2001 From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:09:53 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BA=86=E9=83=A8?= =?UTF-8?q?=E5=88=86=E5=8A=A8=E4=BD=9C=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/behavior_lib/_base/Act.py | 13 +---- robowaiter/behavior_lib/_base/Behavior.py | 14 +++++- robowaiter/behavior_lib/act/Clean.py | 8 ++-- robowaiter/behavior_lib/act/Make.py | 4 +- robowaiter/behavior_lib/act/MoveTo.py | 30 +++++++++--- robowaiter/behavior_lib/act/PickUp.py | 8 ++-- robowaiter/behavior_lib/act/PutDown.py | 11 +++-- robowaiter/behavior_lib/act/Turn.py | 48 ++++++++++++------- robowaiter/behavior_lib/cond/At.py | 11 +++-- robowaiter/behavior_lib/cond/Holding.py | 25 ++++++++++ robowaiter/behavior_lib/cond/Is.py | 36 ++++++++++++++ robowaiter/behavior_lib/cond/On.py | 9 ++-- robowaiter/llm_client/data/test_questions.txt | 3 +- .../llm_client/data_raw/test_questions.csv | 11 ++++- robowaiter/scene/tasks/VLM.py | 4 +- run_robowaiter.py | 2 +- sub_task.ptml | 7 ++- 17 files changed, 175 insertions(+), 69 deletions(-) create mode 100644 robowaiter/behavior_lib/cond/Holding.py create mode 100644 robowaiter/behavior_lib/cond/Is.py diff --git a/robowaiter/behavior_lib/_base/Act.py b/robowaiter/behavior_lib/_base/Act.py index d2b14fb..c18677a 100644 --- a/robowaiter/behavior_lib/_base/Act.py +++ b/robowaiter/behavior_lib/_base/Act.py @@ -3,18 +3,7 @@ from robowaiter.behavior_lib._base.Behavior import Bahavior class Act(Bahavior): print_name_prefix = "act " type = 'Act' - all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'} - all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk', - 'VacuumCup'} - place_xyz_dic={ - 'Bar': (247.0, 520.0, 100.0), - 'Bar2': (240.0, 40.0, 70.0), - 'WaterTable':(-70.0, 500.0, 107), - 'CoffeeTable':(247.0, 520.0, 100.0), # 位置需要更改！！！ - 'Table1': (247.0, 520.0, 100.0),# 位置需要更改！！！ - 'Table2': (-55.0, 0.0, 107), - 'Table3':(-55.0, 150.0, 107) - } + def __init__(self,*args): super().__init__(*args) self.info = self.get_info(*args) diff --git a/robowaiter/behavior_lib/_base/Behavior.py b/robowaiter/behavior_lib/_base/Behavior.py index b80c634..b62d7b7 100644 --- a/robowaiter/behavior_lib/_base/Behavior.py +++ b/robowaiter/behavior_lib/_base/Behavior.py @@ -13,7 +13,19 @@ class Bahavior(ptree.behaviour.Behaviour): ''' scene = None print_name_prefix = "" - + all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'} + all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk', + 'VacuumCup'} + place_xyz_dic={ + 'Bar': (247.0, 520.0, 100.0), + 'Bar2': (240.0, 40.0, 70.0), + 'WaterTable':(-70.0, 500.0, 107), + 'CoffeeTable':(247.0, 520.0, 100.0), # 位置需要更改！！！ + 'Table1': (247.0, 520.0, 100.0),# 位置需要更改！！！ + 'Table2': (-55.0, 0.0, 107), + 'Table3':(-55.0, 150.0, 107) + } + @classmethod def get_ins_name(cls,*args): name = cls.__name__ diff --git a/robowaiter/behavior_lib/act/Clean.py b/robowaiter/behavior_lib/act/Clean.py index 81c7af1..cb7db73 100644 --- a/robowaiter/behavior_lib/act/Clean.py +++ b/robowaiter/behavior_lib/act/Clean.py @@ -28,13 +28,13 @@ class Clean(Act): info["pre"]= {f'Holding(Nothing)'} if arg == "Table1": info["add"]= {f'Is(Table1,Clean)'} - info["del"] = {f'Is(Table1,Dirty)'} + info["del_set"] = {f'Is(Table1,Dirty)'} elif arg == "Floor": info["add"] = {f'Is(Floor,Clean)'} - info["del"] = {f'Is(Floor,Dirty)'} + info["del_set"] = {f'Is(Floor,Dirty)'} elif arg == "Chairs": info["add"] = {f'Is(Chairs,Clean)'} - info["del"] = {f'Is(Chairs,Dirty)'} + info["del_set"] = {f'Is(Chairs,Dirty)'} return info def _update(self) -> ptree.common.Status: @@ -43,5 +43,5 @@ class Clean(Act): self.scene.op_task_execute(self.op_type) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/act/Make.py b/robowaiter/behavior_lib/act/Make.py index ef97eb9..86dea7f 100644 --- a/robowaiter/behavior_lib/act/Make.py +++ b/robowaiter/behavior_lib/act/Make.py @@ -26,7 +26,7 @@ class Make(Act): def get_info(cls,arg): info = {} info["pre"]= {f'Holding(Nothing)'} - info['del'] = set() + info['del_set'] = set() if arg == "Coffee": info["add"]= {f'On(Coffee,CoffeeTable)'} elif arg == "Water": @@ -41,5 +41,5 @@ class Make(Act): self.scene.op_task_execute(self.op_type) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/act/MoveTo.py b/robowaiter/behavior_lib/act/MoveTo.py index f04ad19..8b24e72 100644 --- a/robowaiter/behavior_lib/act/MoveTo.py +++ b/robowaiter/behavior_lib/act/MoveTo.py @@ -14,23 +14,39 @@ class MoveTo(Act): @classmethod - def get_info(self,arg): + def get_info(cls,arg): info = {} + info['pre'] = set() info["add"] = {f'At(Robot,{arg})'} - info["del"] = {f'At(Robot,{place})' for place in self.valid_args if place != arg} + info["del_set"] = {f'At(Robot,{place})' for place in cls.valid_args if place != arg} return info def _update(self) -> ptree.common.Status: # self.scene.test_move() - navigator = Navigator(scene=self.scene, area_range=[-350, 600, -400, 1450], map=self.scene.state["map"]["2d"]) - goal = self.scene.state['map']['obj_pos'][self.args[0]] - navigator.navigate(goal, animation=False) + # navigator = Navigator(scene=self.scene, area_range=[-350, 600, -400, 1450], map=self.scene.state["map"]["2d"]) + # goal = self.scene.state['map']['obj_pos'][self.args[0]] + # navigator.navigate(goal, animation=False) - self.scene.state['condition_set'].add('At(Robot,Table)') + if self.target_place in Act.place_xyz_dic: + goal = Act.place_xyz_dic[self.target_place] + self.scene.walk_to(goal[0],goal[1]) + else: + # 获取obj_id + for id,obj in enumerate(self.scene.objects): + if obj.name == self.target_place: + obj_id = id + break + + obj_info = self.scene.objects[obj_id] + obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z + self.scene.walk_to(obj_x,obj_y) # goal = self.scene.state['map']['obj_pos'][self.args[0]] - # self.scene.walk_to(goal[0],goal[1]) + # self.scene.walk_to(goal[0],goal[1]) # X, Y, Yaw=None, velocity=200, dis_limit=0 + + self.scene.state["condition_set"] |= (self.info["add"]) + self.scene.state["condition_set"] -= self.info["del_set"] return ptree.common.Status.RUNNING diff --git a/robowaiter/behavior_lib/act/PickUp.py b/robowaiter/behavior_lib/act/PickUp.py index 2e3f0f1..b2a2363 100644 --- a/robowaiter/behavior_lib/act/PickUp.py +++ b/robowaiter/behavior_lib/act/PickUp.py @@ -6,18 +6,18 @@ from robowaiter.behavior_lib._base.Behavior import Status class PickUp(Act): can_be_expanded = True num_args = 1 - + valid_args = Act.all_object def __init__(self, *args): super().__init__(*args) self.target_obj = self.args[0] @classmethod - def get_info(self,arg): + def get_info(cls,arg): info = {} info["pre"] = {f'At(Robot,{arg})','Holding(Nothing)'} info["add"] = {f'Holding({arg})'} - info["del"] = {f'Holding(Nothing)'} + info["del_set"] = {f'Holding(Nothing)'} return info @@ -30,5 +30,5 @@ class PickUp(Act): self.scene.op_task_execute(op_type, obj_id=obj_id) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING diff --git a/robowaiter/behavior_lib/act/PutDown.py b/robowaiter/behavior_lib/act/PutDown.py index 0fe363c..30a6476 100644 --- a/robowaiter/behavior_lib/act/PutDown.py +++ b/robowaiter/behavior_lib/act/PutDown.py @@ -2,10 +2,13 @@ import py_trees as ptree from typing import Any from robowaiter.behavior_lib._base.Act import Act from robowaiter.behavior_lib._base.Behavior import Status +import itertools class PutDown(Act): can_be_expanded = True - num_args = 1 + num_args = 2 + + valid_args = tuple(itertools.product(Act.all_object, Act.all_place)) def __init__(self, *args): super().__init__(*args) @@ -14,11 +17,11 @@ class PutDown(Act): @classmethod - def get_info(self,arg): + def get_info(cls,*arg): info = {} info["pre"] = {f'Holding({arg[0]})',f'At(Robot,{arg[1]})'} info["add"] = {f'Holding(Nothing)',f'At({arg[0]},{arg[1]})'} - info["del"] = {f'Holding(Nothing)'} + info["del_set"] = {f'Holding(Nothing)'} return info @@ -31,5 +34,5 @@ class PutDown(Act): self.scene.op_task_execute(op_type, release_pos=release_pos) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING diff --git a/robowaiter/behavior_lib/act/Turn.py b/robowaiter/behavior_lib/act/Turn.py index ed47bde..73d0551 100644 --- a/robowaiter/behavior_lib/act/Turn.py +++ b/robowaiter/behavior_lib/act/Turn.py @@ -2,12 +2,19 @@ import py_trees as ptree from typing import Any from robowaiter.behavior_lib._base.Act import Act from robowaiter.behavior_lib._base.Behavior import Status +import itertools -class Clean(Act): +class Turn(Act): can_be_expanded = True - num_args = 1 - valid_args = [('AC','ACTemperature','TubeLight','HallLight','Curtain'), - ('Off','On','Up','Down','Clean','Dirty')] + num_args = 2 + valid_args = [('AC','TubeLight','HallLight','Curtain'), + ('On','Off')] + + valid_args = list(itertools.product(valid_args[0], valid_args[1])) + valid_args.extend([('ACTemperature','Up'),('ACTemperature','Down')]) + valid_args = tuple(valid_args) + + def __init__(self, *args): super().__init__(*args) @@ -39,19 +46,26 @@ class Clean(Act): self.op_type = 12 @classmethod - def get_info(cls,arg): + def get_info(cls,*arg): info = {} - # 明天写 - # info["pre"]= {f'Holding(Nothing)'} - # if arg == "Table1": - # info["add"]= {f'Is(Table1,Clean)'} - # info["del"] = {f'Is(Table1,Dirty)'} - # elif arg == "Floor": - # info["add"] = {f'Is(Floor,Clean)'} - # info["del"] = {f'Is(Floor,Dirty)'} - # elif arg == "Chairs": - # info["add"] = {f'Is(Chairs,Clean)'} - # info["del"] = {f'Is(Chairs,Dirty)'} + if arg[0]=="TubeLight" or arg[0]=="HallLight" or arg[0]=="Curtain" or arg[0]=='AC': + if arg[1]=="On": + info["pre"] = {f'Is({arg[0]},Off)'} + info["add"] = {f'Is({arg[0]},On)'} + info["del_set"] = {f'Is({arg[0]},Off)'} + elif arg[1]=="Off": + info["pre"] = {f'Is({arg[0]},On)'} + info["add"] = {f'Is({arg[0]},Off)'} + info["del_set"] = {f'Is({arg[0]},On)'} + elif arg[0]=='ACTemperature': + if arg[1]=="Up": + info["pre"] = {f'Is({arg[0]},Down)'} + info["add"] = {f'Is({arg[0]},Up)'} + info["del_set"] = {f'Is({arg[0]},Down)'} + elif arg[1]=="Donw": + info["pre"] = {f'Is({arg[0]},Up)'} + info["add"] = {f'Is({arg[0]},Down)'} + info["del_set"] = {f'Is({arg[0]},Up)'} return info def _update(self) -> ptree.common.Status: @@ -60,5 +74,5 @@ class Clean(Act): self.scene.op_task_execute(self.op_type) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/cond/At.py b/robowaiter/behavior_lib/cond/At.py index 1c88605..4797557 100644 --- a/robowaiter/behavior_lib/cond/At.py +++ b/robowaiter/behavior_lib/cond/At.py @@ -1,13 +1,15 @@ import py_trees as ptree from typing import Any from robowaiter.behavior_lib._base.Cond import Cond +import itertools class At(Cond): can_be_expanded = True num_params = 2 - valid_params = ''' - Robot, Bar - ''' + + valid_args = list(itertools.product(('Robot','Customer'), tuple(Cond.all_object | Cond.all_place | {'Customer'}))) + valid_args.remove(('Customer','Customer')) + valid_args = tuple(valid_args) def __init__(self,*args): super().__init__(*args) @@ -15,9 +17,8 @@ class At(Cond): def _update(self) -> ptree.common.Status: # if self.scene.status? - arg_str = self.arg_str - if f'At({arg_str})' in self.scene.state["condition_set"]: + if self.name in self.scene.state["condition_set"]: return ptree.common.Status.SUCCESS else: return ptree.common.Status.FAILURE diff --git a/robowaiter/behavior_lib/cond/Holding.py b/robowaiter/behavior_lib/cond/Holding.py new file mode 100644 index 0000000..0e6a8d4 --- /dev/null +++ b/robowaiter/behavior_lib/cond/Holding.py @@ -0,0 +1,25 @@ +import py_trees as ptree +from typing import Any +from robowaiter.behavior_lib._base.Cond import Cond + +class Holding(Cond): + can_be_expanded = True + num_params = 2 + valid_args = [tuple(Cond.all_object|{'Nothing'})] + + def __init__(self,*args): + super().__init__(*args) + + + def _update(self) -> ptree.common.Status: + # if self.scene.status? + + if self.name in self.scene.state["condition_set"]: + return ptree.common.Status.SUCCESS + else: + return ptree.common.Status.FAILURE + + # if self.scene.state['chat_list'] == []: + # return ptree.common.Status.FAILURE + # else: + # return ptree.common.Status.SUCCESS diff --git a/robowaiter/behavior_lib/cond/Is.py b/robowaiter/behavior_lib/cond/Is.py new file mode 100644 index 0000000..1f95a6d --- /dev/null +++ b/robowaiter/behavior_lib/cond/Is.py @@ -0,0 +1,36 @@ +import py_trees as ptree +from typing import Any +from robowaiter.behavior_lib._base.Cond import Cond +import itertools + +class Is(Cond): + can_be_expanded = True + num_params = 2 + valid_params1 = [('AC','TubeLight','HallLight','Curtain'), + ('On','Off')] + valid_params2 = [('Table1','Floor','Chairs'), + ('Clean','Dirty')] + valid_params3 = [('ACTemperature'), + ('Up','Down')] + + valid_args = list(itertools.product(valid_params1[0], valid_params1[1])) + valid_args.extend(list(itertools.product(valid_params2[0], valid_params2[1]))) + valid_args.extend(list(itertools.product(valid_params3[0], valid_params3[1]))) + valid_args = tuple(valid_args) + + def __init__(self,*args): + super().__init__(*args) + + + def _update(self) -> ptree.common.Status: + # if self.scene.status? + + if self.name in self.scene.state["condition_set"]: + return ptree.common.Status.SUCCESS + else: + return ptree.common.Status.FAILURE + + # if self.scene.state['chat_list'] == []: + # return ptree.common.Status.FAILURE + # else: + # return ptree.common.Status.SUCCESS diff --git a/robowaiter/behavior_lib/cond/On.py b/robowaiter/behavior_lib/cond/On.py index e679cfd..ea3ff50 100644 --- a/robowaiter/behavior_lib/cond/On.py +++ b/robowaiter/behavior_lib/cond/On.py @@ -5,9 +5,9 @@ from robowaiter.behavior_lib._base.Cond import Cond class On(Cond): can_be_expanded = True num_params = 2 - valid_params = ''' - Robot, Bar - ''' + valid_params = [tuple(Cond.all_object), + tuple(Cond.all_place)] + def __init__(self,*args): super().__init__(*args) @@ -15,9 +15,8 @@ class On(Cond): def _update(self) -> ptree.common.Status: # if self.scene.status? - arg_str = self.arg_str - if f'At({arg_str})' in self.scene.state["condition_set"]: + if self.name in self.scene.state["condition_set"]: return ptree.common.Status.SUCCESS else: return ptree.common.Status.FAILURE diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt index aa9f62c..42f5756 100644 --- a/robowaiter/llm_client/data/test_questions.txt +++ b/robowaiter/llm_client/data/test_questions.txt @@ -1,2 +1 @@ -{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLN：前往桌子": {"Answer": "测试VLN：前往桌子", "Goal": "{\"At(Robot,Table)\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"At(Water,WaterTable)\"}"}} - +{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM：前往桌子2": {"Answer": " 测试VLM：前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}} diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv index de611cc..fda2291 100644 --- a/robowaiter/llm_client/data_raw/test_questions.csv +++ b/robowaiter/llm_client/data_raw/test_questions.csv @@ -1,5 +1,12 @@ Question,Answer,Goal VLMһ,VLMһ,"{""At(Coffee,Bar)""}" -VLNǰ,VLNǰ,"{""At(Robot,Table)""}" +VLMǰ2, VLMǰ2,"{""At(Robot,Table2)""}" AEM,AEM,"{""EnvExplored()""}" -VLMһˮ,VLMһˮ,"{""At(Water,WaterTable)""}" +VLMһˮ,VLMһˮ,"{""On(Water,WaterTable)""}" +VLMյ,VLMյ,"{""Is(AC,On)""}" +VLMش,VLMش,"{""Is(HallLight,Off)""}" +VLMͲ,VLMͲ,"{""Is(TubeLight,Off)""}" +VLMش,VLMش,"{""Is(Curtain,Off)""}" +VLMϵ,VLMϵ,"{""Is(Floor,Clean)""}" +VLM,VLM,"{""Is(Table1,Clean)""}" +VLMѱŵTable2,VLMѱŵTable2,"{""On(BottledDrink,Table2)""}" diff --git a/robowaiter/scene/tasks/VLM.py b/robowaiter/scene/tasks/VLM.py index d8e646f..a7acf23 100644 --- a/robowaiter/scene/tasks/VLM.py +++ b/robowaiter/scene/tasks/VLM.py @@ -11,11 +11,13 @@ class SceneVLM(Scene): super().__init__(robot) # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) self.event_list = [ - (5, self.create_chat_event("测试VLM：做一杯咖啡")), + # (5, self.create_chat_event("测试VLM：做一杯咖啡")), # (5, self.create_chat_event("测试VLM：倒一杯水")), + (5, self.create_chat_event("测试VLM：开空调")), ] def _reset(self): + self.state["condition_set"] = {'At(Robot,Bar)','Holding(Nothing)','Is(AC,Off)'} pass def _run(self, op_type=7): diff --git a/run_robowaiter.py b/run_robowaiter.py index 8592471..1c11b81 100644 --- a/run_robowaiter.py +++ b/run_robowaiter.py @@ -1,7 +1,7 @@ import os from robowaiter import Robot, task_map -TASK_NAME = 'VLN' +TASK_NAME = 'VLM' # create robot project_path = "./robowaiter" diff --git a/sub_task.ptml b/sub_task.ptml index e0a32df..e2a81fa 100644 --- a/sub_task.ptml +++ b/sub_task.ptml @@ -1,4 +1,7 @@ selector{ -cond At(Robot,Table) -act MoveTo(Table) +cond Is(AC,On) +sequence{ +cond Is(AC,Off) +act Turn(AC,On) +} } From c9e8dc3981419e787c6ec453ae52f564aa57cf3d Mon Sep 17 00:00:00 2001 From: ChenXL97 <908926798@qq.com> Date: Tue, 14 Nov 2023 12:10:23 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E7=8E=B0=E5=9C=A8=E5=8F=AF=E4=BB=A5?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E5=A4=A7=E6=A8=A1=E5=9E=8B=E7=9A=84function?= =?UTF-8?q?=20call?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 1 + robowaiter/behavior_lib/_base/Behavior.py | 5 +- robowaiter/llm_client/openai_api_request.py | 67 ++++++++++++ robowaiter/llm_client/tool_api.py | 86 +++++++++++++++ robowaiter/llm_client/tool_api_request.py | 67 ++++++++++++ robowaiter/llm_client/tool_register.py | 115 ++++++++++++++++++++ 6 files changed, 337 insertions(+), 4 deletions(-) create mode 100644 robowaiter/llm_client/openai_api_request.py create mode 100644 robowaiter/llm_client/tool_api.py create mode 100644 robowaiter/llm_client/tool_api_request.py create mode 100644 robowaiter/llm_client/tool_register.py diff --git a/requirements.txt b/requirements.txt index 3c8b9f6..7659ed1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ pytorch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0 cudatoolkit=11.3 +loguru \ No newline at end of file diff --git a/robowaiter/behavior_lib/_base/Behavior.py b/robowaiter/behavior_lib/_base/Behavior.py index b80c634..36f95aa 100644 --- a/robowaiter/behavior_lib/_base/Behavior.py +++ b/robowaiter/behavior_lib/_base/Behavior.py @@ -24,10 +24,7 @@ class Bahavior(ptree.behaviour.Behaviour): return ins_name def __init__(self,*args): - name = self.__class__.__name__ - if len(args)>0: - name = f'{name}({",".join(list(args))})' - self.name = name + self.name = Bahavior.get_ins_name(*args) #get valid args # self.valid_arg_list = [] # lines = self.valid_params.strip().splitlines() diff --git a/robowaiter/llm_client/openai_api_request.py b/robowaiter/llm_client/openai_api_request.py new file mode 100644 index 0000000..ebaa273 --- /dev/null +++ b/robowaiter/llm_client/openai_api_request.py @@ -0,0 +1,67 @@ +# 使用curl命令测试返回 +# curl -X POST "http://127.0.0.1:8000/v1/chat/completions" \ +# -H "Content-Type: application/json" \ +# -d "{\"model\": \"chatglm3-6b\", \"messages\": [{\"role\": \"system\", \"content\": \"You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.\"}, {\"role\": \"user\", \"content\": \"你好，给我讲一个故事，大概100字\"}], \"stream\": false, \"max_tokens\": 100, \"temperature\": 0.8, \"top_p\": 0.8}" + +# 使用Python代码测返回 +import requests +import json + +import urllib3 +######################################## +# 该文件实现了与大模型的简单通信 +######################################## + +# 忽略https的安全性警告 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址 + +def create_chat_completion(model, messages, use_stream=False): + data = { + "model": model, # 模型名称 + "messages": messages, # 会话历史 + "stream": use_stream, # 是否流式响应 + "max_tokens": 100, # 最多生成字数 + "temperature": 0.8, # 温度 + "top_p": 0.8, # 采样概率 + } + + response = requests.post(f"{base_url}/v1/chat/completions", json=data, stream=use_stream, verify=False) + if response.status_code == 200: + if use_stream: + # 处理流式响应 + for line in response.iter_lines(): + if line: + decoded_line = line.decode('utf-8')[6:] + try: + response_json = json.loads(decoded_line) + content = response_json.get("choices", [{}])[0].get("delta", {}).get("content", "") + print(content) + except: + print("Special Token:", decoded_line) + else: + # 处理非流式响应 + decoded_line = response.json() + print(decoded_line) + content = decoded_line.get("choices", [{}])[0].get("message", "").get("content", "") + print(content) + else: + print("Error:", response.status_code) + return None + + +if __name__ == "__main__": + chat_messages = [ + { + "role": "system", + "content": "You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.", + }, + { + "role": "user", + "content": "你好，给我讲一个故事，大概100字" + } + ] + create_chat_completion("chatglm3-6b", chat_messages, use_stream=False) + + diff --git a/robowaiter/llm_client/tool_api.py b/robowaiter/llm_client/tool_api.py new file mode 100644 index 0000000..2780a14 --- /dev/null +++ b/robowaiter/llm_client/tool_api.py @@ -0,0 +1,86 @@ +import json + +import openai +from colorama import init, Fore +from loguru import logger + +from tool_register import get_tools, dispatch_tool + +init(autoreset=True) + +# 使用Python代码测返回 +import requests +import json + +import urllib3 +######################################## +# 该文件实现了与大模型的简单通信 +######################################## + +# 忽略https的安全性警告 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址 + +def get_response(**kwargs): + data = kwargs + + response = requests.post(f"{base_url}/v1/chat/completions", json=data, stream=data["stream"], verify=False) + decoded_line = response.json() + return decoded_line + +functions = get_tools() + +def run_conversation(query: str, stream=False, functions=None, max_retry=5): + params = dict(model="chatglm3", messages=[{"role": "user", "content": query}], stream=stream) + if functions: + params["functions"] = functions + response = get_response(**params) + + for _ in range(max_retry): + if response["choices"][0]["message"].get("function_call"): + function_call = response["choices"][0]["message"]["function_call"] + logger.info(f"Function Call Response: {function_call}") + function_args = json.loads(function_call["arguments"]) + tool_response = dispatch_tool(function_call["name"], function_args) + logger.info(f"Tool Call Response: {tool_response}") + + params["messages"].append(response["choices"][0]["message"]) + params["messages"].append( + { + "role": "function", + "name": function_call["name"], + "content": tool_response, # 调用函数返回结果 + } + ) + else: + reply = response["choices"][0]["message"]["content"] + logger.info(f"Final Reply: \n{reply}") + return + + response = get_response(**params) + + +if __name__ == "__main__": + + # chat_messages = [ + # { + # "role": "system", + # "content": "You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.", + # }, + # { + # "role": "user", + # "content": "你好，给我讲一个故事，大概100字" + # } + # ] + # create_chat_completion("chatglm3-6b", chat_messages, use_stream=False) + + + + query = "你是谁" + run_conversation(query, stream=False) + + logger.info("\n=========== next conversation ===========") + + query = "帮我查询北京的天气怎么样" + run_conversation(query, functions=functions, stream=False) diff --git a/robowaiter/llm_client/tool_api_request.py b/robowaiter/llm_client/tool_api_request.py new file mode 100644 index 0000000..ebaa273 --- /dev/null +++ b/robowaiter/llm_client/tool_api_request.py @@ -0,0 +1,67 @@ +# 使用curl命令测试返回 +# curl -X POST "http://127.0.0.1:8000/v1/chat/completions" \ +# -H "Content-Type: application/json" \ +# -d "{\"model\": \"chatglm3-6b\", \"messages\": [{\"role\": \"system\", \"content\": \"You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.\"}, {\"role\": \"user\", \"content\": \"你好，给我讲一个故事，大概100字\"}], \"stream\": false, \"max_tokens\": 100, \"temperature\": 0.8, \"top_p\": 0.8}" + +# 使用Python代码测返回 +import requests +import json + +import urllib3 +######################################## +# 该文件实现了与大模型的简单通信 +######################################## + +# 忽略https的安全性警告 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址 + +def create_chat_completion(model, messages, use_stream=False): + data = { + "model": model, # 模型名称 + "messages": messages, # 会话历史 + "stream": use_stream, # 是否流式响应 + "max_tokens": 100, # 最多生成字数 + "temperature": 0.8, # 温度 + "top_p": 0.8, # 采样概率 + } + + response = requests.post(f"{base_url}/v1/chat/completions", json=data, stream=use_stream, verify=False) + if response.status_code == 200: + if use_stream: + # 处理流式响应 + for line in response.iter_lines(): + if line: + decoded_line = line.decode('utf-8')[6:] + try: + response_json = json.loads(decoded_line) + content = response_json.get("choices", [{}])[0].get("delta", {}).get("content", "") + print(content) + except: + print("Special Token:", decoded_line) + else: + # 处理非流式响应 + decoded_line = response.json() + print(decoded_line) + content = decoded_line.get("choices", [{}])[0].get("message", "").get("content", "") + print(content) + else: + print("Error:", response.status_code) + return None + + +if __name__ == "__main__": + chat_messages = [ + { + "role": "system", + "content": "You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.", + }, + { + "role": "user", + "content": "你好，给我讲一个故事，大概100字" + } + ] + create_chat_completion("chatglm3-6b", chat_messages, use_stream=False) + + diff --git a/robowaiter/llm_client/tool_register.py b/robowaiter/llm_client/tool_register.py new file mode 100644 index 0000000..39ae2fc --- /dev/null +++ b/robowaiter/llm_client/tool_register.py @@ -0,0 +1,115 @@ +import inspect +import traceback +from copy import deepcopy +from pprint import pformat +from types import GenericAlias +from typing import get_origin, Annotated + +_TOOL_HOOKS = {} +_TOOL_DESCRIPTIONS = {} + + +def register_tool(func: callable): + tool_name = func.__name__ + tool_description = inspect.getdoc(func).strip() + python_params = inspect.signature(func).parameters + tool_params = [] + for name, param in python_params.items(): + annotation = param.annotation + if annotation is inspect.Parameter.empty: + raise TypeError(f"Parameter `{name}` missing type annotation") + if get_origin(annotation) != Annotated: + raise TypeError(f"Annotation type for `{name}` must be typing.Annotated") + + typ, (description, required) = annotation.__origin__, annotation.__metadata__ + typ: str = str(typ) if isinstance(typ, GenericAlias) else typ.__name__ + if not isinstance(description, str): + raise TypeError(f"Description for `{name}` must be a string") + if not isinstance(required, bool): + raise TypeError(f"Required for `{name}` must be a bool") + + tool_params.append({ + "name": name, + "description": description, + "type": typ, + "required": required + }) + tool_def = { + "name": tool_name, + "description": tool_description, + "params": tool_params + } + + print("[registered tool] " + pformat(tool_def)) + _TOOL_HOOKS[tool_name] = func + _TOOL_DESCRIPTIONS[tool_name] = tool_def + + return func + + +def dispatch_tool(tool_name: str, tool_params: dict) -> str: + if tool_name not in _TOOL_HOOKS: + return f"Tool `{tool_name}` not found. Please use a provided tool." + tool_call = _TOOL_HOOKS[tool_name] + try: + ret = tool_call(**tool_params) + except: + ret = traceback.format_exc() + return str(ret) + + +def get_tools() -> dict: + return deepcopy(_TOOL_DESCRIPTIONS) + + +# Tool Definitions + +@register_tool +def random_number_generator( + seed: Annotated[int, 'The random seed used by the generator', True], + range: Annotated[tuple[int, int], 'The range of the generated numbers', True], +) -> int: + """ + Generates a random number x, s.t. range[0] <= x < range[1] + """ + if not isinstance(seed, int): + raise TypeError("Seed must be an integer") + if not isinstance(range, tuple): + raise TypeError("Range must be a tuple") + if not isinstance(range[0], int) or not isinstance(range[1], int): + raise TypeError("Range must be a tuple of integers") + + import random + return random.Random(seed).randint(*range) + + +@register_tool +def get_weather( + city_name: Annotated[str, 'The name of the city to be queried', True], +) -> str: + """ + Get the current weather for `city_name` + """ + + if not isinstance(city_name, str): + raise TypeError("City name must be a string") + + key_selection = { + "current_condition": ["temp_C", "FeelsLikeC", "humidity", "weatherDesc", "observation_time"], + } + import requests + try: + resp = requests.get(f"https://wttr.in/{city_name}?format=j1") + resp.raise_for_status() + resp = resp.json() + ret = {k: {_v: resp[k][0][_v] for _v in v} for k, v in key_selection.items()} + except: + import traceback + ret = "Error encountered while fetching weather data!\n" + traceback.format_exc() + + return str(ret) + + +if __name__ == "__main__": + print(dispatch_tool("get_weather", {"city_name": "beijing"})) + print(get_tools()) From 01c111db42b03cf13b2bb49c0d7011896a5e21e7 Mon Sep 17 00:00:00 2001 From: Netceor <45135347+Netceor@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:41:02 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8A=B9=E5=B8=83?= =?UTF-8?q?=E6=A1=8C=E7=9A=84=E7=89=A9=E5=93=81=E6=8A=93=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/scene/tasks/VLM.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/robowaiter/scene/tasks/VLM.py b/robowaiter/scene/tasks/VLM.py index a7acf23..fec3af5 100644 --- a/robowaiter/scene/tasks/VLM.py +++ b/robowaiter/scene/tasks/VLM.py @@ -30,11 +30,12 @@ class SceneVLM(Scene): # 16: 抓操作需要传入物品id，17: 放操作需要传入放置位置周围的空地区域(桌边而不是桌上) # if op_type == 16: # self.gen_obj() - # self.op_task_execute(op_type, obj_id=0) + # self.op_task_execute(op_type, obj_id=1) + # op_type = 17 # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 70.0], 水杯桌:[-70.0, 500.0, 107] - # # 桌子1:[-55.0, 0.0, 107],桌子2:[-55.0, 150.0, 107] - # elif op_type == 17: self.op_task_execute(op_type, release_pos=[247.0, 520.0, 100.0])#[-55.0, 150.0, 107] - # else: + # # 桌子1:[-55.0, 0.0, 107],桌子2:[-55.0, 150.0, 107], 抹布桌:[300.0, 935.0, 98.0] + # if op_type == 17: self.op_task_execute(op_type, release_pos=[300.0, 935.0, 98.0]) #[325.0, 860.0, 100] + # if op_type not in [16,17]: # self.move_task_area(op_type) # self.op_task_execute(op_type) pass From e6cd1e337b1c2e71ace5cf64057bb4e9bab6a60e Mon Sep 17 00:00:00 2001 From: Netceor <45135347+Netceor@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:42:03 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8A=B9=E5=B8=83?= =?UTF-8?q?=E6=A1=8C=E7=9A=84=E7=89=A9=E5=93=81=E6=8A=93=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/scene/scene.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py index 00887c8..a1018a5 100644 --- a/robowaiter/scene/scene.py +++ b/robowaiter/scene/scene.py @@ -418,10 +418,12 @@ class Scene: def gen_obj(self,h=100): # 4;冰红(盒) 5;酸奶 7:保温杯 9;冰红(瓶) 13:代语词典 14:cake 61:甜牛奶 - type= 9 #9 scene = stub.Observe(GrabSim_pb2.SceneID(value=self.sceneID)) ginger_loc = [scene.location.X, scene.location.Y, scene.location.Z] - obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z = h, roll=0, pitch=0, yaw=0, type=type)] + obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=9), + # GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9), + GrabSim_pb2.ObjectList.Object(x=300, y=935, z = 88, roll=0, pitch=0, yaw=0, type=7), + ] scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID)) time.sleep(1.0) @@ -434,11 +436,17 @@ class Scene: # Robot obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z walk_v = [obj_x+50, obj_y] + [180, 180, 0] + if obj_y>=820 and obj_y<= 1200 and obj_x>=240 and obj_x<= 500: # 物品位于斜的抹布桌上 ([240,500],[820,1200]) + walk_v = [obj_x+40, obj_y-35, 130, 180, 0] + obj_x += 3 + obj_y += 2.5 + # walk_v = [obj_x,obj_y-30,130, 180, 0] action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v) scene = stub.Do(action) time.sleep(1.0) + # Finger - self.ik_control_joints(2, obj_x-9, obj_y+0.5, obj_z) # -10, 0, 0 + self.ik_control_joints(2, obj_x-9, obj_y, obj_z) # -10, 0, 0 time.sleep(3.0) # Grasp Obj print('------------------grasp_obj----------------------') @@ -461,9 +469,6 @@ class Scene: angle[0] = 15 angle[19] = -15 angle[20] = -30 - for i in range(18,21): - print("name:",scene.joints[i].name,"angle:",scene.joints[i].angle) - # print("angle:",angle) action = GrabSim_pb2.Action(scene=self.sceneID,action=GrabSim_pb2.Action.ActionType.RotateJoints, # 弯腰 values=angle) scene = stub.Do(action) @@ -471,13 +476,20 @@ class Scene: def release_obj(self,release_pos): print("------------------Move to Realese Position----------------------") - walk_v = [release_pos[i] for i in range(2)] - action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v + [180,180,0]) + walk_v = [release_pos[i] for i in range(2)] + [180,180,0] + if release_pos==[300.0, 935.0, 98.0]: + walk_v[2] = 130 + + action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v) scene = stub.Do(action) print("------------------release_obj----------------------") - self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2]) - time.sleep(2.0) - self.robo_stoop_parallel() + if release_pos==[300.0, 935.0, 98.0]: + self.ik_control_joints(2, release_pos[0], release_pos[1], release_pos[2]) + time.sleep(2.0) + else: + self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2]) + time.sleep(2.0) + self.robo_stoop_parallel() action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Release, values=[1]) scene = stub.Do(action) @@ -487,7 +499,7 @@ class Scene: return True # 执行过程：输出"开始(任务名)" -> 按步骤数执行任务 -> Robot输出成功或失败的对话 - def op_task_execute(self,op_type,obj_id=0,yaw=180,release_pos=[240,-140]): + def op_task_execute(self,op_type,obj_id=0,release_pos=[240,-140]): self.control_robot_action(0, 1, "开始"+self.op_dialog[op_type]) # 开始制作咖啡 if op_type in [13,14,15]: # 调整空调:13代表按开关,14升温,15降温 result = self.adjust_kongtiao(op_type) From 71f758e9df09999f96736cc23a6e4aa4b793f7ab Mon Sep 17 00:00:00 2001 From: Netceor <45135347+Netceor@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:55:07 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8A=B9=E5=B8=83?= =?UTF-8?q?=E6=A1=8C=E7=9A=84=E6=8A=93=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/scene/tasks/VLM.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/robowaiter/scene/tasks/VLM.py b/robowaiter/scene/tasks/VLM.py index fec3af5..85052d5 100644 --- a/robowaiter/scene/tasks/VLM.py +++ b/robowaiter/scene/tasks/VLM.py @@ -30,11 +30,10 @@ class SceneVLM(Scene): # 16: 抓操作需要传入物品id，17: 放操作需要传入放置位置周围的空地区域(桌边而不是桌上) # if op_type == 16: # self.gen_obj() - # self.op_task_execute(op_type, obj_id=1) - # op_type = 17 + # self.op_task_execute(op_type, obj_id=0) # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 70.0], 水杯桌:[-70.0, 500.0, 107] - # # 桌子1:[-55.0, 0.0, 107],桌子2:[-55.0, 150.0, 107], 抹布桌:[300.0, 935.0, 98.0] - # if op_type == 17: self.op_task_execute(op_type, release_pos=[300.0, 935.0, 98.0]) #[325.0, 860.0, 100] + # # 桌子1:[-55.0, 0.0, 107],桌子2:[-55.0, 150.0, 107], 抹布桌:[340.0, 900.0, 98.0] + # if op_type == 17: self.op_task_execute(op_type, release_pos=[340.0, 900.0, 99.0]) #[325.0, 860.0, 100] # if op_type not in [16,17]: # self.move_task_area(op_type) # self.op_task_execute(op_type) From 257cc0297cc4d15a16d9b3055ef7a63ba4bcbabc Mon Sep 17 00:00:00 2001 From: Netceor <45135347+Netceor@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:55:46 +0800 Subject: [PATCH 6/7] Update scene.py --- robowaiter/scene/scene.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py index a1018a5..17a0ccd 100644 --- a/robowaiter/scene/scene.py +++ b/robowaiter/scene/scene.py @@ -422,7 +422,7 @@ class Scene: ginger_loc = [scene.location.X, scene.location.Y, scene.location.Z] obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=9), # GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9), - GrabSim_pb2.ObjectList.Object(x=300, y=935, z = 88, roll=0, pitch=0, yaw=0, type=7), + GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=7), ] scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID)) time.sleep(1.0) @@ -477,14 +477,13 @@ class Scene: def release_obj(self,release_pos): print("------------------Move to Realese Position----------------------") walk_v = [release_pos[i] for i in range(2)] + [180,180,0] - if release_pos==[300.0, 935.0, 98.0]: + if release_pos==[340.0, 900.0, 99.0]: walk_v[2] = 130 - action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v) scene = stub.Do(action) print("------------------release_obj----------------------") - if release_pos==[300.0, 935.0, 98.0]: - self.ik_control_joints(2, release_pos[0], release_pos[1], release_pos[2]) + if release_pos==[340.0, 900.0, 99.0]: + self.ik_control_joints(2, 300.0, 935, release_pos[2]) time.sleep(2.0) else: self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2]) From 5f0520d1ebd5ffd5256179c42bd9a26d3cc8554a Mon Sep 17 00:00:00 2001 From: ChenXL97 <908926798@qq.com> Date: Tue, 14 Nov 2023 15:26:02 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E5=B7=A5=E5=85=B7=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/llm_client/tool_api.py | 18 ++-- robowaiter/llm_client/tool_api_request.py | 67 ------------- robowaiter/llm_client/tool_register.py | 109 +++++++++++++++------- 3 files changed, 83 insertions(+), 111 deletions(-) delete mode 100644 robowaiter/llm_client/tool_api_request.py diff --git a/robowaiter/llm_client/tool_api.py b/robowaiter/llm_client/tool_api.py index 2780a14..03a9fb3 100644 --- a/robowaiter/llm_client/tool_api.py +++ b/robowaiter/llm_client/tool_api.py @@ -5,16 +5,14 @@ from colorama import init, Fore from loguru import logger from tool_register import get_tools, dispatch_tool - -init(autoreset=True) - -# 使用Python代码测返回 import requests import json import urllib3 +init(autoreset=True) + ######################################## -# 该文件实现了与大模型的简单通信 +# 该文件实现了与大模型的通信以及工具调用 ######################################## # 忽略https的安全性警告 @@ -77,10 +75,10 @@ if __name__ == "__main__": - query = "你是谁" - run_conversation(query, stream=False) + # query = "你是谁" + # run_conversation(query, stream=False) + # + # logger.info("\n=========== next conversation ===========") - logger.info("\n=========== next conversation ===========") - - query = "帮我查询北京的天气怎么样" + query = "洗手间在哪儿" run_conversation(query, functions=functions, stream=False) diff --git a/robowaiter/llm_client/tool_api_request.py b/robowaiter/llm_client/tool_api_request.py deleted file mode 100644 index ebaa273..0000000 --- a/robowaiter/llm_client/tool_api_request.py +++ /dev/null @@ -1,67 +0,0 @@ -# 使用curl命令测试返回 -# curl -X POST "http://127.0.0.1:8000/v1/chat/completions" \ -# -H "Content-Type: application/json" \ -# -d "{\"model\": \"chatglm3-6b\", \"messages\": [{\"role\": \"system\", \"content\": \"You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.\"}, {\"role\": \"user\", \"content\": \"你好，给我讲一个故事，大概100字\"}], \"stream\": false, \"max_tokens\": 100, \"temperature\": 0.8, \"top_p\": 0.8}" - -# 使用Python代码测返回 -import requests -import json - -import urllib3 -######################################## -# 该文件实现了与大模型的简单通信 -######################################## - -# 忽略https的安全性警告 -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址 - -def create_chat_completion(model, messages, use_stream=False): - data = { - "model": model, # 模型名称 - "messages": messages, # 会话历史 - "stream": use_stream, # 是否流式响应 - "max_tokens": 100, # 最多生成字数 - "temperature": 0.8, # 温度 - "top_p": 0.8, # 采样概率 - } - - response = requests.post(f"{base_url}/v1/chat/completions", json=data, stream=use_stream, verify=False) - if response.status_code == 200: - if use_stream: - # 处理流式响应 - for line in response.iter_lines(): - if line: - decoded_line = line.decode('utf-8')[6:] - try: - response_json = json.loads(decoded_line) - content = response_json.get("choices", [{}])[0].get("delta", {}).get("content", "") - print(content) - except: - print("Special Token:", decoded_line) - else: - # 处理非流式响应 - decoded_line = response.json() - print(decoded_line) - content = decoded_line.get("choices", [{}])[0].get("message", "").get("content", "") - print(content) - else: - print("Error:", response.status_code) - return None - - -if __name__ == "__main__": - chat_messages = [ - { - "role": "system", - "content": "You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.", - }, - { - "role": "user", - "content": "你好，给我讲一个故事，大概100字" - } - ] - create_chat_completion("chatglm3-6b", chat_messages, use_stream=False) - - diff --git a/robowaiter/llm_client/tool_register.py b/robowaiter/llm_client/tool_register.py index 39ae2fc..ac04c32 100644 --- a/robowaiter/llm_client/tool_register.py +++ b/robowaiter/llm_client/tool_register.py @@ -64,50 +64,91 @@ def get_tools() -> dict: # Tool Definitions -@register_tool -def random_number_generator( - seed: Annotated[int, 'The random seed used by the generator', True], - range: Annotated[tuple[int, int], 'The range of the generated numbers', True], -) -> int: - """ - Generates a random number x, s.t. range[0] <= x < range[1] - """ - if not isinstance(seed, int): - raise TypeError("Seed must be an integer") - if not isinstance(range, tuple): - raise TypeError("Range must be a tuple") - if not isinstance(range[0], int) or not isinstance(range[1], int): - raise TypeError("Range must be a tuple of integers") +# @register_tool +# def random_number_generator( +# seed: Annotated[int, 'The random seed used by the generator', True], +# range: Annotated[tuple[int, int], 'The range of the generated numbers', True], +# ) -> int: +# """ +# Generates a random number x, s.t. range[0] <= x < range[1] +# """ +# if not isinstance(seed, int): +# raise TypeError("Seed must be an integer") +# if not isinstance(range, tuple): +# raise TypeError("Range must be a tuple") +# if not isinstance(range[0], int) or not isinstance(range[1], int): +# raise TypeError("Range must be a tuple of integers") +# +# import random +# return random.Random(seed).randint(*range) - import random - return random.Random(seed).randint(*range) +# @register_tool +# def get_weather( +# city_name: Annotated[str, 'The name of the city to be queried', True], +# ) -> str: +# """ +# Get the current weather for `city_name` +# """ +# +# if not isinstance(city_name, str): +# raise TypeError("City name must be a string") +# +# key_selection = { +# "current_condition": ["temp_C", "FeelsLikeC", "humidity", "weatherDesc", "observation_time"], +# } +# import requests +# try: +# resp = requests.get(f"https://wttr.in/{city_name}?format=j1") +# resp.raise_for_status() +# resp = resp.json() +# ret = {k: {_v: resp[k][0][_v] for _v in v} for k, v in key_selection.items()} +# except: +# import traceback +# ret = "Error encountered while fetching weather data!\n" + traceback.format_exc() +# +# return str(ret) + + +# @register_tool +# def add( +# a: Annotated[int, '需要相加的第1个数', True], +# b: Annotated[int, '需要相加的第2个数', True] +# ) -> int: +# """ +# 获取 `a` + `b` 的值 +# """ +# +# if (not isinstance(a, int)) or (not isinstance(b, int)): +# raise TypeError("相加的数必须为整数") +# +# return int(a+b) @register_tool -def get_weather( - city_name: Annotated[str, 'The name of the city to be queried', True], +def create_sub_task( + goal: Annotated[str, '用于子任务的目标状态集合', True] ) -> str: """ - Get the current weather for `city_name` + 当需要完成具身任务（如做咖啡，拿放物体，扫地，前往某位置）时，调用该函数，根据用户的提示进行意图理解，生成子任务的目标状态集合，以一阶逻辑的形式来表示，例如：前往桌子的目标状态为{At(Robot,Table)}，做咖啡的目标状态为{On(Coffee,Bar)}等 """ - if not isinstance(city_name, str): - raise TypeError("City name must be a string") + return goal - key_selection = { - "current_condition": ["temp_C", "FeelsLikeC", "humidity", "weatherDesc", "observation_time"], - } - import requests - try: - resp = requests.get(f"https://wttr.in/{city_name}?format=j1") - resp.raise_for_status() - resp = resp.json() - ret = {k: {_v: resp[k][0][_v] for _v in v} for k, v in key_selection.items()} - except: - import traceback - ret = "Error encountered while fetching weather data!\n" + traceback.format_exc() +@register_tool +def find_near_object( + object: Annotated[str, '需要判断所在位置的物体', True] +) -> str: + """ + 在场景中找到相邻的物体，并说出 `object` 在输出物体的附近 + """ + near_object = None + if object == "Table": + near_object = "Bar" + if object == "洗手间": + near_object = "大门" + + return near_object - return str(ret) if __name__ == "__main__":