From f30745058c55f9dbe175258c88598083590bcda5 Mon Sep 17 00:00:00 2001 From: wuziji <2193177243@qq.com> Date: Tue, 14 Nov 2023 09:23:59 +0800 Subject: [PATCH 1/2] =?UTF-8?q?New:=20=E5=A2=9E=E5=8A=A0=E6=9E=9A=E4=B8=BE?= =?UTF-8?q?=E6=89=80=E6=9C=89=E7=9B=AE=E6=A0=87=E7=8A=B6=E6=80=81=E7=9A=84?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataset/goal_states_generation.py | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/robowaiter/behavior_tree/dataset/goal_states_generation.py b/robowaiter/behavior_tree/dataset/goal_states_generation.py index 550d86f..3cc2770 100644 --- a/robowaiter/behavior_tree/dataset/goal_states_generation.py +++ b/robowaiter/behavior_tree/dataset/goal_states_generation.py @@ -29,6 +29,34 @@ def single_predict_generation(oplist_1, oplist_2, predict_pattern) -> str: raise RuntimeError('Incorrect predict pattern!') +def enumerate_predict(oplist_1, oplist_2, predict_pattern) -> [int, list]: + count = 0 + res = [] + + match predict_pattern: + case 'at': + pattern = f'At(%s, %s)' + case 'is': + pattern = f'Is(%s, %s)' + case 'hold': + pattern = f'Holding(%s)' + case 'on': + pattern = f'On(%s, %s)' + case _: + raise RuntimeError('Incorrect predict pattern!') + + for str_1 in oplist_1: + if oplist_2: + for str_2 in oplist_2: + count += 1 + res.append({pattern % (str_1, str_2)}) + else: + count += 1 + res.append({pattern % str_1}) + + return count, res + + def generate_goal_states(vln_num: int, vlm_num: int, opentask_num: int): # res stores lists of sets, while each state represent in set. res = [] @@ -63,4 +91,21 @@ def generate_goal_states(vln_num: int, vlm_num: int, opentask_num: int): return res -generate_goal_states(30, 6, 6) +def enumerate_goal_states(): + # goal states for VLN + count_vln, list_vln = enumerate_predict(['Robot'], Place, 'at') + print(f'VLN 任务的目标状态数：{count_vln}') + + # goal states for VLM + count_vlm_1, list_vlm_1 = enumerate_predict(['Robot'], Place, 'at') + count_vlm_2, list_vlm_2 = enumerate_predict(Operable, ['0', '1'], 'is') + print(f'VLM 任务的目标状态数：{count_vlm_1 * count_vlm_2}') + + # goal states for open-task + count_opentask_1, list_opentask_1 = enumerate_predict(['Robot'], Place, 'at') + count_opentask_2, list_opentask_2 = enumerate_predict(Object, Place, 'on') + print(f'Open-task-1 任务的目标状态数：{count_opentask_1 * count_opentask_2}') + + +# generate_goal_states(30, 6, 6) +enumerate_goal_states() From da224f745d03903ec104a21a3b8fca23c4926e90 Mon Sep 17 00:00:00 2001 From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:09:53 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BA=86=E9=83=A8?= =?UTF-8?q?=E5=88=86=E5=8A=A8=E4=BD=9C=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/behavior_lib/_base/Act.py | 13 +---- robowaiter/behavior_lib/_base/Behavior.py | 14 +++++- robowaiter/behavior_lib/act/Clean.py | 8 ++-- robowaiter/behavior_lib/act/Make.py | 4 +- robowaiter/behavior_lib/act/MoveTo.py | 30 +++++++++--- robowaiter/behavior_lib/act/PickUp.py | 8 ++-- robowaiter/behavior_lib/act/PutDown.py | 11 +++-- robowaiter/behavior_lib/act/Turn.py | 48 ++++++++++++------- robowaiter/behavior_lib/cond/At.py | 11 +++-- robowaiter/behavior_lib/cond/Holding.py | 25 ++++++++++ robowaiter/behavior_lib/cond/Is.py | 36 ++++++++++++++ robowaiter/behavior_lib/cond/On.py | 9 ++-- robowaiter/llm_client/data/test_questions.txt | 3 +- .../llm_client/data_raw/test_questions.csv | 11 ++++- robowaiter/scene/tasks/VLM.py | 4 +- run_robowaiter.py | 2 +- sub_task.ptml | 7 ++- 17 files changed, 175 insertions(+), 69 deletions(-) create mode 100644 robowaiter/behavior_lib/cond/Holding.py create mode 100644 robowaiter/behavior_lib/cond/Is.py diff --git a/robowaiter/behavior_lib/_base/Act.py b/robowaiter/behavior_lib/_base/Act.py index d2b14fb..c18677a 100644 --- a/robowaiter/behavior_lib/_base/Act.py +++ b/robowaiter/behavior_lib/_base/Act.py @@ -3,18 +3,7 @@ from robowaiter.behavior_lib._base.Behavior import Bahavior class Act(Bahavior): print_name_prefix = "act " type = 'Act' - all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'} - all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk', - 'VacuumCup'} - place_xyz_dic={ - 'Bar': (247.0, 520.0, 100.0), - 'Bar2': (240.0, 40.0, 70.0), - 'WaterTable':(-70.0, 500.0, 107), - 'CoffeeTable':(247.0, 520.0, 100.0), # 位置需要更改！！！ - 'Table1': (247.0, 520.0, 100.0),# 位置需要更改！！！ - 'Table2': (-55.0, 0.0, 107), - 'Table3':(-55.0, 150.0, 107) - } + def __init__(self,*args): super().__init__(*args) self.info = self.get_info(*args) diff --git a/robowaiter/behavior_lib/_base/Behavior.py b/robowaiter/behavior_lib/_base/Behavior.py index b80c634..b62d7b7 100644 --- a/robowaiter/behavior_lib/_base/Behavior.py +++ b/robowaiter/behavior_lib/_base/Behavior.py @@ -13,7 +13,19 @@ class Bahavior(ptree.behaviour.Behaviour): ''' scene = None print_name_prefix = "" - + all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'} + all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk', + 'VacuumCup'} + place_xyz_dic={ + 'Bar': (247.0, 520.0, 100.0), + 'Bar2': (240.0, 40.0, 70.0), + 'WaterTable':(-70.0, 500.0, 107), + 'CoffeeTable':(247.0, 520.0, 100.0), # 位置需要更改！！！ + 'Table1': (247.0, 520.0, 100.0),# 位置需要更改！！！ + 'Table2': (-55.0, 0.0, 107), + 'Table3':(-55.0, 150.0, 107) + } + @classmethod def get_ins_name(cls,*args): name = cls.__name__ diff --git a/robowaiter/behavior_lib/act/Clean.py b/robowaiter/behavior_lib/act/Clean.py index 81c7af1..cb7db73 100644 --- a/robowaiter/behavior_lib/act/Clean.py +++ b/robowaiter/behavior_lib/act/Clean.py @@ -28,13 +28,13 @@ class Clean(Act): info["pre"]= {f'Holding(Nothing)'} if arg == "Table1": info["add"]= {f'Is(Table1,Clean)'} - info["del"] = {f'Is(Table1,Dirty)'} + info["del_set"] = {f'Is(Table1,Dirty)'} elif arg == "Floor": info["add"] = {f'Is(Floor,Clean)'} - info["del"] = {f'Is(Floor,Dirty)'} + info["del_set"] = {f'Is(Floor,Dirty)'} elif arg == "Chairs": info["add"] = {f'Is(Chairs,Clean)'} - info["del"] = {f'Is(Chairs,Dirty)'} + info["del_set"] = {f'Is(Chairs,Dirty)'} return info def _update(self) -> ptree.common.Status: @@ -43,5 +43,5 @@ class Clean(Act): self.scene.op_task_execute(self.op_type) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/act/Make.py b/robowaiter/behavior_lib/act/Make.py index ef97eb9..86dea7f 100644 --- a/robowaiter/behavior_lib/act/Make.py +++ b/robowaiter/behavior_lib/act/Make.py @@ -26,7 +26,7 @@ class Make(Act): def get_info(cls,arg): info = {} info["pre"]= {f'Holding(Nothing)'} - info['del'] = set() + info['del_set'] = set() if arg == "Coffee": info["add"]= {f'On(Coffee,CoffeeTable)'} elif arg == "Water": @@ -41,5 +41,5 @@ class Make(Act): self.scene.op_task_execute(self.op_type) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/act/MoveTo.py b/robowaiter/behavior_lib/act/MoveTo.py index f04ad19..8b24e72 100644 --- a/robowaiter/behavior_lib/act/MoveTo.py +++ b/robowaiter/behavior_lib/act/MoveTo.py @@ -14,23 +14,39 @@ class MoveTo(Act): @classmethod - def get_info(self,arg): + def get_info(cls,arg): info = {} + info['pre'] = set() info["add"] = {f'At(Robot,{arg})'} - info["del"] = {f'At(Robot,{place})' for place in self.valid_args if place != arg} + info["del_set"] = {f'At(Robot,{place})' for place in cls.valid_args if place != arg} return info def _update(self) -> ptree.common.Status: # self.scene.test_move() - navigator = Navigator(scene=self.scene, area_range=[-350, 600, -400, 1450], map=self.scene.state["map"]["2d"]) - goal = self.scene.state['map']['obj_pos'][self.args[0]] - navigator.navigate(goal, animation=False) + # navigator = Navigator(scene=self.scene, area_range=[-350, 600, -400, 1450], map=self.scene.state["map"]["2d"]) + # goal = self.scene.state['map']['obj_pos'][self.args[0]] + # navigator.navigate(goal, animation=False) - self.scene.state['condition_set'].add('At(Robot,Table)') + if self.target_place in Act.place_xyz_dic: + goal = Act.place_xyz_dic[self.target_place] + self.scene.walk_to(goal[0],goal[1]) + else: + # 获取obj_id + for id,obj in enumerate(self.scene.objects): + if obj.name == self.target_place: + obj_id = id + break + + obj_info = self.scene.objects[obj_id] + obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z + self.scene.walk_to(obj_x,obj_y) # goal = self.scene.state['map']['obj_pos'][self.args[0]] - # self.scene.walk_to(goal[0],goal[1]) + # self.scene.walk_to(goal[0],goal[1]) # X, Y, Yaw=None, velocity=200, dis_limit=0 + + self.scene.state["condition_set"] |= (self.info["add"]) + self.scene.state["condition_set"] -= self.info["del_set"] return ptree.common.Status.RUNNING diff --git a/robowaiter/behavior_lib/act/PickUp.py b/robowaiter/behavior_lib/act/PickUp.py index 2e3f0f1..b2a2363 100644 --- a/robowaiter/behavior_lib/act/PickUp.py +++ b/robowaiter/behavior_lib/act/PickUp.py @@ -6,18 +6,18 @@ from robowaiter.behavior_lib._base.Behavior import Status class PickUp(Act): can_be_expanded = True num_args = 1 - + valid_args = Act.all_object def __init__(self, *args): super().__init__(*args) self.target_obj = self.args[0] @classmethod - def get_info(self,arg): + def get_info(cls,arg): info = {} info["pre"] = {f'At(Robot,{arg})','Holding(Nothing)'} info["add"] = {f'Holding({arg})'} - info["del"] = {f'Holding(Nothing)'} + info["del_set"] = {f'Holding(Nothing)'} return info @@ -30,5 +30,5 @@ class PickUp(Act): self.scene.op_task_execute(op_type, obj_id=obj_id) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING diff --git a/robowaiter/behavior_lib/act/PutDown.py b/robowaiter/behavior_lib/act/PutDown.py index 0fe363c..30a6476 100644 --- a/robowaiter/behavior_lib/act/PutDown.py +++ b/robowaiter/behavior_lib/act/PutDown.py @@ -2,10 +2,13 @@ import py_trees as ptree from typing import Any from robowaiter.behavior_lib._base.Act import Act from robowaiter.behavior_lib._base.Behavior import Status +import itertools class PutDown(Act): can_be_expanded = True - num_args = 1 + num_args = 2 + + valid_args = tuple(itertools.product(Act.all_object, Act.all_place)) def __init__(self, *args): super().__init__(*args) @@ -14,11 +17,11 @@ class PutDown(Act): @classmethod - def get_info(self,arg): + def get_info(cls,*arg): info = {} info["pre"] = {f'Holding({arg[0]})',f'At(Robot,{arg[1]})'} info["add"] = {f'Holding(Nothing)',f'At({arg[0]},{arg[1]})'} - info["del"] = {f'Holding(Nothing)'} + info["del_set"] = {f'Holding(Nothing)'} return info @@ -31,5 +34,5 @@ class PutDown(Act): self.scene.op_task_execute(op_type, release_pos=release_pos) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING diff --git a/robowaiter/behavior_lib/act/Turn.py b/robowaiter/behavior_lib/act/Turn.py index ed47bde..73d0551 100644 --- a/robowaiter/behavior_lib/act/Turn.py +++ b/robowaiter/behavior_lib/act/Turn.py @@ -2,12 +2,19 @@ import py_trees as ptree from typing import Any from robowaiter.behavior_lib._base.Act import Act from robowaiter.behavior_lib._base.Behavior import Status +import itertools -class Clean(Act): +class Turn(Act): can_be_expanded = True - num_args = 1 - valid_args = [('AC','ACTemperature','TubeLight','HallLight','Curtain'), - ('Off','On','Up','Down','Clean','Dirty')] + num_args = 2 + valid_args = [('AC','TubeLight','HallLight','Curtain'), + ('On','Off')] + + valid_args = list(itertools.product(valid_args[0], valid_args[1])) + valid_args.extend([('ACTemperature','Up'),('ACTemperature','Down')]) + valid_args = tuple(valid_args) + + def __init__(self, *args): super().__init__(*args) @@ -39,19 +46,26 @@ class Clean(Act): self.op_type = 12 @classmethod - def get_info(cls,arg): + def get_info(cls,*arg): info = {} - # 明天写 - # info["pre"]= {f'Holding(Nothing)'} - # if arg == "Table1": - # info["add"]= {f'Is(Table1,Clean)'} - # info["del"] = {f'Is(Table1,Dirty)'} - # elif arg == "Floor": - # info["add"] = {f'Is(Floor,Clean)'} - # info["del"] = {f'Is(Floor,Dirty)'} - # elif arg == "Chairs": - # info["add"] = {f'Is(Chairs,Clean)'} - # info["del"] = {f'Is(Chairs,Dirty)'} + if arg[0]=="TubeLight" or arg[0]=="HallLight" or arg[0]=="Curtain" or arg[0]=='AC': + if arg[1]=="On": + info["pre"] = {f'Is({arg[0]},Off)'} + info["add"] = {f'Is({arg[0]},On)'} + info["del_set"] = {f'Is({arg[0]},Off)'} + elif arg[1]=="Off": + info["pre"] = {f'Is({arg[0]},On)'} + info["add"] = {f'Is({arg[0]},Off)'} + info["del_set"] = {f'Is({arg[0]},On)'} + elif arg[0]=='ACTemperature': + if arg[1]=="Up": + info["pre"] = {f'Is({arg[0]},Down)'} + info["add"] = {f'Is({arg[0]},Up)'} + info["del_set"] = {f'Is({arg[0]},Down)'} + elif arg[1]=="Donw": + info["pre"] = {f'Is({arg[0]},Up)'} + info["add"] = {f'Is({arg[0]},Down)'} + info["del_set"] = {f'Is({arg[0]},Up)'} return info def _update(self) -> ptree.common.Status: @@ -60,5 +74,5 @@ class Clean(Act): self.scene.op_task_execute(self.op_type) self.scene.state["condition_set"].union(self.info["add"]) - self.scene.state["condition_set"] -= self.info["del"] + self.scene.state["condition_set"] -= self.info["del_set"] return Status.RUNNING \ No newline at end of file diff --git a/robowaiter/behavior_lib/cond/At.py b/robowaiter/behavior_lib/cond/At.py index 1c88605..4797557 100644 --- a/robowaiter/behavior_lib/cond/At.py +++ b/robowaiter/behavior_lib/cond/At.py @@ -1,13 +1,15 @@ import py_trees as ptree from typing import Any from robowaiter.behavior_lib._base.Cond import Cond +import itertools class At(Cond): can_be_expanded = True num_params = 2 - valid_params = ''' - Robot, Bar - ''' + + valid_args = list(itertools.product(('Robot','Customer'), tuple(Cond.all_object | Cond.all_place | {'Customer'}))) + valid_args.remove(('Customer','Customer')) + valid_args = tuple(valid_args) def __init__(self,*args): super().__init__(*args) @@ -15,9 +17,8 @@ class At(Cond): def _update(self) -> ptree.common.Status: # if self.scene.status? - arg_str = self.arg_str - if f'At({arg_str})' in self.scene.state["condition_set"]: + if self.name in self.scene.state["condition_set"]: return ptree.common.Status.SUCCESS else: return ptree.common.Status.FAILURE diff --git a/robowaiter/behavior_lib/cond/Holding.py b/robowaiter/behavior_lib/cond/Holding.py new file mode 100644 index 0000000..0e6a8d4 --- /dev/null +++ b/robowaiter/behavior_lib/cond/Holding.py @@ -0,0 +1,25 @@ +import py_trees as ptree +from typing import Any +from robowaiter.behavior_lib._base.Cond import Cond + +class Holding(Cond): + can_be_expanded = True + num_params = 2 + valid_args = [tuple(Cond.all_object|{'Nothing'})] + + def __init__(self,*args): + super().__init__(*args) + + + def _update(self) -> ptree.common.Status: + # if self.scene.status? + + if self.name in self.scene.state["condition_set"]: + return ptree.common.Status.SUCCESS + else: + return ptree.common.Status.FAILURE + + # if self.scene.state['chat_list'] == []: + # return ptree.common.Status.FAILURE + # else: + # return ptree.common.Status.SUCCESS diff --git a/robowaiter/behavior_lib/cond/Is.py b/robowaiter/behavior_lib/cond/Is.py new file mode 100644 index 0000000..1f95a6d --- /dev/null +++ b/robowaiter/behavior_lib/cond/Is.py @@ -0,0 +1,36 @@ +import py_trees as ptree +from typing import Any +from robowaiter.behavior_lib._base.Cond import Cond +import itertools + +class Is(Cond): + can_be_expanded = True + num_params = 2 + valid_params1 = [('AC','TubeLight','HallLight','Curtain'), + ('On','Off')] + valid_params2 = [('Table1','Floor','Chairs'), + ('Clean','Dirty')] + valid_params3 = [('ACTemperature'), + ('Up','Down')] + + valid_args = list(itertools.product(valid_params1[0], valid_params1[1])) + valid_args.extend(list(itertools.product(valid_params2[0], valid_params2[1]))) + valid_args.extend(list(itertools.product(valid_params3[0], valid_params3[1]))) + valid_args = tuple(valid_args) + + def __init__(self,*args): + super().__init__(*args) + + + def _update(self) -> ptree.common.Status: + # if self.scene.status? + + if self.name in self.scene.state["condition_set"]: + return ptree.common.Status.SUCCESS + else: + return ptree.common.Status.FAILURE + + # if self.scene.state['chat_list'] == []: + # return ptree.common.Status.FAILURE + # else: + # return ptree.common.Status.SUCCESS diff --git a/robowaiter/behavior_lib/cond/On.py b/robowaiter/behavior_lib/cond/On.py index e679cfd..ea3ff50 100644 --- a/robowaiter/behavior_lib/cond/On.py +++ b/robowaiter/behavior_lib/cond/On.py @@ -5,9 +5,9 @@ from robowaiter.behavior_lib._base.Cond import Cond class On(Cond): can_be_expanded = True num_params = 2 - valid_params = ''' - Robot, Bar - ''' + valid_params = [tuple(Cond.all_object), + tuple(Cond.all_place)] + def __init__(self,*args): super().__init__(*args) @@ -15,9 +15,8 @@ class On(Cond): def _update(self) -> ptree.common.Status: # if self.scene.status? - arg_str = self.arg_str - if f'At({arg_str})' in self.scene.state["condition_set"]: + if self.name in self.scene.state["condition_set"]: return ptree.common.Status.SUCCESS else: return ptree.common.Status.FAILURE diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt index aa9f62c..42f5756 100644 --- a/robowaiter/llm_client/data/test_questions.txt +++ b/robowaiter/llm_client/data/test_questions.txt @@ -1,2 +1 @@ -{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLN：前往桌子": {"Answer": "测试VLN：前往桌子", "Goal": "{\"At(Robot,Table)\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"At(Water,WaterTable)\"}"}} - +{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM：前往桌子2": {"Answer": " 测试VLM：前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}} diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv index de611cc..fda2291 100644 --- a/robowaiter/llm_client/data_raw/test_questions.csv +++ b/robowaiter/llm_client/data_raw/test_questions.csv @@ -1,5 +1,12 @@ Question,Answer,Goal VLMһ,VLMһ,"{""At(Coffee,Bar)""}" -VLNǰ,VLNǰ,"{""At(Robot,Table)""}" +VLMǰ2, VLMǰ2,"{""At(Robot,Table2)""}" AEM,AEM,"{""EnvExplored()""}" -VLMһˮ,VLMһˮ,"{""At(Water,WaterTable)""}" +VLMһˮ,VLMһˮ,"{""On(Water,WaterTable)""}" +VLMյ,VLMյ,"{""Is(AC,On)""}" +VLMش,VLMش,"{""Is(HallLight,Off)""}" +VLMͲ,VLMͲ,"{""Is(TubeLight,Off)""}" +VLMش,VLMش,"{""Is(Curtain,Off)""}" +VLMϵ,VLMϵ,"{""Is(Floor,Clean)""}" +VLM,VLM,"{""Is(Table1,Clean)""}" +VLMѱŵTable2,VLMѱŵTable2,"{""On(BottledDrink,Table2)""}" diff --git a/robowaiter/scene/tasks/VLM.py b/robowaiter/scene/tasks/VLM.py index d8e646f..a7acf23 100644 --- a/robowaiter/scene/tasks/VLM.py +++ b/robowaiter/scene/tasks/VLM.py @@ -11,11 +11,13 @@ class SceneVLM(Scene): super().__init__(robot) # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) self.event_list = [ - (5, self.create_chat_event("测试VLM：做一杯咖啡")), + # (5, self.create_chat_event("测试VLM：做一杯咖啡")), # (5, self.create_chat_event("测试VLM：倒一杯水")), + (5, self.create_chat_event("测试VLM：开空调")), ] def _reset(self): + self.state["condition_set"] = {'At(Robot,Bar)','Holding(Nothing)','Is(AC,Off)'} pass def _run(self, op_type=7): diff --git a/run_robowaiter.py b/run_robowaiter.py index 8592471..1c11b81 100644 --- a/run_robowaiter.py +++ b/run_robowaiter.py @@ -1,7 +1,7 @@ import os from robowaiter import Robot, task_map -TASK_NAME = 'VLN' +TASK_NAME = 'VLM' # create robot project_path = "./robowaiter" diff --git a/sub_task.ptml b/sub_task.ptml index e0a32df..e2a81fa 100644 --- a/sub_task.ptml +++ b/sub_task.ptml @@ -1,4 +1,7 @@ selector{ -cond At(Robot,Table) -act MoveTo(Table) +cond Is(AC,On) +sequence{ +cond Is(AC,Off) +act Turn(AC,On) +} }