EmbodiedAI
/
RoboWaiter
mirror of https://github.com/HPCL-EI/RoboWaiter.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:HPCL-EI/RoboWaiter

This commit is contained in:
ChenXL97 2023-11-14 12:10:28 +08:00
parent c9e8dc3981 da224f745d
commit 9fd0d2abfe
18 changed files with 221 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
robowaiter/behavior_lib/_base/Act.py
View File

@ -3,18 +3,7 @@ from robowaiter.behavior_lib._base.Behavior import Bahavior
class Act(Bahavior):
print_name_prefix = "act "
type = 'Act'
all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'}
all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk',
'VacuumCup'}
place_xyz_dic={
'Bar': (247.0, 520.0, 100.0),
'Bar2': (240.0, 40.0, 70.0),
'WaterTable':(-70.0, 500.0, 107),
'CoffeeTable':(247.0, 520.0, 100.0), # 位置需要更改！！！
'Table1': (247.0, 520.0, 100.0),# 位置需要更改！！！
'Table2': (-55.0, 0.0, 107),
'Table3':(-55.0, 150.0, 107)
}
def __init__(self,*args):
super().__init__(*args)
self.info = self.get_info(*args)

14
robowaiter/behavior_lib/_base/Behavior.py
View File

@ -13,7 +13,19 @@ class Bahavior(ptree.behaviour.Behaviour):
'''
scene = None
print_name_prefix = ""
all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'}
all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk',
'VacuumCup'}
place_xyz_dic={
'Bar': (247.0, 520.0, 100.0),
'Bar2': (240.0, 40.0, 70.0),
'WaterTable':(-70.0, 500.0, 107),
'CoffeeTable':(247.0, 520.0, 100.0), # 位置需要更改！！！
'Table1': (247.0, 520.0, 100.0),# 位置需要更改！！！
'Table2': (-55.0, 0.0, 107),
'Table3':(-55.0, 150.0, 107)
}
@classmethod
def get_ins_name(cls,*args):
name = cls.__name__

8
robowaiter/behavior_lib/act/Clean.py
View File

@ -28,13 +28,13 @@ class Clean(Act):
info["pre"]= {f'Holding(Nothing)'}
if arg == "Table1":
info["add"]= {f'Is(Table1,Clean)'}
info["del"] = {f'Is(Table1,Dirty)'}
info["del_set"] = {f'Is(Table1,Dirty)'}
elif arg == "Floor":
info["add"] = {f'Is(Floor,Clean)'}
info["del"] = {f'Is(Floor,Dirty)'}
info["del_set"] = {f'Is(Floor,Dirty)'}
elif arg == "Chairs":
info["add"] = {f'Is(Chairs,Clean)'}
info["del"] = {f'Is(Chairs,Dirty)'}
info["del_set"] = {f'Is(Chairs,Dirty)'}
return info
def _update(self) -> ptree.common.Status:
@ -43,5 +43,5 @@ class Clean(Act):
self.scene.op_task_execute(self.op_type)
self.scene.state["condition_set"].union(self.info["add"])
self.scene.state["condition_set"] -= self.info["del"]
self.scene.state["condition_set"] -= self.info["del_set"]
return Status.RUNNING

4
robowaiter/behavior_lib/act/Make.py
View File

@ -26,7 +26,7 @@ class Make(Act):
def get_info(cls,arg):
info = {}
info["pre"]= {f'Holding(Nothing)'}
info['del'] = set()
info['del_set'] = set()
if arg == "Coffee":
info["add"]= {f'On(Coffee,CoffeeTable)'}
elif arg == "Water":
@ -41,5 +41,5 @@ class Make(Act):
self.scene.op_task_execute(self.op_type)
self.scene.state["condition_set"].union(self.info["add"])
self.scene.state["condition_set"] -= self.info["del"]
self.scene.state["condition_set"] -= self.info["del_set"]
return Status.RUNNING

30
robowaiter/behavior_lib/act/MoveTo.py
View File

@ -14,23 +14,39 @@ class MoveTo(Act):
@classmethod
def get_info(self,arg):
def get_info(cls,arg):
info = {}
info['pre'] = set()
info["add"] = {f'At(Robot,{arg})'}
info["del"] = {f'At(Robot,{place})' for place in self.valid_args if place != arg}
info["del_set"] = {f'At(Robot,{place})' for place in cls.valid_args if place != arg}
return info
def _update(self) -> ptree.common.Status:
# self.scene.test_move()
navigator = Navigator(scene=self.scene, area_range=[-350, 600, -400, 1450], map=self.scene.state["map"]["2d"])
goal = self.scene.state['map']['obj_pos'][self.args[0]]
navigator.navigate(goal, animation=False)
# navigator = Navigator(scene=self.scene, area_range=[-350, 600, -400, 1450], map=self.scene.state["map"]["2d"])
# goal = self.scene.state['map']['obj_pos'][self.args[0]]
# navigator.navigate(goal, animation=False)
self.scene.state['condition_set'].add('At(Robot,Table)')
if self.target_place in Act.place_xyz_dic:
goal = Act.place_xyz_dic[self.target_place]
self.scene.walk_to(goal[0],goal[1])
else:
# 获取obj_id
for id,obj in enumerate(self.scene.objects):
if obj.name == self.target_place:
obj_id = id
break
obj_info = self.scene.objects[obj_id]
obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z
self.scene.walk_to(obj_x,obj_y)
# goal = self.scene.state['map']['obj_pos'][self.args[0]]
# self.scene.walk_to(goal[0],goal[1])
# self.scene.walk_to(goal[0],goal[1]) # X, Y, Yaw=None, velocity=200, dis_limit=0
self.scene.state["condition_set"] |= (self.info["add"])
self.scene.state["condition_set"] -= self.info["del_set"]
return ptree.common.Status.RUNNING

8
robowaiter/behavior_lib/act/PickUp.py
View File

@ -6,18 +6,18 @@ from robowaiter.behavior_lib._base.Behavior import Status
class PickUp(Act):
can_be_expanded = True
num_args = 1
valid_args = Act.all_object
def __init__(self, *args):
super().__init__(*args)
self.target_obj = self.args[0]
@classmethod
def get_info(self,arg):
def get_info(cls,arg):
info = {}
info["pre"] = {f'At(Robot,{arg})','Holding(Nothing)'}
info["add"] = {f'Holding({arg})'}
info["del"] = {f'Holding(Nothing)'}
info["del_set"] = {f'Holding(Nothing)'}
return info
@ -30,5 +30,5 @@ class PickUp(Act):
self.scene.op_task_execute(op_type, obj_id=obj_id)
self.scene.state["condition_set"].union(self.info["add"])
self.scene.state["condition_set"] -= self.info["del"]
self.scene.state["condition_set"] -= self.info["del_set"]
return Status.RUNNING

11
robowaiter/behavior_lib/act/PutDown.py
View File

@ -2,10 +2,13 @@ import py_trees as ptree
from typing import Any
from robowaiter.behavior_lib._base.Act import Act
from robowaiter.behavior_lib._base.Behavior import Status
import itertools
class PutDown(Act):
can_be_expanded = True
num_args = 1
num_args = 2
valid_args = tuple(itertools.product(Act.all_object, Act.all_place))
def __init__(self, *args):
super().__init__(*args)
@ -14,11 +17,11 @@ class PutDown(Act):
@classmethod
def get_info(self,arg):
def get_info(cls,*arg):
info = {}
info["pre"] = {f'Holding({arg[0]})',f'At(Robot,{arg[1]})'}
info["add"] = {f'Holding(Nothing)',f'At({arg[0]},{arg[1]})'}
info["del"] = {f'Holding(Nothing)'}
info["del_set"] = {f'Holding(Nothing)'}
return info
@ -31,5 +34,5 @@ class PutDown(Act):
self.scene.op_task_execute(op_type, release_pos=release_pos)
self.scene.state["condition_set"].union(self.info["add"])
self.scene.state["condition_set"] -= self.info["del"]
self.scene.state["condition_set"] -= self.info["del_set"]
return Status.RUNNING

48
robowaiter/behavior_lib/act/Turn.py
View File

@ -2,12 +2,19 @@ import py_trees as ptree
from typing import Any
from robowaiter.behavior_lib._base.Act import Act
from robowaiter.behavior_lib._base.Behavior import Status
import itertools
class Clean(Act):
class Turn(Act):
can_be_expanded = True
num_args = 1
valid_args = [('AC','ACTemperature','TubeLight','HallLight','Curtain'),
('Off','On','Up','Down','Clean','Dirty')]
num_args = 2
valid_args = [('AC','TubeLight','HallLight','Curtain'),
('On','Off')]
valid_args = list(itertools.product(valid_args[0], valid_args[1]))
valid_args.extend([('ACTemperature','Up'),('ACTemperature','Down')])
valid_args = tuple(valid_args)
def __init__(self, *args):
super().__init__(*args)
@ -39,19 +46,26 @@ class Clean(Act):
self.op_type = 12
@classmethod
def get_info(cls,arg):
def get_info(cls,*arg):
info = {}
# 明天写
# info["pre"]= {f'Holding(Nothing)'}
# if arg == "Table1":
# info["add"]= {f'Is(Table1,Clean)'}
# info["del"] = {f'Is(Table1,Dirty)'}
# elif arg == "Floor":
# info["add"] = {f'Is(Floor,Clean)'}
# info["del"] = {f'Is(Floor,Dirty)'}
# elif arg == "Chairs":
# info["add"] = {f'Is(Chairs,Clean)'}
# info["del"] = {f'Is(Chairs,Dirty)'}
if arg[0]=="TubeLight" or arg[0]=="HallLight" or arg[0]=="Curtain" or arg[0]=='AC':
if arg[1]=="On":
info["pre"] = {f'Is({arg[0]},Off)'}
info["add"] = {f'Is({arg[0]},On)'}
info["del_set"] = {f'Is({arg[0]},Off)'}
elif arg[1]=="Off":
info["pre"] = {f'Is({arg[0]},On)'}
info["add"] = {f'Is({arg[0]},Off)'}
info["del_set"] = {f'Is({arg[0]},On)'}
elif arg[0]=='ACTemperature':
if arg[1]=="Up":
info["pre"] = {f'Is({arg[0]},Down)'}
info["add"] = {f'Is({arg[0]},Up)'}
info["del_set"] = {f'Is({arg[0]},Down)'}
elif arg[1]=="Donw":
info["pre"] = {f'Is({arg[0]},Up)'}
info["add"] = {f'Is({arg[0]},Down)'}
info["del_set"] = {f'Is({arg[0]},Up)'}
return info
def _update(self) -> ptree.common.Status:
@ -60,5 +74,5 @@ class Clean(Act):
self.scene.op_task_execute(self.op_type)
self.scene.state["condition_set"].union(self.info["add"])
self.scene.state["condition_set"] -= self.info["del"]
self.scene.state["condition_set"] -= self.info["del_set"]
return Status.RUNNING

11
robowaiter/behavior_lib/cond/At.py
View File

@ -1,13 +1,15 @@
import py_trees as ptree
from typing import Any
from robowaiter.behavior_lib._base.Cond import Cond
import itertools
class At(Cond):
can_be_expanded = True
num_params = 2
valid_params = '''
Robot, Bar
'''
valid_args = list(itertools.product(('Robot','Customer'), tuple(Cond.all_object | Cond.all_place | {'Customer'})))
valid_args.remove(('Customer','Customer'))
valid_args = tuple(valid_args)
def __init__(self,*args):
super().__init__(*args)
@ -15,9 +17,8 @@ class At(Cond):
def _update(self) -> ptree.common.Status:
# if self.scene.status?
arg_str = self.arg_str
if f'At({arg_str})' in self.scene.state["condition_set"]:
if self.name in self.scene.state["condition_set"]:
return ptree.common.Status.SUCCESS
else:
return ptree.common.Status.FAILURE

25
robowaiter/behavior_lib/cond/Holding.py Normal file
View File

@ -0,0 +1,25 @@
import py_trees as ptree
from typing import Any
from robowaiter.behavior_lib._base.Cond import Cond
class Holding(Cond):
can_be_expanded = True
num_params = 2
valid_args = [tuple(Cond.all_object|{'Nothing'})]
def __init__(self,*args):
super().__init__(*args)
def _update(self) -> ptree.common.Status:
# if self.scene.status?
if self.name in self.scene.state["condition_set"]:
return ptree.common.Status.SUCCESS
else:
return ptree.common.Status.FAILURE
# if self.scene.state['chat_list'] == []:
# return ptree.common.Status.FAILURE
# else:
# return ptree.common.Status.SUCCESS

36
robowaiter/behavior_lib/cond/Is.py Normal file
View File

@ -0,0 +1,36 @@
import py_trees as ptree
from typing import Any
from robowaiter.behavior_lib._base.Cond import Cond
import itertools
class Is(Cond):
can_be_expanded = True
num_params = 2
valid_params1 = [('AC','TubeLight','HallLight','Curtain'),
('On','Off')]
valid_params2 = [('Table1','Floor','Chairs'),
('Clean','Dirty')]
valid_params3 = [('ACTemperature'),
('Up','Down')]
valid_args = list(itertools.product(valid_params1[0], valid_params1[1]))
valid_args.extend(list(itertools.product(valid_params2[0], valid_params2[1])))
valid_args.extend(list(itertools.product(valid_params3[0], valid_params3[1])))
valid_args = tuple(valid_args)
def __init__(self,*args):
super().__init__(*args)
def _update(self) -> ptree.common.Status:
# if self.scene.status?
if self.name in self.scene.state["condition_set"]:
return ptree.common.Status.SUCCESS
else:
return ptree.common.Status.FAILURE
# if self.scene.state['chat_list'] == []:
# return ptree.common.Status.FAILURE
# else:
# return ptree.common.Status.SUCCESS

9
robowaiter/behavior_lib/cond/On.py
View File

@ -5,9 +5,9 @@ from robowaiter.behavior_lib._base.Cond import Cond
class On(Cond):
can_be_expanded = True
num_params = 2
valid_params = '''
Robot, Bar
'''
valid_params = [tuple(Cond.all_object),
tuple(Cond.all_place)]
def __init__(self,*args):
super().__init__(*args)
@ -15,9 +15,8 @@ class On(Cond):
def _update(self) -> ptree.common.Status:
# if self.scene.status?
arg_str = self.arg_str
if f'At({arg_str})' in self.scene.state["condition_set"]:
if self.name in self.scene.state["condition_set"]:
return ptree.common.Status.SUCCESS
else:
return ptree.common.Status.FAILURE

47
robowaiter/behavior_tree/dataset/goal_states_generation.py
View File

@ -29,6 +29,34 @@ def single_predict_generation(oplist_1, oplist_2, predict_pattern) -> str:
raise RuntimeError('Incorrect predict pattern!')
def enumerate_predict(oplist_1, oplist_2, predict_pattern) -> [int, list]:
count = 0
res = []
match predict_pattern:
case 'at':
pattern = f'At(%s, %s)'
case 'is':
pattern = f'Is(%s, %s)'
case 'hold':
pattern = f'Holding(%s)'
case 'on':
pattern = f'On(%s, %s)'
case _:
raise RuntimeError('Incorrect predict pattern!')
for str_1 in oplist_1:
if oplist_2:
for str_2 in oplist_2:
count += 1
res.append({pattern % (str_1, str_2)})
else:
count += 1
res.append({pattern % str_1})
return count, res
def generate_goal_states(vln_num: int, vlm_num: int, opentask_num: int):
# res stores lists of sets, while each state represent in set.
res = []
@ -63,4 +91,21 @@ def generate_goal_states(vln_num: int, vlm_num: int, opentask_num: int):
return res
generate_goal_states(30, 6, 6)
def enumerate_goal_states():
# goal states for VLN
count_vln, list_vln = enumerate_predict(['Robot'], Place, 'at')
print(f'VLN 任务的目标状态数：{count_vln}')
# goal states for VLM
count_vlm_1, list_vlm_1 = enumerate_predict(['Robot'], Place, 'at')
count_vlm_2, list_vlm_2 = enumerate_predict(Operable, ['0', '1'], 'is')
print(f'VLM 任务的目标状态数：{count_vlm_1 * count_vlm_2}')
# goal states for open-task
count_opentask_1, list_opentask_1 = enumerate_predict(['Robot'], Place, 'at')
count_opentask_2, list_opentask_2 = enumerate_predict(Object, Place, 'on')
print(f'Open-task-1 任务的目标状态数：{count_opentask_1 * count_opentask_2}')
# generate_goal_states(30, 6, 6)
enumerate_goal_states()

3
robowaiter/llm_client/data/test_questions.txt
View File

@ -1,2 +1 @@
{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLN：前往桌子": {"Answer": "测试VLN：前往桌子", "Goal": "{\"At(Robot,Table)\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"At(Water,WaterTable)\"}"}}
{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM：前往桌子2": {"Answer": " 测试VLM：前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}}

11
robowaiter/llm_client/data_raw/test_questions.csv
View File

@ -1,5 +1,12 @@
Question,Answer,Goal
测试VLM：做一杯咖啡,测试VLM：做一杯咖啡,"{""At(Coffee,Bar)""}"
测试VLN：前往桌子,测试VLN：前往桌子,"{""At(Robot,Table)""}"
测试VLM：前往桌子2, 测试VLM：前往桌子2,"{""At(Robot,Table2)""}"
测试AEM,测试AEM,"{""EnvExplored()""}"
测试VLM：倒一杯水,测试VLM：倒一杯水,"{""At(Water,WaterTable)""}"
测试VLM：倒一杯水,测试VLM：倒一杯水,"{""On(Water,WaterTable)""}"
测试VLM：开空调,测试VLM：开空调,"{""Is(AC,On)""}"
测试VLM：关大厅灯,测试VLM：关大厅灯,"{""Is(HallLight,Off)""}"
测试VLM：关筒灯,测试VLM：关筒灯,"{""Is(TubeLight,Off)""}"
测试VLM：关窗帘,测试VLM：关窗,"{""Is(Curtain,Off)""}"
测试VLM：拖地,测试VLM：拖地,"{""Is(Floor,Clean)""}"
测试VLM：擦桌子,测试VLM：擦桌子,"{""Is(Table1,Clean)""}"
测试VLM：把冰红茶放到Table2,测试VLM：把冰红茶放到Table2,"{""On(BottledDrink,Table2)""}"

1 Question Answer Goal
2 测试VLM：做一杯咖啡 测试VLM：做一杯咖啡 {"At(Coffee,Bar)"}
3 测试VLN：前往桌子 测试VLM：前往桌子2 测试VLN：前往桌子 测试VLM：前往桌子2 {"At(Robot,Table)"} {"At(Robot,Table2)"}
4 测试AEM 测试AEM {"EnvExplored()"}
5 测试VLM：倒一杯水 测试VLM：倒一杯水 {"At(Water,WaterTable)"} {"On(Water,WaterTable)"}
6 测试VLM：开空调 测试VLM：开空调 {"Is(AC,On)"}
7 测试VLM：关大厅灯 测试VLM：关大厅灯 {"Is(HallLight,Off)"}
8 测试VLM：关筒灯 测试VLM：关筒灯 {"Is(TubeLight,Off)"}
9 测试VLM：关窗帘 测试VLM：关窗 {"Is(Curtain,Off)"}
10 测试VLM：拖地 测试VLM：拖地 {"Is(Floor,Clean)"}
11 测试VLM：擦桌子 测试VLM：擦桌子 {"Is(Table1,Clean)"}
12 测试VLM：把冰红茶放到Table2 测试VLM：把冰红茶放到Table2 {"On(BottledDrink,Table2)"}

4
robowaiter/scene/tasks/VLM.py
View File

@ -11,11 +11,13 @@ class SceneVLM(Scene):
super().__init__(robot)
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
self.event_list = [
(5, self.create_chat_event("测试VLM：做一杯咖啡")),
# (5, self.create_chat_event("测试VLM：做一杯咖啡")),
# (5, self.create_chat_event("测试VLM：倒一杯水")),
(5, self.create_chat_event("测试VLM：开空调")),
]
def _reset(self):
self.state["condition_set"] = {'At(Robot,Bar)','Holding(Nothing)','Is(AC,Off)'}
pass
def _run(self, op_type=7):

2
run_robowaiter.py
View File

@ -1,7 +1,7 @@
import os
from robowaiter import Robot, task_map
TASK_NAME = 'VLN'
TASK_NAME = 'VLM'
# create robot
project_path = "./robowaiter"

7
sub_task.ptml
View File

@ -1,4 +1,7 @@
selector{
cond At(Robot,Table)
act MoveTo(Table)
cond Is(AC,On)
sequence{
cond Is(AC,Off)
act Turn(AC,On)
}
}