EmbodiedAI
/
RoboWaiter
mirror of https://github.com/HPCL-EI/RoboWaiter.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:HPCL-EI/RoboWaiter

This commit is contained in:
ChenXL97 2023-11-14 19:41:38 +08:00
parent 14e2fce6d1 a012fabf6d
commit 3e7f631f3c
12 changed files with 6241 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
robowaiter/behavior_lib/act/PutDown.py
View File

@ -21,7 +21,7 @@ class PutDown(Act):
info = {}
info["pre"] = {f'Holding({arg[0]})',f'At(Robot,{arg[1]})'}
info["add"] = {f'Holding(Nothing)',f'At({arg[0]},{arg[1]})'}
info["del_set"] = {f'Holding(Nothing)'}
info["del_set"] = {f'Holding({arg[0]})'}
return info

18
robowaiter/behavior_lib/act/Turn.py
View File

@ -41,29 +41,33 @@ class Turn(Act):
self.op_type = 10
elif self.target_obj=="Curtain":
if self.op == 'On':
self.op_type = 11
elif self.op == 'Off':
self.op_type = 12
elif self.op == 'Off':
self.op_type = 11
@classmethod
def get_info(cls,*arg):
info = {}
info["pre"] = set()
if arg[0]=="TubeLight" or arg[0]=="HallLight" or arg[0]=="Curtain" or arg[0]=='AC':
if arg[0]!="Curtain":
info["pre"] |= {f'Holding(Nothing)'}
if arg[1]=="On":
info["pre"] = {f'Is({arg[0]},Off)'}
info["pre"] |= {f'Is({arg[0]},Off)'}
info["add"] = {f'Is({arg[0]},On)'}
info["del_set"] = {f'Is({arg[0]},Off)'}
elif arg[1]=="Off":
info["pre"] = {f'Is({arg[0]},On)'}
info["pre"] |= {f'Is({arg[0]},On)'}
info["add"] = {f'Is({arg[0]},Off)'}
info["del_set"] = {f'Is({arg[0]},On)'}
elif arg[0]=='ACTemperature':
info["pre"] = {f'Holding(Nothing)'}
if arg[1]=="Up":
info["pre"] = {f'Is({arg[0]},Down)'}
info["pre"] |= {f'Is({arg[0]},Down)'}
info["add"] = {f'Is({arg[0]},Up)'}
info["del_set"] = {f'Is({arg[0]},Down)'}
elif arg[1]=="Donw":
info["pre"] = {f'Is({arg[0]},Up)'}
elif arg[1]=="Down":
info["pre"] |= {f'Is({arg[0]},Up)'}
info["add"] = {f'Is({arg[0]},Down)'}
info["del_set"] = {f'Is({arg[0]},Up)'}
return info

2
robowaiter/behavior_lib/cond/HasSubTask.py
View File

@ -8,7 +8,7 @@ class HasSubTask(Cond):
def _update(self) -> ptree.common.Status:
# if self.scene.status?
if self.scene.sub_task_seq.children == []:
if not self.scene.sub_task_seq or not self.scene.sub_task_seq.children:
return ptree.common.Status.FAILURE
else:
return ptree.common.Status.SUCCESS

5000
robowaiter/behavior_tree/dataset/goal_states.txt Normal file
View File

File diff suppressed because it is too large Load Diff

144
robowaiter/behavior_tree/dataset/goal_states_generation.py
View File

@ -1,6 +1,10 @@
# the empty string '' represents robot holds nothing
Object = ['Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk',
'VacuumCup', '']
import os
import re
Object = ['Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk', 'VacuumCup', 'Nothing']
Cookable = ['Coffee', 'Water', 'Dessert']
Place = ['Bar', 'WaterTable', 'CoffeeTable', 'Bar2', 'Table1', 'Table2', 'Table3']
@ -35,13 +39,13 @@ def enumerate_predict(oplist_1, oplist_2, predict_pattern) -> [int, list]:
match predict_pattern:
case 'at':
pattern = f'At(%s, %s)'
pattern = f'At(%s,%s)'
case 'is':
pattern = f'Is(%s, %s)'
pattern = f'Is(%s,%s)'
case 'hold':
pattern = f'Holding(%s)'
case 'on':
pattern = f'On(%s, %s)'
pattern = f'On(%s,%s)'
case _:
raise RuntimeError('Incorrect predict pattern!')
@ -49,10 +53,10 @@ def enumerate_predict(oplist_1, oplist_2, predict_pattern) -> [int, list]:
if oplist_2:
for str_2 in oplist_2:
count += 1
res.append({pattern % (str_1, str_2)})
res.append(pattern % (str_1, str_2))
else:
count += 1
res.append({pattern % str_1})
res.append(pattern % str_1)
return count, res
@ -85,27 +89,127 @@ def generate_goal_states(vln_num: int, vlm_num: int, opentask_num: int):
}
)
# print(res)
# print(len(res))
print(res)
print(len(res))
return res
def enumerate_goal_states():
# goal states for VLN
def enumerate_goal_states(total: int):
res = []
point_15 = int(total * .15)
point_10 = int(total * .10)
# goal states for VLN, .15
count_vln, list_vln = enumerate_predict(['Robot'], Place, 'at')
print(f'VLN 任务的目标状态数：{count_vln}')
list_vln = ['{%s}'%i for i in list_vln]
if count_vln < point_15:
list_vln *= point_15 // count_vln
for i in range(0, point_15 - len(list_vln)):
list_vln.append('{%s}' % single_predict_generation(['Robot'], Place, 'at'))
# print(f'VLN 任务的目标状态数：{count_vln}')
res += list_vln
# goal states for VLM
count_vlm_1, list_vlm_1 = enumerate_predict(['Robot'], Place, 'at')
# goal states for VLM-1, 0.15
count_vlm_1, list_vlm_1 = enumerate_predict(Object, Place, 'on')
list_vlm_1 = ['{%s}' % i for i in list_vlm_1]
if count_vlm_1 < point_15:
list_vlm_1 *= point_15 // count_vlm_1
for i in range(0, point_15 - len(list_vlm_1)):
list_vlm_1.append('{%s}' % (single_predict_generation(Object, Place, 'on')))
res += list_vlm_1
# goal states for VLM-2, 0.15
count_vlm_2, list_vlm_2 = enumerate_predict(Operable, ['0', '1'], 'is')
print(f'VLM 任务的目标状态数：{count_vlm_1 * count_vlm_2}')
list_vlm_2 = ['{%s}' % i for i in list_vlm_2]
if count_vlm_2 < point_15:
list_vlm_2 *= point_15 // count_vlm_2
for i in range(0, point_15 - len(list_vlm_2)):
list_vlm_2.append('{%s}' % single_predict_generation(Operable, ['0', '1'], 'is'))
res += list_vlm_2
# goal states for open-task
count_opentask_1, list_opentask_1 = enumerate_predict(['Robot'], Place, 'at')
count_opentask_2, list_opentask_2 = enumerate_predict(Object, Place, 'on')
print(f'Open-task-1 任务的目标状态数：{count_opentask_1 * count_opentask_2}')
# goal states for VLM-3, 0.1
count_vlm_3, list_vlm_3 = enumerate_predict(Object, None, 'hold')
list_vlm_3 = ['{%s}' % i for i in list_vlm_3]
if count_vlm_3 < point_10:
list_vlm_3 *= point_10 // count_vlm_3
for i in range(0, point_10 - len(list_vlm_3)):
list_vlm_3.append('{%s}' % single_predict_generation(Object, None, 'hold'))
res += list_vlm_3
# goal states for OT, 0.15
count_ot, list_ot = enumerate_predict(Cookable, Place, 'on')
list_ot = ['{%s}' % i for i in list_ot]
if count_ot < point_15:
list_ot *= point_15 // count_ot
for i in range(0, point_15 - len(list_ot)):
list_ot.append('{%s}' % single_predict_generation(Cookable, Place, 'on'))
res += list_ot
# goal states for compound-1, 0.1
count_1, list_1 = enumerate_predict(['Robot'], Place, 'at')
count_2, list_2 = enumerate_predict(Object, Place, 'on')
list_tmp = []
for i in list_1:
for j in list_2:
list_tmp.append('{%s,%s}' % (i, j))
if len(list_tmp) < point_10:
list_tmp *= point_10 // len(list_tmp)
list_tmp += list_tmp[0:point_10-len(list_tmp)]
else:
list_tmp = list_tmp[:point_10]
res += list_tmp
# goal states for compound-2, 0.1
count_1, list_1 = enumerate_predict(['Robot'], Place, 'at')
count_2, list_2 = enumerate_predict(Operable, ['0', '1'], 'is')
list_tmp = []
for i in list_1:
for j in list_2:
list_tmp.append('{%s,%s}' % (i, j))
if len(list_tmp) < point_10:
list_tmp *= point_10 // len(list_tmp)
list_tmp += list_tmp[0:point_10 - len(list_tmp)]
else:
list_tmp = list_tmp[:point_10]
res += list_tmp
# goal states for compound-3, 0.1
count_1, list_1 = enumerate_predict(Cookable, Place, 'on')
count_2, list_2 = enumerate_predict(Operable, ['0', '1'], 'is')
list_tmp = []
for i in list_1:
for j in list_2:
list_tmp.append('{%s,%s}' % (i, j))
if len(list_tmp) < point_10:
list_tmp *= point_10 // len(list_tmp)
list_tmp += list_tmp[0:point_10 - len(list_tmp)]
else:
list_tmp = list_tmp[:point_10]
res += list_tmp
# # goal states for VLM-1, 0.15
# count_vlm_1, list_vlm_1 = enumerate_predict(['Robot'], Place, 'at')
# count_vlm_2, list_vlm_2 = enumerate_predict(Operable, ['0', '1'], 'is')
# print(f'VLM 任务的目标状态数：{count_vlm_1 * count_vlm_2}')
#
# # goal states for open-task
# count_opentask_1, list_opentask_1 = enumerate_predict(['Robot'], Place, 'at')
# count_opentask_2, list_opentask_2 = enumerate_predict(Object, Place, 'on')
# print(f'Open-task-1 任务的目标状态数：{count_opentask_1 * count_opentask_2}')
with open(os.path.join('./goal_states.txt'), 'w+') as file:
for i in res:
if 'Is' in i and 'ACTemperature' in i:
i = re.sub('0', 'Up', i)
i = re.sub('1', 'Down', i)
elif 'Is' in i and ('AC' in i or 'HallLight' in i or 'TubeLight' in i or 'Curtain' in i):
i = re.sub('0', 'Off', i)
i = re.sub('1', 'On', i)
file.write(i+'\n')
# generate_goal_states(30, 6, 6)
enumerate_goal_states()
enumerate_goal_states(5000)

2
robowaiter/behavior_tree/obtea/OptimalBTExpansionAlgorithm.py
View File

@ -183,7 +183,7 @@ class OptBTExpAlgorithm:
if isinstance(child, Leaf):
if child.type == 'cond':
self.ptml_string += "cond "
c_set_str = ', '.join(map(str, child.content)) + "\n"
c_set_str = '\n cond '.join(map(str, child.content)) + "\n"
self.ptml_string += c_set_str
elif child.type == 'act':
if '(' not in child.content.name:

2
robowaiter/llm_client/data/test_questions.txt
View File

@ -1 +1 @@
{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM：前往桌子2": {"Answer": " 测试VLM：前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}}
{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM：前往桌子2": {"Answer": " 测试VLM：前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关空调": {"Answer": "测试VLM：关空调", "Goal": "{\"Is(AC,Off)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：开大厅灯": {"Answer": "测试VLM：开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：开筒灯": {"Answer": "测试VLM：开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：开窗帘": {"Answer": "测试VLM：开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：整理椅子": {"Answer": "测试VLM：整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}}

7
robowaiter/llm_client/data_raw/test_questions.csv
View File

@ -4,9 +4,14 @@ Question,Answer,Goal
测试AEM,测试AEM,"{""EnvExplored()""}"
测试VLM：倒一杯水,测试VLM：倒一杯水,"{""On(Water,WaterTable)""}"
测试VLM：开空调,测试VLM：开空调,"{""Is(AC,On)""}"
测试VLM：关空调,测试VLM：关空调,"{""Is(AC,Off)""}"
测试VLM：关大厅灯,测试VLM：关大厅灯,"{""Is(HallLight,Off)""}"
测试VLM：开大厅灯,测试VLM：开大厅灯,"{""Is(HallLight,On)""}"
测试VLM：关筒灯,测试VLM：关筒灯,"{""Is(TubeLight,Off)""}"
测试VLM：关窗帘,测试VLM：关窗,"{""Is(Curtain,Off)""}"
测试VLM：开筒灯,测试VLM：开筒灯,"{""Is(TubeLight,On)""}"
测试VLM：关窗帘,测试VLM：关窗帘,"{""Is(Curtain,Off)""}"
测试VLM：开窗帘,测试VLM：开窗帘,"{""Is(Curtain,On)""}"
测试VLM：拖地,测试VLM：拖地,"{""Is(Floor,Clean)""}"
测试VLM：擦桌子,测试VLM：擦桌子,"{""Is(Table1,Clean)""}"
测试VLM：整理椅子,测试VLM：整理椅子,"{""Is(Chairs,Clean)""}"
测试VLM：把冰红茶放到Table2,测试VLM：把冰红茶放到Table2,"{""On(BottledDrink,Table2)""}"

1 Question Answer Goal
4 测试AEM 测试AEM {"EnvExplored()"}
5 测试VLM：倒一杯水 测试VLM：倒一杯水 {"On(Water,WaterTable)"}
6 测试VLM：开空调 测试VLM：开空调 {"Is(AC,On)"}
7 测试VLM：关空调 测试VLM：关空调 {"Is(AC,Off)"}
8 测试VLM：关大厅灯 测试VLM：关大厅灯 {"Is(HallLight,Off)"}
9 测试VLM：开大厅灯 测试VLM：开大厅灯 {"Is(HallLight,On)"}
10 测试VLM：关筒灯 测试VLM：关筒灯 {"Is(TubeLight,Off)"}
11 测试VLM：关窗帘 测试VLM：开筒灯 测试VLM：关窗 测试VLM：开筒灯 {"Is(Curtain,Off)"} {"Is(TubeLight,On)"}
12 测试VLM：关窗帘 测试VLM：关窗帘 {"Is(Curtain,Off)"}
13 测试VLM：开窗帘 测试VLM：开窗帘 {"Is(Curtain,On)"}
14 测试VLM：拖地 测试VLM：拖地 {"Is(Floor,Clean)"}
15 测试VLM：擦桌子 测试VLM：擦桌子 {"Is(Table1,Clean)"}
16 测试VLM：整理椅子 测试VLM：整理椅子 {"Is(Chairs,Clean)"}
17 测试VLM：把冰红茶放到Table2 测试VLM：把冰红茶放到Table2 {"On(BottledDrink,Table2)"}

9
robowaiter/robot/robot.py
View File

@ -48,7 +48,8 @@ class Robot(object):
print("\n--------------------")
print(f"首次运行行为树扩展算法，收集到{len(self.action_list)}个有效动作:")
for a in self.action_list:
print(a.name)
if "Turn" in a.name:
print(a.name)
print("--------------------\n")
@ -70,7 +71,11 @@ class Robot(object):
del_sub_tree.set_scene(self.scene)
seq.add_child(del_sub_tree)
self.scene.sub_task_seq.add_child(seq)
if self.scene.sub_task_seq:
self.scene.sub_task_seq.add_child(seq)
else:
print('Warning: have none sub task sequence')
self.scene.sub_task_seq = seq
print("当前行为树为：")
print_tree_from_root(self.bt.root)

7
robowaiter/scene/scene.py
View File

@ -416,13 +416,14 @@ class Scene:
self.robo_recover()
return True
def gen_obj(self,h=100):
def gen_obj(self,type=5,h=100):
# 4;冰红(盒) 5;酸奶 7:保温杯 9;冰红(瓶) 13:代语词典 14:cake 61:甜牛奶
type= 5 #9
scene = stub.Observe(GrabSim_pb2.SceneID(value=self.sceneID))
ginger_loc = [scene.location.X, scene.location.Y, scene.location.Z]
obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=9),
obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=5),
# GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9),
GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=7),
GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=9),
]
scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID))
time.sleep(1.0)

22
robowaiter/scene/tasks/VLM.py
View File

@ -11,16 +11,30 @@ class SceneVLM(Scene):
super().__init__(robot)
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
self.event_list = [
# (5, self.create_chat_event("测试VLM：做一杯咖啡")),
(5, self.create_chat_event("测试VLM：做一杯咖啡")),
# (5, self.create_chat_event("测试VLM：倒一杯水")),
(5, self.create_chat_event("测试VLM：开空调")),
# (5, self.create_chat_event("测试VLM：开空调")),
# (5, self.create_chat_event("测试VLM：关空调")),
# (5, self.create_chat_event("测试VLM：开大厅灯")),
# (5, self.create_chat_event("测试VLM：拖地")),
# (5, self.create_chat_event("测试VLM：擦桌子")),
# (5, self.create_chat_event("测试VLM：整理椅子")),
# (5, self.create_chat_event("测试VLM：把冰红茶放到Table2")),
# (5, self.create_chat_event("测试VLM：关大厅灯"))
]
def _reset(self):
self.state["condition_set"] = {'At(Robot,Bar)','Holding(Nothing)','Is(AC,Off)'}
self.state["condition_set"] = {'At(Robot,Bar)','Is(AC,Off)', 'Holding(Nothing)', # 'Holding(Yogurt)', #'Holding(Nothing)',
'Is(HallLight,Off)','Is(TubeLight,On)','Is(Curtain,On)',
'Is(Table1,Dirty)','Is(Floor,Dirty)','Is(Chairs,Dirty)'}
# self.gen_obj(type=5)
# # self.gen_obj(type=9)
# self.op_task_execute(op_type=16, obj_id=0)
# self.move_task_area(op_type=4)
pass
def _run(self, op_type=7):
def _run(self, op_type=10):
# 共17个操作
# "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7
# "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12

1070
sub_task.ptml
View File

File diff suppressed because it is too large Load Diff