EmbodiedAI
/
RoboWaiter
mirror of https://github.com/HPCL-EI/RoboWaiter.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

This commit is contained in:
wuziji 2023-11-14 17:31:23 +08:00
parent 0a3e3a8f3c 69dd918675
commit 7c3ba9edee
8 changed files with 365 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
robowaiter/behavior_lib/act/PutDown.py
View File

@ -21,7 +21,7 @@ class PutDown(Act):
info = {}
info["pre"] = {f'Holding({arg[0]})',f'At(Robot,{arg[1]})'}
info["add"] = {f'Holding(Nothing)',f'At({arg[0]},{arg[1]})'}
info["del_set"] = {f'Holding(Nothing)'}
info["del_set"] = {f'Holding({arg[0]})'}
return info

18
robowaiter/behavior_lib/act/Turn.py
View File

@ -41,29 +41,33 @@ class Turn(Act):
self.op_type = 10
elif self.target_obj=="Curtain":
if self.op == 'On':
self.op_type = 11
elif self.op == 'Off':
self.op_type = 12
elif self.op == 'Off':
self.op_type = 11
@classmethod
def get_info(cls,*arg):
info = {}
info["pre"] = set()
if arg[0]=="TubeLight" or arg[0]=="HallLight" or arg[0]=="Curtain" or arg[0]=='AC':
if arg[0]!="Curtain":
info["pre"] |= {f'Holding(Nothing)'}
if arg[1]=="On":
info["pre"] = {f'Is({arg[0]},Off)'}
info["pre"] |= {f'Is({arg[0]},Off)'}
info["add"] = {f'Is({arg[0]},On)'}
info["del_set"] = {f'Is({arg[0]},Off)'}
elif arg[1]=="Off":
info["pre"] = {f'Is({arg[0]},On)'}
info["pre"] |= {f'Is({arg[0]},On)'}
info["add"] = {f'Is({arg[0]},Off)'}
info["del_set"] = {f'Is({arg[0]},On)'}
elif arg[0]=='ACTemperature':
info["pre"] = {f'Holding(Nothing)'}
if arg[1]=="Up":
info["pre"] = {f'Is({arg[0]},Down)'}
info["pre"] |= {f'Is({arg[0]},Down)'}
info["add"] = {f'Is({arg[0]},Up)'}
info["del_set"] = {f'Is({arg[0]},Down)'}
elif arg[1]=="Donw":
info["pre"] = {f'Is({arg[0]},Up)'}
elif arg[1]=="Down":
info["pre"] |= {f'Is({arg[0]},Up)'}
info["add"] = {f'Is({arg[0]},Down)'}
info["del_set"] = {f'Is({arg[0]},Up)'}
return info

2
robowaiter/llm_client/data/test_questions.txt
View File

@ -1 +1 @@
{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM：前往桌子2": {"Answer": " 测试VLM：前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}}
{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM：前往桌子2": {"Answer": " 测试VLM：前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关空调": {"Answer": "测试VLM：关空调", "Goal": "{\"Is(AC,Off)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：开大厅灯": {"Answer": "测试VLM：开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：开筒灯": {"Answer": "测试VLM：开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：开窗帘": {"Answer": "测试VLM：开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：整理椅子": {"Answer": "测试VLM：整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}}

7
robowaiter/llm_client/data_raw/test_questions.csv
View File

@ -4,9 +4,14 @@ Question,Answer,Goal
测试AEM,测试AEM,"{""EnvExplored()""}"
测试VLM：倒一杯水,测试VLM：倒一杯水,"{""On(Water,WaterTable)""}"
测试VLM：开空调,测试VLM：开空调,"{""Is(AC,On)""}"
测试VLM：关空调,测试VLM：关空调,"{""Is(AC,Off)""}"
测试VLM：关大厅灯,测试VLM：关大厅灯,"{""Is(HallLight,Off)""}"
测试VLM：开大厅灯,测试VLM：开大厅灯,"{""Is(HallLight,On)""}"
测试VLM：关筒灯,测试VLM：关筒灯,"{""Is(TubeLight,Off)""}"
测试VLM：关窗帘,测试VLM：关窗,"{""Is(Curtain,Off)""}"
测试VLM：开筒灯,测试VLM：开筒灯,"{""Is(TubeLight,On)""}"
测试VLM：关窗帘,测试VLM：关窗帘,"{""Is(Curtain,Off)""}"
测试VLM：开窗帘,测试VLM：开窗帘,"{""Is(Curtain,On)""}"
测试VLM：拖地,测试VLM：拖地,"{""Is(Floor,Clean)""}"
测试VLM：擦桌子,测试VLM：擦桌子,"{""Is(Table1,Clean)""}"
测试VLM：整理椅子,测试VLM：整理椅子,"{""Is(Chairs,Clean)""}"
测试VLM：把冰红茶放到Table2,测试VLM：把冰红茶放到Table2,"{""On(BottledDrink,Table2)""}"

1 Question Answer Goal
4 测试AEM 测试AEM {"EnvExplored()"}
5 测试VLM：倒一杯水 测试VLM：倒一杯水 {"On(Water,WaterTable)"}
6 测试VLM：开空调 测试VLM：开空调 {"Is(AC,On)"}
7 测试VLM：关空调 测试VLM：关空调 {"Is(AC,Off)"}
8 测试VLM：关大厅灯 测试VLM：关大厅灯 {"Is(HallLight,Off)"}
9 测试VLM：开大厅灯 测试VLM：开大厅灯 {"Is(HallLight,On)"}
10 测试VLM：关筒灯 测试VLM：关筒灯 {"Is(TubeLight,Off)"}
11 测试VLM：关窗帘 测试VLM：开筒灯 测试VLM：关窗 测试VLM：开筒灯 {"Is(Curtain,Off)"} {"Is(TubeLight,On)"}
12 测试VLM：关窗帘 测试VLM：关窗帘 {"Is(Curtain,Off)"}
13 测试VLM：开窗帘 测试VLM：开窗帘 {"Is(Curtain,On)"}
14 测试VLM：拖地 测试VLM：拖地 {"Is(Floor,Clean)"}
15 测试VLM：擦桌子 测试VLM：擦桌子 {"Is(Table1,Clean)"}
16 测试VLM：整理椅子 测试VLM：整理椅子 {"Is(Chairs,Clean)"}
17 测试VLM：把冰红茶放到Table2 测试VLM：把冰红茶放到Table2 {"On(BottledDrink,Table2)"}

3
robowaiter/robot/robot.py
View File

@ -48,7 +48,8 @@ class Robot(object):
print("\n--------------------")
print(f"首次运行行为树扩展算法，收集到{len(self.action_list)}个有效动作:")
for a in self.action_list:
print(a.name)
if "Turn" in a.name:
print(a.name)
print("--------------------\n")

7
robowaiter/scene/scene.py
View File

@ -416,13 +416,14 @@ class Scene:
self.robo_recover()
return True
def gen_obj(self,h=100):
def gen_obj(self,type=5,h=100):
# 4;冰红(盒) 5;酸奶 7:保温杯 9;冰红(瓶) 13:代语词典 14:cake 61:甜牛奶
type= 5 #9
scene = stub.Observe(GrabSim_pb2.SceneID(value=self.sceneID))
ginger_loc = [scene.location.X, scene.location.Y, scene.location.Z]
obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=9),
obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=5),
# GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9),
GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=7),
GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=9),
]
scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID))
time.sleep(1.0)

20
robowaiter/scene/tasks/VLM.py
View File

@ -13,14 +13,28 @@ class SceneVLM(Scene):
self.event_list = [
# (5, self.create_chat_event("测试VLM：做一杯咖啡")),
# (5, self.create_chat_event("测试VLM：倒一杯水")),
(5, self.create_chat_event("测试VLM：开空调")),
# (5, self.create_chat_event("测试VLM：开空调")),
# (5, self.create_chat_event("测试VLM：关空调")),
# (5, self.create_chat_event("测试VLM：开大厅灯")),
(5, self.create_chat_event("测试VLM：拖地")),
# (5, self.create_chat_event("测试VLM：擦桌子")),
# (5, self.create_chat_event("测试VLM：整理椅子")),
# (5, self.create_chat_event("测试VLM：把冰红茶放到Table2")),
# (5, self.create_chat_event("测试VLM：关大厅灯"))
]
def _reset(self):
self.state["condition_set"] = {'At(Robot,Bar)','Holding(Nothing)','Is(AC,Off)'}
self.state["condition_set"] = {'At(Robot,Bar)','Is(AC,Off)', 'Holding(Nothing)', # 'Holding(Yogurt)', #'Holding(Nothing)',
'Is(HallLight,Off)','Is(TubeLight,On)','Is(Curtain,On)',
'Is(Table1,Dirty)','Is(Floor,Dirty)','Is(Chairs,Dirty)'}
# self.gen_obj(type=5)
# # self.gen_obj(type=9)
# self.op_task_execute(op_type=16, obj_id=0)
# self.move_task_area(op_type=4)
pass
def _run(self, op_type=7):
def _run(self, op_type=10):
# 共17个操作
# "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7
# "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12

326
sub_task.ptml
View File

@ -1,7 +1,327 @@
selector{
cond Is(AC,On)
cond Is(Floor,Clean)
sequence{
cond Is(AC,Off)
act Turn(AC,On)
cond Holding(Nothing)
act Clean(Floor)
}
sequence{
cond Holding(Milk), At(Robot,Table2)
act PutDown(Milk,Table2)
}
sequence{
cond Holding(Milk), At(Robot,WaterTable)
act PutDown(Milk,WaterTable)
}
sequence{
cond At(Robot,CoffeeTable), Holding(Milk)
act PutDown(Milk,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(Milk)
act PutDown(Milk,Bar)
}
sequence{
cond Holding(Milk), At(Robot,Table1)
act PutDown(Milk,Table1)
}
sequence{
cond At(Robot,Table3), Holding(Milk)
act PutDown(Milk,Table3)
}
sequence{
cond Holding(Milk), At(Robot,Bar2)
act PutDown(Milk,Bar2)
}
sequence{
cond Holding(Coffee), At(Robot,Table2)
act PutDown(Coffee,Table2)
}
sequence{
cond At(Robot,WaterTable), Holding(Coffee)
act PutDown(Coffee,WaterTable)
}
sequence{
cond At(Robot,CoffeeTable), Holding(Coffee)
act PutDown(Coffee,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(Coffee)
act PutDown(Coffee,Bar)
}
sequence{
cond At(Robot,Table1), Holding(Coffee)
act PutDown(Coffee,Table1)
}
sequence{
cond At(Robot,Table3), Holding(Coffee)
act PutDown(Coffee,Table3)
}
sequence{
cond At(Robot,Bar2), Holding(Coffee)
act PutDown(Coffee,Bar2)
}
sequence{
cond Holding(Softdrink), At(Robot,Table2)
act PutDown(Softdrink,Table2)
}
sequence{
cond Holding(Softdrink), At(Robot,WaterTable)
act PutDown(Softdrink,WaterTable)
}
sequence{
cond At(Robot,CoffeeTable), Holding(Softdrink)
act PutDown(Softdrink,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(Softdrink)
act PutDown(Softdrink,Bar)
}
sequence{
cond Holding(Softdrink), At(Robot,Table1)
act PutDown(Softdrink,Table1)
}
sequence{
cond At(Robot,Table3), Holding(Softdrink)
act PutDown(Softdrink,Table3)
}
sequence{
cond Holding(Softdrink), At(Robot,Bar2)
act PutDown(Softdrink,Bar2)
}
sequence{
cond Holding(MilkDrink), At(Robot,Table2)
act PutDown(MilkDrink,Table2)
}
sequence{
cond Holding(MilkDrink), At(Robot,WaterTable)
act PutDown(MilkDrink,WaterTable)
}
sequence{
cond At(Robot,CoffeeTable), Holding(MilkDrink)
act PutDown(MilkDrink,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(MilkDrink)
act PutDown(MilkDrink,Bar)
}
sequence{
cond Holding(MilkDrink), At(Robot,Table1)
act PutDown(MilkDrink,Table1)
}
sequence{
cond At(Robot,Table3), Holding(MilkDrink)
act PutDown(MilkDrink,Table3)
}
sequence{
cond Holding(MilkDrink), At(Robot,Bar2)
act PutDown(MilkDrink,Bar2)
}
sequence{
cond Holding(Water), At(Robot,Table2)
act PutDown(Water,Table2)
}
sequence{
cond Holding(Water), At(Robot,WaterTable)
act PutDown(Water,WaterTable)
}
sequence{
cond At(Robot,CoffeeTable), Holding(Water)
act PutDown(Water,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(Water)
act PutDown(Water,Bar)
}
sequence{
cond Holding(Water), At(Robot,Table1)
act PutDown(Water,Table1)
}
sequence{
cond At(Robot,Table3), Holding(Water)
act PutDown(Water,Table3)
}
sequence{
cond Holding(Water), At(Robot,Bar2)
act PutDown(Water,Bar2)
}
sequence{
cond Holding(ADMilk), At(Robot,Table2)
act PutDown(ADMilk,Table2)
}
sequence{
cond At(Robot,WaterTable), Holding(ADMilk)
act PutDown(ADMilk,WaterTable)
}
sequence{
cond At(Robot,CoffeeTable), Holding(ADMilk)
act PutDown(ADMilk,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(ADMilk)
act PutDown(ADMilk,Bar)
}
sequence{
cond At(Robot,Table1), Holding(ADMilk)
act PutDown(ADMilk,Table1)
}
sequence{
cond At(Robot,Table3), Holding(ADMilk)
act PutDown(ADMilk,Table3)
}
sequence{
cond At(Robot,Bar2), Holding(ADMilk)
act PutDown(ADMilk,Bar2)
}
sequence{
cond At(Robot,Table2), Holding(VacuumCup)
act PutDown(VacuumCup,Table2)
}
sequence{
cond At(Robot,WaterTable), Holding(VacuumCup)
act PutDown(VacuumCup,WaterTable)
}
sequence{
cond At(Robot,CoffeeTable), Holding(VacuumCup)
act PutDown(VacuumCup,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(VacuumCup)
act PutDown(VacuumCup,Bar)
}
sequence{
cond At(Robot,Table1), Holding(VacuumCup)
act PutDown(VacuumCup,Table1)
}
sequence{
cond At(Robot,Table3), Holding(VacuumCup)
act PutDown(VacuumCup,Table3)
}
sequence{
cond At(Robot,Bar2), Holding(VacuumCup)
act PutDown(VacuumCup,Bar2)
}
sequence{
cond Holding(BottledDrink), At(Robot,Table2)
act PutDown(BottledDrink,Table2)
}
sequence{
cond Holding(BottledDrink), At(Robot,WaterTable)
act PutDown(BottledDrink,WaterTable)
}
sequence{
cond At(Robot,CoffeeTable), Holding(BottledDrink)
act PutDown(BottledDrink,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(BottledDrink)
act PutDown(BottledDrink,Bar)
}
sequence{
cond Holding(BottledDrink), At(Robot,Table1)
act PutDown(BottledDrink,Table1)
}
sequence{
cond At(Robot,Table3), Holding(BottledDrink)
act PutDown(BottledDrink,Table3)
}
sequence{
cond Holding(BottledDrink), At(Robot,Bar2)
act PutDown(BottledDrink,Bar2)
}
sequence{
cond Holding(Dessert), At(Robot,Table2)
act PutDown(Dessert,Table2)
}
sequence{
cond Holding(Dessert), At(Robot,WaterTable)
act PutDown(Dessert,WaterTable)
}
sequence{
cond Holding(Dessert), At(Robot,CoffeeTable)
act PutDown(Dessert,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(Dessert)
act PutDown(Dessert,Bar)
}
sequence{
cond Holding(Dessert), At(Robot,Table1)
act PutDown(Dessert,Table1)
}
sequence{
cond At(Robot,Table3), Holding(Dessert)
act PutDown(Dessert,Table3)
}
sequence{
cond Holding(Dessert), At(Robot,Bar2)
act PutDown(Dessert,Bar2)
}
sequence{
cond Holding(Yogurt), At(Robot,Table2)
act PutDown(Yogurt,Table2)
}
sequence{
cond Holding(Yogurt), At(Robot,WaterTable)
act PutDown(Yogurt,WaterTable)
}
sequence{
cond Holding(Yogurt), At(Robot,CoffeeTable)
act PutDown(Yogurt,CoffeeTable)
}
sequence{
cond At(Robot,Bar), Holding(Yogurt)
act PutDown(Yogurt,Bar)
}
sequence{
cond Holding(Yogurt), At(Robot,Table1)
act PutDown(Yogurt,Table1)
}
sequence{
cond At(Robot,Table3), Holding(Yogurt)
act PutDown(Yogurt,Table3)
}
sequence{
cond Holding(Yogurt), At(Robot,Bar2)
act PutDown(Yogurt,Bar2)
}
sequence{
cond Holding(Milk)
act MoveTo(Table2)
}
sequence{
cond Holding(Coffee)
act MoveTo(Table2)
}
sequence{
cond Holding(Softdrink)
act MoveTo(Table2)
}
sequence{
cond Holding(MilkDrink)
act MoveTo(Table2)
}
sequence{
cond Holding(Water)
act MoveTo(Table2)
}
sequence{
cond Holding(ADMilk)
act MoveTo(Table2)
}
sequence{
cond Holding(VacuumCup)
act MoveTo(Table2)
}
sequence{
cond Holding(BottledDrink)
act MoveTo(Table2)
}
sequence{
cond Holding(Dessert)
act MoveTo(Table2)
}
sequence{
cond Holding(Yogurt)
act MoveTo(Table2)
}
}