diff --git a/robowaiter/behavior_lib/act/PutDown.py b/robowaiter/behavior_lib/act/PutDown.py index 30a6476..532fd31 100644 --- a/robowaiter/behavior_lib/act/PutDown.py +++ b/robowaiter/behavior_lib/act/PutDown.py @@ -21,7 +21,7 @@ class PutDown(Act): info = {} info["pre"] = {f'Holding({arg[0]})',f'At(Robot,{arg[1]})'} info["add"] = {f'Holding(Nothing)',f'At({arg[0]},{arg[1]})'} - info["del_set"] = {f'Holding(Nothing)'} + info["del_set"] = {f'Holding({arg[0]})'} return info diff --git a/robowaiter/behavior_lib/act/Turn.py b/robowaiter/behavior_lib/act/Turn.py index 73d0551..b499e24 100644 --- a/robowaiter/behavior_lib/act/Turn.py +++ b/robowaiter/behavior_lib/act/Turn.py @@ -41,29 +41,33 @@ class Turn(Act): self.op_type = 10 elif self.target_obj=="Curtain": if self.op == 'On': - self.op_type = 11 - elif self.op == 'Off': self.op_type = 12 + elif self.op == 'Off': + self.op_type = 11 @classmethod def get_info(cls,*arg): info = {} + info["pre"] = set() if arg[0]=="TubeLight" or arg[0]=="HallLight" or arg[0]=="Curtain" or arg[0]=='AC': + if arg[0]!="Curtain": + info["pre"] |= {f'Holding(Nothing)'} if arg[1]=="On": - info["pre"] = {f'Is({arg[0]},Off)'} + info["pre"] |= {f'Is({arg[0]},Off)'} info["add"] = {f'Is({arg[0]},On)'} info["del_set"] = {f'Is({arg[0]},Off)'} elif arg[1]=="Off": - info["pre"] = {f'Is({arg[0]},On)'} + info["pre"] |= {f'Is({arg[0]},On)'} info["add"] = {f'Is({arg[0]},Off)'} info["del_set"] = {f'Is({arg[0]},On)'} elif arg[0]=='ACTemperature': + info["pre"] = {f'Holding(Nothing)'} if arg[1]=="Up": - info["pre"] = {f'Is({arg[0]},Down)'} + info["pre"] |= {f'Is({arg[0]},Down)'} info["add"] = {f'Is({arg[0]},Up)'} info["del_set"] = {f'Is({arg[0]},Down)'} - elif arg[1]=="Donw": - info["pre"] = {f'Is({arg[0]},Up)'} + elif arg[1]=="Down": + info["pre"] |= {f'Is({arg[0]},Up)'} info["add"] = {f'Is({arg[0]},Down)'} info["del_set"] = {f'Is({arg[0]},Up)'} return info diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt index 42f5756..44b4914 100644 --- a/robowaiter/llm_client/data/test_questions.txt +++ b/robowaiter/llm_client/data/test_questions.txt @@ -1 +1 @@ -{"测试VLM:做一杯咖啡": {"Answer": "测试VLM:做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM:前往桌子2": {"Answer": " 测试VLM:前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM:倒一杯水": {"Answer": "测试VLM:倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM:开空调": {"Answer": "测试VLM:开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM:关大厅灯": {"Answer": "测试VLM:关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM:关筒灯": {"Answer": "测试VLM:关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM:关窗帘": {"Answer": "测试VLM:关窗", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM:拖地": {"Answer": "测试VLM:拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM:擦桌子": {"Answer": "测试VLM:擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM:把冰红茶放到Table2": {"Answer": "测试VLM:把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}} +{"测试VLM:做一杯咖啡": {"Answer": "测试VLM:做一杯咖啡", "Goal": "{\"At(Coffee,Bar)\"}"}, "测试VLM:前往桌子2": {"Answer": " 测试VLM:前往桌子2", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM:倒一杯水": {"Answer": "测试VLM:倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM:开空调": {"Answer": "测试VLM:开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM:关空调": {"Answer": "测试VLM:关空调", "Goal": "{\"Is(AC,Off)\"}"}, "测试VLM:关大厅灯": {"Answer": "测试VLM:关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM:开大厅灯": {"Answer": "测试VLM:开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "测试VLM:关筒灯": {"Answer": "测试VLM:关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM:开筒灯": {"Answer": "测试VLM:开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "测试VLM:关窗帘": {"Answer": "测试VLM:关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM:开窗帘": {"Answer": "测试VLM:开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "测试VLM:拖地": {"Answer": "测试VLM:拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM:擦桌子": {"Answer": "测试VLM:擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM:整理椅子": {"Answer": "测试VLM:整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "测试VLM:把冰红茶放到Table2": {"Answer": "测试VLM:把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}} diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv index fda2291..9313d61 100644 --- a/robowaiter/llm_client/data_raw/test_questions.csv +++ b/robowaiter/llm_client/data_raw/test_questions.csv @@ -4,9 +4,14 @@ Question,Answer,Goal AEM,AEM,"{""EnvExplored()""}" VLMһˮ,VLMһˮ,"{""On(Water,WaterTable)""}" VLMյ,VLMյ,"{""Is(AC,On)""}" +VLMؿյ,VLMؿյ,"{""Is(AC,Off)""}" VLMش,VLMش,"{""Is(HallLight,Off)""}" +VLM,VLM,"{""Is(HallLight,On)""}" VLMͲ,VLMͲ,"{""Is(TubeLight,Off)""}" -VLMش,VLMش,"{""Is(Curtain,Off)""}" +VLMͲ,VLMͲ,"{""Is(TubeLight,On)""}" +VLMش,VLMش,"{""Is(Curtain,Off)""}" +VLM,VLM,"{""Is(Curtain,On)""}" VLMϵ,VLMϵ,"{""Is(Floor,Clean)""}" VLM,VLM,"{""Is(Table1,Clean)""}" +VLM,VLM,"{""Is(Chairs,Clean)""}" VLMѱŵTable2,VLMѱŵTable2,"{""On(BottledDrink,Table2)""}" diff --git a/robowaiter/robot/robot.py b/robowaiter/robot/robot.py index 185c7d7..ec2dadd 100644 --- a/robowaiter/robot/robot.py +++ b/robowaiter/robot/robot.py @@ -48,7 +48,8 @@ class Robot(object): print("\n--------------------") print(f"首次运行行为树扩展算法,收集到{len(self.action_list)}个有效动作:") for a in self.action_list: - print(a.name) + if "Turn" in a.name: + print(a.name) print("--------------------\n") diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py index 17a0ccd..3c350b8 100644 --- a/robowaiter/scene/scene.py +++ b/robowaiter/scene/scene.py @@ -416,13 +416,14 @@ class Scene: self.robo_recover() return True - def gen_obj(self,h=100): + def gen_obj(self,type=5,h=100): # 4;冰红(盒) 5;酸奶 7:保温杯 9;冰红(瓶) 13:代语词典 14:cake 61:甜牛奶 + type= 5 #9 scene = stub.Observe(GrabSim_pb2.SceneID(value=self.sceneID)) ginger_loc = [scene.location.X, scene.location.Y, scene.location.Z] - obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=9), + obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=5), # GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9), - GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=7), + GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=9), ] scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID)) time.sleep(1.0) diff --git a/robowaiter/scene/tasks/VLM.py b/robowaiter/scene/tasks/VLM.py index 85052d5..eae59a5 100644 --- a/robowaiter/scene/tasks/VLM.py +++ b/robowaiter/scene/tasks/VLM.py @@ -13,14 +13,28 @@ class SceneVLM(Scene): self.event_list = [ # (5, self.create_chat_event("测试VLM:做一杯咖啡")), # (5, self.create_chat_event("测试VLM:倒一杯水")), - (5, self.create_chat_event("测试VLM:开空调")), + # (5, self.create_chat_event("测试VLM:开空调")), + # (5, self.create_chat_event("测试VLM:关空调")), + # (5, self.create_chat_event("测试VLM:开大厅灯")), + (5, self.create_chat_event("测试VLM:拖地")), + # (5, self.create_chat_event("测试VLM:擦桌子")), + # (5, self.create_chat_event("测试VLM:整理椅子")), + # (5, self.create_chat_event("测试VLM:把冰红茶放到Table2")), + # (5, self.create_chat_event("测试VLM:关大厅灯")) ] def _reset(self): - self.state["condition_set"] = {'At(Robot,Bar)','Holding(Nothing)','Is(AC,Off)'} + self.state["condition_set"] = {'At(Robot,Bar)','Is(AC,Off)', 'Holding(Nothing)', # 'Holding(Yogurt)', #'Holding(Nothing)', + 'Is(HallLight,Off)','Is(TubeLight,On)','Is(Curtain,On)', + 'Is(Table1,Dirty)','Is(Floor,Dirty)','Is(Chairs,Dirty)'} + # self.gen_obj(type=5) + # # self.gen_obj(type=9) + # self.op_task_execute(op_type=16, obj_id=0) + # self.move_task_area(op_type=4) pass - def _run(self, op_type=7): + def _run(self, op_type=10): + # 共17个操作 # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7 # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12 diff --git a/sub_task.ptml b/sub_task.ptml index e2a81fa..7671dad 100644 --- a/sub_task.ptml +++ b/sub_task.ptml @@ -1,7 +1,327 @@ selector{ -cond Is(AC,On) +cond Is(Floor,Clean) sequence{ -cond Is(AC,Off) -act Turn(AC,On) +cond Holding(Nothing) +act Clean(Floor) +} +sequence{ +cond Holding(Milk), At(Robot,Table2) +act PutDown(Milk,Table2) +} +sequence{ +cond Holding(Milk), At(Robot,WaterTable) +act PutDown(Milk,WaterTable) +} +sequence{ +cond At(Robot,CoffeeTable), Holding(Milk) +act PutDown(Milk,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(Milk) +act PutDown(Milk,Bar) +} +sequence{ +cond Holding(Milk), At(Robot,Table1) +act PutDown(Milk,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(Milk) +act PutDown(Milk,Table3) +} +sequence{ +cond Holding(Milk), At(Robot,Bar2) +act PutDown(Milk,Bar2) +} +sequence{ +cond Holding(Coffee), At(Robot,Table2) +act PutDown(Coffee,Table2) +} +sequence{ +cond At(Robot,WaterTable), Holding(Coffee) +act PutDown(Coffee,WaterTable) +} +sequence{ +cond At(Robot,CoffeeTable), Holding(Coffee) +act PutDown(Coffee,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(Coffee) +act PutDown(Coffee,Bar) +} +sequence{ +cond At(Robot,Table1), Holding(Coffee) +act PutDown(Coffee,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(Coffee) +act PutDown(Coffee,Table3) +} +sequence{ +cond At(Robot,Bar2), Holding(Coffee) +act PutDown(Coffee,Bar2) +} +sequence{ +cond Holding(Softdrink), At(Robot,Table2) +act PutDown(Softdrink,Table2) +} +sequence{ +cond Holding(Softdrink), At(Robot,WaterTable) +act PutDown(Softdrink,WaterTable) +} +sequence{ +cond At(Robot,CoffeeTable), Holding(Softdrink) +act PutDown(Softdrink,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(Softdrink) +act PutDown(Softdrink,Bar) +} +sequence{ +cond Holding(Softdrink), At(Robot,Table1) +act PutDown(Softdrink,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(Softdrink) +act PutDown(Softdrink,Table3) +} +sequence{ +cond Holding(Softdrink), At(Robot,Bar2) +act PutDown(Softdrink,Bar2) +} +sequence{ +cond Holding(MilkDrink), At(Robot,Table2) +act PutDown(MilkDrink,Table2) +} +sequence{ +cond Holding(MilkDrink), At(Robot,WaterTable) +act PutDown(MilkDrink,WaterTable) +} +sequence{ +cond At(Robot,CoffeeTable), Holding(MilkDrink) +act PutDown(MilkDrink,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(MilkDrink) +act PutDown(MilkDrink,Bar) +} +sequence{ +cond Holding(MilkDrink), At(Robot,Table1) +act PutDown(MilkDrink,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(MilkDrink) +act PutDown(MilkDrink,Table3) +} +sequence{ +cond Holding(MilkDrink), At(Robot,Bar2) +act PutDown(MilkDrink,Bar2) +} +sequence{ +cond Holding(Water), At(Robot,Table2) +act PutDown(Water,Table2) +} +sequence{ +cond Holding(Water), At(Robot,WaterTable) +act PutDown(Water,WaterTable) +} +sequence{ +cond At(Robot,CoffeeTable), Holding(Water) +act PutDown(Water,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(Water) +act PutDown(Water,Bar) +} +sequence{ +cond Holding(Water), At(Robot,Table1) +act PutDown(Water,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(Water) +act PutDown(Water,Table3) +} +sequence{ +cond Holding(Water), At(Robot,Bar2) +act PutDown(Water,Bar2) +} +sequence{ +cond Holding(ADMilk), At(Robot,Table2) +act PutDown(ADMilk,Table2) +} +sequence{ +cond At(Robot,WaterTable), Holding(ADMilk) +act PutDown(ADMilk,WaterTable) +} +sequence{ +cond At(Robot,CoffeeTable), Holding(ADMilk) +act PutDown(ADMilk,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(ADMilk) +act PutDown(ADMilk,Bar) +} +sequence{ +cond At(Robot,Table1), Holding(ADMilk) +act PutDown(ADMilk,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(ADMilk) +act PutDown(ADMilk,Table3) +} +sequence{ +cond At(Robot,Bar2), Holding(ADMilk) +act PutDown(ADMilk,Bar2) +} +sequence{ +cond At(Robot,Table2), Holding(VacuumCup) +act PutDown(VacuumCup,Table2) +} +sequence{ +cond At(Robot,WaterTable), Holding(VacuumCup) +act PutDown(VacuumCup,WaterTable) +} +sequence{ +cond At(Robot,CoffeeTable), Holding(VacuumCup) +act PutDown(VacuumCup,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(VacuumCup) +act PutDown(VacuumCup,Bar) +} +sequence{ +cond At(Robot,Table1), Holding(VacuumCup) +act PutDown(VacuumCup,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(VacuumCup) +act PutDown(VacuumCup,Table3) +} +sequence{ +cond At(Robot,Bar2), Holding(VacuumCup) +act PutDown(VacuumCup,Bar2) +} +sequence{ +cond Holding(BottledDrink), At(Robot,Table2) +act PutDown(BottledDrink,Table2) +} +sequence{ +cond Holding(BottledDrink), At(Robot,WaterTable) +act PutDown(BottledDrink,WaterTable) +} +sequence{ +cond At(Robot,CoffeeTable), Holding(BottledDrink) +act PutDown(BottledDrink,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(BottledDrink) +act PutDown(BottledDrink,Bar) +} +sequence{ +cond Holding(BottledDrink), At(Robot,Table1) +act PutDown(BottledDrink,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(BottledDrink) +act PutDown(BottledDrink,Table3) +} +sequence{ +cond Holding(BottledDrink), At(Robot,Bar2) +act PutDown(BottledDrink,Bar2) +} +sequence{ +cond Holding(Dessert), At(Robot,Table2) +act PutDown(Dessert,Table2) +} +sequence{ +cond Holding(Dessert), At(Robot,WaterTable) +act PutDown(Dessert,WaterTable) +} +sequence{ +cond Holding(Dessert), At(Robot,CoffeeTable) +act PutDown(Dessert,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(Dessert) +act PutDown(Dessert,Bar) +} +sequence{ +cond Holding(Dessert), At(Robot,Table1) +act PutDown(Dessert,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(Dessert) +act PutDown(Dessert,Table3) +} +sequence{ +cond Holding(Dessert), At(Robot,Bar2) +act PutDown(Dessert,Bar2) +} +sequence{ +cond Holding(Yogurt), At(Robot,Table2) +act PutDown(Yogurt,Table2) +} +sequence{ +cond Holding(Yogurt), At(Robot,WaterTable) +act PutDown(Yogurt,WaterTable) +} +sequence{ +cond Holding(Yogurt), At(Robot,CoffeeTable) +act PutDown(Yogurt,CoffeeTable) +} +sequence{ +cond At(Robot,Bar), Holding(Yogurt) +act PutDown(Yogurt,Bar) +} +sequence{ +cond Holding(Yogurt), At(Robot,Table1) +act PutDown(Yogurt,Table1) +} +sequence{ +cond At(Robot,Table3), Holding(Yogurt) +act PutDown(Yogurt,Table3) +} +sequence{ +cond Holding(Yogurt), At(Robot,Bar2) +act PutDown(Yogurt,Bar2) +} +sequence{ +cond Holding(Milk) +act MoveTo(Table2) +} +sequence{ +cond Holding(Coffee) +act MoveTo(Table2) +} +sequence{ +cond Holding(Softdrink) +act MoveTo(Table2) +} +sequence{ +cond Holding(MilkDrink) +act MoveTo(Table2) +} +sequence{ +cond Holding(Water) +act MoveTo(Table2) +} +sequence{ +cond Holding(ADMilk) +act MoveTo(Table2) +} +sequence{ +cond Holding(VacuumCup) +act MoveTo(Table2) +} +sequence{ +cond Holding(BottledDrink) +act MoveTo(Table2) +} +sequence{ +cond Holding(Dessert) +act MoveTo(Table2) +} +sequence{ +cond Holding(Yogurt) +act MoveTo(Table2) } }