From b4e04641eb5d3f34f97bb1c559d28eded8b49eaa Mon Sep 17 00:00:00 2001 From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:48:03 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BA=86=20VLM=20=E7=9A=84?= =?UTF-8?q?=E4=B8=A4=E4=B8=AA=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/behavior_lib/_base/Behavior.py | 2 +- robowaiter/behavior_lib/act/GreetCustomer.py | 2 +- .../tasks/CafeDailyOperations/CafeOneDay.py | 93 ++++++++++++++++ .../tasks/CafeDailyOperations/VLN_all.py | 2 +- .../VLN_greet_and_order.py | 32 +++--- robowaiter/scene/tasks/VLM/VLM_2_AC.py | 103 +++++++++++++++++- robowaiter/scene/tasks/VLN/VLN_greet_lead.py | 40 +++++++ 7 files changed, 253 insertions(+), 21 deletions(-) create mode 100644 robowaiter/scene/tasks/CafeDailyOperations/CafeOneDay.py create mode 100644 robowaiter/scene/tasks/VLN/VLN_greet_lead.py diff --git a/robowaiter/behavior_lib/_base/Behavior.py b/robowaiter/behavior_lib/_base/Behavior.py index d070cba..0f53678 100644 --- a/robowaiter/behavior_lib/_base/Behavior.py +++ b/robowaiter/behavior_lib/_base/Behavior.py @@ -53,7 +53,7 @@ class Bahavior(ptree.behaviour.Behaviour): } place_xy_yaw_dic={ - 'Bar': (247.0, 520.0, 180.0), # (247.0, 520.0, 100.0) + 'Bar': (247.0, 520.0, 180), # (247.0, 520.0, 100.0) 'Bar2': (240.0, 40.0, 100.0), 'WaterTable': (-70.0, 500.0, 107), 'CoffeeTable': (250.0, 310.0, 100.0), diff --git a/robowaiter/behavior_lib/act/GreetCustomer.py b/robowaiter/behavior_lib/act/GreetCustomer.py index 2e87c24..fdf0608 100644 --- a/robowaiter/behavior_lib/act/GreetCustomer.py +++ b/robowaiter/behavior_lib/act/GreetCustomer.py @@ -21,7 +21,7 @@ class GreetCustomer(Act): def _update(self) -> ptree.common.Status: - goal = Act.place_xyz_dic['Bar'] + goal = Act.place_xy_yaw_dic['Bar'] self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0) # self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？") if self.scene.show_bubble: diff --git a/robowaiter/scene/tasks/CafeDailyOperations/CafeOneDay.py b/robowaiter/scene/tasks/CafeDailyOperations/CafeOneDay.py new file mode 100644 index 0000000..34de4cc --- /dev/null +++ b/robowaiter/scene/tasks/CafeDailyOperations/CafeOneDay.py @@ -0,0 +1,93 @@ +""" +视觉语言操作 +机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板 +""" + +import time +from robowaiter.scene.scene import Scene + + +class SceneVLM(Scene): + + def __init__(self, robot): + super().__init__(robot) + # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) + + self.scene_flag = 1 + self.st1 = 3 + # self.st2 = self.st1 + 30 + # self.st3 = self.st2 + 65 + self.st2 = 3 + self.st3 = 3 + self.st4 = 3 + + self.signal_event_list = [ + + # 场景1：带小女孩找阳光下的空位 + (3, self.add_walker, (5, 230, 1200)), # 0号"Girl02_C_3" + (1, self.control_walker, (0, False, 200, 60, 520, 0)), + (9, self.customer_say, (0, "早上好呀，我想找个能晒太阳的地方。")), + (-1, self.customer_say, (0, "可以带我过去嘛？")), + (0, self.control_walker, (0, False, 50, 140, 1200, 180)), # 小女孩站在了 BrightTable1 旁边就餐啦 + + # # 场景2：有个胖胖男人点单交流并要咖啡，帮他送去角落的桌子 + # (3, self.add_walker, (5, 230, 1200)), # 小女孩 + # # # 上述准备 + (10, self.add_walker, (26, -28, -150, 90)), + (0, self.add_walker, (10, -70, -200, -45)), + (6, self.customer_say, (1, "嘿，RoboWaiter，过来一下！")), + (8, self.control_walkers_and_say, ([[[1, False, 100, -18, -200, -90, "你们这有什么饮料嘛？"]]])), + # 20 胖胖男到了 BrightTable6 + (2, self.customer_say, (1, "咖啡有哪些呢？")), # 10 + (2, self.customer_say, (1, "来杯卡布奇诺吧。")), # 15 + + + + ] + + def _reset(self): + self.gen_obj() + # self.add_walkers([[47, 920]]) + pass + + def _run(self, op_type=10): + # 一个行人从门口走到 吧台 + # 打招呼需要什么 + # 行人说 哪里有位置，想晒个太阳 + # 带领行人去有太阳的地方 + # 行人说 有点热 + # 好的，这就去开空调 + self.walker_followed = False + pass + + def _step(self): + + if self.scene_flag == 1: + # 如果机器人不在 吧台 + if self.walker_followed: + return + end = [self.status.location.X, self.status.location.Y] + if end[1] >= 600 or end[1] <= 450 or end[0] >= 250: + # if int(self.status.location.X)!=247 or int(self.status.location.X)!=520: + self.walker_followed = True + self.control_walkers_and_say([[0, False, 150, end[0], end[1], 90, "谢谢！"]]) + self.scene_flag += 1 + + # 获得所有顾客的名字 + # print("=================") + # for cus in self.status.walkers: + # print(cus) + # print("=================") + pass + + +if __name__ == '__main__': + import os + from robowaiter.robot.robot import Robot + + robot = Robot() + + # create task + task = SceneVLM(robot) + task.reset() + task.run() diff --git a/robowaiter/scene/tasks/CafeDailyOperations/VLN_all.py b/robowaiter/scene/tasks/CafeDailyOperations/VLN_all.py index fafa549..8926794 100644 --- a/robowaiter/scene/tasks/CafeDailyOperations/VLN_all.py +++ b/robowaiter/scene/tasks/CafeDailyOperations/VLN_all.py @@ -45,7 +45,7 @@ class SceneVLM(Scene): # # 场景3：有位女士要杯水和冰红茶 (30 ,self.add_walkers,([[[21, 65, 1000, -90],[32, -80, 850, 135],[1, 60, 420, 135]]])), (0, self.control_walker, (5, True, 50, 250, 1200, 180)), #设置id=4 的2小男孩随机游走红随机游走 - (0, self.add_walkers, ([[[48, 60, 520,0], [31, 60, 600, -90], [20, 60, 680, -90],[9, 60, 760, -90],[29, -290, 400, 180]]])), + (0, self.add_walkers, ([[[48, 60, 520, 0], [31, 60, 600, -90], [20, 60, 680, -90],[9, 60, 760, -90],[29, -290, 400, 180]]])), (5, self.customer_say, (6, "哎呦，今天这么多人，还有空位吗？")),# 女士问 (15, self.customer_say, (6, "我带着孩子呢，想要宽敞亮堂的地方。")), # 女士问 diff --git a/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py index df238a9..654ab2e 100644 --- a/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py +++ b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py @@ -36,11 +36,11 @@ class SceneVLM(Scene): # # 场景2：有个胖胖男人点单交流并要咖啡，帮他送去角落的桌子 (3, self.add_walker, (5, 230, 1200)), # 小女孩 # # # 上述准备 - (10, self.add_walker, (26, -28, -150, 90)), + (3, self.add_walker, (26, -28, -150, 90)), (0, self.add_walker, (10, -70, -200, -45)), - (6, self.customer_say, (1, "嘿，RoboWaiter，过来一下！")), - (8, self.control_walkers_and_say, ([[[1, False, 100, -18, -200, -90, "你们这有什么饮料嘛？"]]])), # 20 胖胖男到了 BrightTable6 - (2, self.customer_say, (1, "咖啡有哪些呢？")),# 10 + # (6, self.customer_say, (1, "嘿，RoboWaiter，过来一下！")), + # (8, self.control_walkers_and_say, ([[[1, False, 100, -18, -200, -90, "你们这有什么饮料嘛？"]]])), # 20 胖胖男到了 BrightTable6 + # (2, self.customer_say, (1, "咖啡有哪些呢？")),# 10 (2, self.customer_say, (1,"来杯卡布奇诺吧。")), # 15 @@ -52,29 +52,29 @@ class SceneVLM(Scene): # (0, self.add_walker, (21, 65, 1000, -90)), # 男 'BrightTable2': (65, 1000, 135), # (0, self.add_walker, (32, -80, 850, 135)), # 女 'BrightTable3': (-80, 850, 135), # (0, self.add_walker, (1, 60, 420, 135)), # 生成小男孩随机游走 - (0,self.add_walkers,([[[21, 65, 1000, -90],[32, -80, 850, 135],[1, 60, 420, 135]]])), + (3, self.add_walkers,([[[21, 65, 1000, -90],[32, -80, 850, 135],[1, 60, 420, 135],[29, -290, 400, 180]]])), (0, self.control_walker, (5, True, 50, 250, 1200, 180)), #设置id=4 的2小男孩随机游走红随机游走 (0, self.add_walker, (48, 60, 520,0)),# 生成他妈妈 # (0, self.add_walker, (31, 60, 600, -90)), # 女红色排队 7号 # (0, self.add_walker, (20, 60, 680, -90)), # 大胖男色排队 8号 # (0, self.add_walker, (9, 60, 760, -90)), # 男灰黑排队 9号 # (0, self.add_walker, (29, -290, 400, 180)), # 青色女人占了位置 BrightTable5 - (0, self.add_walkers, ([[[48, 60, 520,0], [31, 60, 600, -90], [20, 60, 680, -90],[9, 60, 760, -90],[29, -290, 400, 180]]])), - # - # # (5, self.customer_say, (6, "哎呦，今天这么多人，还有空位吗？")),# 女士问 - # # (15, self.customer_say, (6, "我带着孩子呢，想要宽敞亮堂的地方。")), # 女士问 + (0, self.add_walkers, ([[[48, 60, 520,0], [31, 60, 600, -90], [20, 60, 680, -90],[9, 60, 760, -90]]])), + + (29, self.customer_say, (7, "哎呦，今天这么多人，还有空位吗？")),# 女士问 + (10, self.customer_say, (7, "我带着孩子呢，想要宽敞亮堂的地方。")), # 女士问 # # 好的，我明白了，那么您可以选择我们的家庭亲子座，这样可以容纳您的孩子，并且更加宽敞舒适。 # # 这里可以加一下自主导航和探索，找到一个位置 # # 好的，我明白了，那么我们推荐您到大厅的桌子，那里的空间比较宽敞，环境也比较明亮，适合带着孩子一起用餐。 - # (8, self.customer_say, (6, "大厅的桌子好啊，快带我去呀！")), - # (15, self.control_walker, (6, False, 50,-250, 480, 0)), # #290, 400 - # # (3, self.customer_say, (6, "我想来杯水，帮我孩子拿个酸奶吧。")), + (8, self.customer_say, (7, "大厅的桌子好啊，快带我去呀！")), + (15, self.control_walker, (7, False, 50,-250, 480, 0)), # #290, 400 + # (3, self.customer_say, (7, "我想来杯水，帮我孩子拿个酸奶吧。")), # # ### 9号灰色男 排队排着排着，不排了 - # (0, self.control_walker, (9, False, 100, 100, 760, 180)), - # (0, self.control_walker, (9, True, 100, 0, 0, 180)), + (0, self.control_walker, (10, False, 100, 100, 760, 180)), + (0, self.control_walker, (10, True, 100, 0, 0, 180)), # # # ### 增加场景，孩子说热要开空调 或者9号随机游走和说 - # # (90, self.customer_say, (6, "谢谢！")), #倒水+取放酸奶 90s - # (3, self.customer_say, (6, "谢谢！")), + # # (90, self.customer_say, (7, "谢谢！")), #倒水+取放酸奶 90s + # (3, self.customer_say, (7, "谢谢！")), diff --git a/robowaiter/scene/tasks/VLM/VLM_2_AC.py b/robowaiter/scene/tasks/VLM/VLM_2_AC.py index 219a528..94d2bfe 100644 --- a/robowaiter/scene/tasks/VLM/VLM_2_AC.py +++ b/robowaiter/scene/tasks/VLM/VLM_2_AC.py @@ -13,18 +13,117 @@ class SceneVLM(Scene): self.signal_event_list = [ (3, self.add_walker, (0,0,700)), (1, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw] - (1, self.customer_say, (6, "好热呀，想开空调，想要温度调低点！")), + (3, self.customer_say, (6, "好热呀，想开空调，想要温度调低点！")), (5, self.control_walker, (6, False, 200, 60, 80, 0)), (-1, self.customer_say, (6, "谢谢！这下凉快了！")), #(-100,600) + + # 有人提出要开空调和关窗帘 + # bar (60, 520) + # (28, self.add_walker, (0, 0, 0)), + # (33, self.control_walker, (7, False, 100, 60, 520, 180)), + # (35, self.customer_say, (7,"好热呀！太阳也好大！")), + # (45, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])), + + # (3, self.add_walker, (0, 0, 0)), + # (1, self.control_walker, (5, False, 100, 60, 520, 180)), + # (1, self.customer_say, (5,"好热呀！太阳也好大！")), + # (-1, self.control_walkers_and_say, ([[[5, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])), ] def _reset(self): self.gen_obj() + # self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)', + # 'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)', + # 'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)', + # 'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'} + # # 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220) self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]]) - self.control_walkers(walker_loc=[[-55, 750], [70, -200]],is_autowalk = True) + # [3,1, 880] 1号桌旁边小女孩 + # [31,250, 1200] 最角落QuietTable1女红色 + # [6,-55, 750] 1号桌附近小男孩 + # [10,70, -200] 另一边角落 QuietTable2 男黄色 + # [27,-290, 400, 180] 中间 BrightTable4 女灰 + # [26, 60,-320,90] 另一边角落 BrightTable5 红胖男 + self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True) + + # 0-3男孩 4-7女孩 8-26男 + # 3男孩 31女红 32女灰 10黄色衣服男瘦 9男灰瘦 26红胖男 + # 17 是员工 police + # [0, -150,180] + + + # self.control_walkers(walker_loc=[[-55, 750]],is_autowalk = False) + # 在场景中随机增加一堆行人。 + # walker_loc = [[-55, 750], [70, -200], [250, 1200], [0, 880]] + # controls = [] + # for i in range(len(s.walkers)): + # loc = walker_loc[i] + # is_autowalk = False + # pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180) + # controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=200, pose=pose)) + # scene = sim_client.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=scene_id)) + + # self.gen_obj(type=5) + # self.gen_obj(type=9) + # self.op_task_execute(op_type=16, obj_id=0) + # self.move_task_area(op_type=4) pass def _run(self, op_type=10): + # 共17个操作 + # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7 + # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12 + # "调整空调开关","调高空调温度","调低空调温度", # 13-15 + # "抓握物体","放置物体" # 16-17 + # + # self.gen_obj() + # if op_type <=15: + # self.move_task_area(op_type) + # self.op_task_execute(op_type) + # if op_type == 16: # 16: 抓操作需要传入物品id + # self.move_task_area(op_type, obj_id=0) + # self.op_task_execute(op_type, obj_id=0) + # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107] + # # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0] # 桌子2:[-55.0, 150.0, 107], + # if op_type == 17: # 17: 放操作需要传入放置位置周围的可达区域 + # pos = [240.0, 40.0, 100.0] + # self.move_task_area(op_type, release_pos=pos) + # self.op_task_execute(op_type, release_pos=pos) # [325.0, 860.0, 100] + # + # 流程测试 + # 抓握放置:抓吧台前生成的酸奶，放到抹布桌上 + # self.gen_obj() + # self.move_task_area(16, obj_id=0) + # self.op_task_execute(16, obj_id=0) + # pos = [340.0, 900.0, 99.0] + # self.move_task_area(17, release_pos=pos) + # self.op_task_execute(17, release_pos=pos) + # + # # 做咖啡:做完的咖啡放到水杯桌上 + # self.move_task_area(1) + # self.op_task_execute(1) + # + # self.find_obj("CoffeeCup") + # + # self.move_task_area(16, obj_id=275) + # self.op_task_execute(16, obj_id=275) + # pos = [-70.0, 500.0, 107] + # self.move_task_area(17, release_pos=pos) + # self.op_task_execute(17, release_pos=pos) + # + # # 倒水:倒完的水放到旁边桌子上 + # self.move_task_area(2) + # self.op_task_execute(2) + # + # + # self.move_task_area(16, obj_id=190) + # self.op_task_execute(16, obj_id=190) + # pos = [-55.0, 0.0, 107] + # self.move_task_area(17, release_pos=pos) + # self.op_task_execute(17, release_pos=pos) + # + # self.test_yaw() + pass def _step(self): diff --git a/robowaiter/scene/tasks/VLN/VLN_greet_lead.py b/robowaiter/scene/tasks/VLN/VLN_greet_lead.py new file mode 100644 index 0000000..0df00bd --- /dev/null +++ b/robowaiter/scene/tasks/VLN/VLN_greet_lead.py @@ -0,0 +1,40 @@ +""" +视觉语言操作 +机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板 +""" + +import time +from robowaiter.scene.scene import Scene + +class SceneVLM(Scene): + def __init__(self, robot): + super().__init__(robot) + # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) + self.new_event_list = [ + (3, self.add_walker, (29,60,520)), + (5, self.customer_say, (6,"请问哪里有空位啊？")), + ] + + def _reset(self): + self.gen_obj() + self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]]) + self.control_walkers(walker_loc=[[-55, 750], [70, -200]],is_autowalk = True) + pass + + def _run(self): + pass + + def _step(self): + pass + + +if __name__ == '__main__': + import os + from robowaiter.robot.robot import Robot + + robot = Robot() + + # create task + task = SceneVLM(robot) + task.reset() + task.run()