From 5d2c9009f3f59f6eef5261bf018629f4c0526276 Mon Sep 17 00:00:00 2001 From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com> Date: Sat, 18 Nov 2023 22:50:38 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BA=86=E5=92=96=E5=95=A1?= =?UTF-8?q?=E5=8E=85=E7=9A=84=E4=B8=80=E5=A4=A9=EF=BC=9A=E5=B8=A6=E9=A2=86?= =?UTF-8?q?=E9=A1=BE=E5=AE=A2=E5=92=8C=E7=82=B9=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/behavior_lib/_base/Behavior.py | 2 +- robowaiter/llm_client/data/fix_questions.txt | 15 +++- robowaiter/scene/scene.py | 3 + .../VLN_greet_and_order.py | 86 +++++++++++++++++++ robowaiter/scene/tasks/VLM/VLN_greet_lead.py | 5 +- 5 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py diff --git a/robowaiter/behavior_lib/_base/Behavior.py b/robowaiter/behavior_lib/_base/Behavior.py index 6477fae..635a22c 100644 --- a/robowaiter/behavior_lib/_base/Behavior.py +++ b/robowaiter/behavior_lib/_base/Behavior.py @@ -41,7 +41,7 @@ class Bahavior(ptree.behaviour.Behaviour): 'QuietTable1':(480,1300,90), 'QuietTable2':(250,-240,-65), - 'BrightTable1':(230,1200,135), + 'BrightTable1':(230,1200,-135), 'BrightTable2': (65, 1000, 135), 'BrightTable3': (-80, 850, 135), 'BrightTable4': (-270, 520, 150), diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt index 0fa6d5f..4be5718 100644 --- a/robowaiter/llm_client/data/fix_questions.txt +++ b/robowaiter/llm_client/data/fix_questions.txt @@ -4,7 +4,7 @@ 做一杯咖啡 好的，我马上做咖啡 create_sub_task -{"goal":"On(Coffee,CoffeeTable)"} +{"goal":"On(Coffee,CoffeeTable)"}00 不用了 好的，您有需要再跟我说 @@ -18,7 +18,7 @@ create_sub_task 早上好呀，我想找个能晒太阳的地方。 -没问题，您右手边就有能晒太阳的位置呢。 +您右手边就有能晒太阳的位置呢。 可以带我过去嘛？ @@ -26,6 +26,17 @@ create_sub_task create_sub_task {"goal":"At(Robot,BrightTable1)"} +你们这有什么饮料嘛？ +我们咖啡厅提供各种口味咖啡，水，冰红茶，酸奶等饮品，还提供点心蛋糕等甜品。您想点什么？ + +来杯卡布奇诺吧，我在靠窗边的高脚桌那。 +ok，请稍等！ +create_sub_task +{"goal":"On(Coffee,BrightTable6)"} + + +麻烦啦！ +没事儿，为您服务是我的荣幸！ 来杯酸奶吧。 好的没问题，请稍等！ diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py index a36ded3..a30fd63 100644 --- a/robowaiter/scene/scene.py +++ b/robowaiter/scene/scene.py @@ -585,6 +585,9 @@ class Scene: obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=5), # GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9), GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=9), + + GrabSim_pb2.ObjectList.Object(x=340, y=960, z=88, roll=0, pitch=0, yaw=0, type=9), + ] scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID)) time.sleep(1.0) diff --git a/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py new file mode 100644 index 0000000..ff71316 --- /dev/null +++ b/robowaiter/scene/tasks/CafeDailyOperations/VLN_greet_and_order.py @@ -0,0 +1,86 @@ +""" +视觉语言操作 +机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板 +""" + +import time +from robowaiter.scene.scene import Scene + + + +class SceneVLM(Scene): + + def __init__(self, robot): + super().__init__(robot) + # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) + + self.scene_flag = 1 + self.st1 = 3 + self.st2 = self.st1 + 30 + + self.new_event_list = [ + + # 场景1：带小女孩找阳光下的空位 + (3, self.add_walker, (5, 230, 1200)), + (3, self.control_walker, (0, False, 200, 60, 520, 0)), + (10, self.customer_say, (0, "早上好呀，我想找个能晒太阳的地方。")), + (10, self.customer_say, (0,"可以带我过去嘛？")), + (20, self.control_walker, (0, False, 50, 140, 1200, 180)), # 小女孩站在了 BrightTable1 旁边就餐啦 + + # 场景2：有个胖胖男人点单交流并要咖啡，帮他送去角落的桌子 + # (3, self.add_walker, (5, 230, 1200)), # 小女孩 + # 上述准备 + (self.st2, self.add_walker, (26, 60,-140)), + (self.st2, self.add_walker, (10, -80, -180,-45)), # （-150 -250） + (self.st2, self.control_walker, (1, False, 100, 70, 520, 0)), #3 + (self.st2+15, self.customer_say, (1, "你们这有什么饮料嘛？")), # 10 + (self.st2+15, self.customer_say, (1, "咖啡有哪些呢？")),# 10 + (self.st2+15+5, self.customer_say, (1,"来杯卡布奇诺吧，我在靠窗边的高脚桌那。")), # 15 + (self.st2+15+5+5, self.control_walkers_and_say, ([[[1, False, 100, -30, -200, -90, "麻烦啦！"]]])), # 20 胖胖男到了 BrightTable6 + + + + + ] + + def _reset(self): + self.gen_obj() + # self.add_walkers([[47, 920]]) + pass + + def _run(self, op_type=10): + # 一个行人从门口走到 吧台 + # 打招呼需要什么 + # 行人说 哪里有位置，想晒个太阳 + # 带领行人去有太阳的地方 + # 行人说 有点热 + # 好的，这就去开空调 + self.walker_followed = False + pass + + def _step(self): + + + if self.scene_flag == 1: + # 如果机器人不在 吧台 + if self.walker_followed: + return + end = [self.status.location.X, self.status.location.Y] + if end[1]>=600 or end[1]<=450 or end[0]>=250: + # if int(self.status.location.X)!=247 or int(self.status.location.X)!=520: + self.walker_followed = True + self.control_walkers_and_say([[0,False,150,end[0],end[1],90,"wd！"]]) + self.scene_flag += 1 + + pass + +if __name__ == '__main__': + import os + from robowaiter.robot.robot import Robot + + robot = Robot() + + # create task + task = SceneVLM(robot) + task.reset() + task.run() diff --git a/robowaiter/scene/tasks/VLM/VLN_greet_lead.py b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py index 251d302..6eb8d3e 100644 --- a/robowaiter/scene/tasks/VLM/VLN_greet_lead.py +++ b/robowaiter/scene/tasks/VLM/VLN_greet_lead.py @@ -11,9 +11,8 @@ class SceneVLM(Scene): super().__init__(robot) # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) self.new_event_list = [ - (3, self.add_walker, (5, 230, 1200)), - (5, self.control_walkers_and_say, ([[[0, False, 200, 60, 520, 0, "早上好呀，我想找个能晒太阳的地方。"]]])),# (0, 60, 520)), - (6, self.customer_say, (0,"可以带我过去嘛？")), + (3, self.add_walker, (0,60,520)), + (5, self.customer_say, (0,"可以带我去空位上嘛？我想晒太阳。")), ] def _reset(self):