From 0e55e18cbf91ae390edc7e25eea6675d7028d80f Mon Sep 17 00:00:00 2001 From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com> Date: Wed, 15 Nov 2023 14:30:55 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BA=86=E5=BC=80=E6=94=BE?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=92=8C=20VLM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- robowaiter/algos/navigate/test.py | 5 +- robowaiter/llm_client/data/test_questions.txt | 2 +- .../llm_client/data_raw/test_questions.csv | 1 + robowaiter/scene/tasks/Auto_tasks.py | 18 ++++++- robowaiter/scene/tasks/Open_tasks.py | 11 +++-- robowaiter/scene/tasks/VLM.py | 48 +++++++++---------- 6 files changed, 55 insertions(+), 30 deletions(-) diff --git a/robowaiter/algos/navigate/test.py b/robowaiter/algos/navigate/test.py index 525ad10..c8c4d91 100644 --- a/robowaiter/algos/navigate/test.py +++ b/robowaiter/algos/navigate/test.py @@ -44,9 +44,12 @@ if __name__ == '__main__': scene.control_walker([scene.walker_control_generator(walkerID=1, autowalk=False, speed=50, X=100, Y=150, Yaw=0)]) scene.control_walker([scene.walker_control_generator(walkerID=2, autowalk=False, speed=50, X=0, Y=0, Yaw=0)]) - goal = (-100, 700) + # goal = (-100, 700) # goal = (-300) + # goal = (340.0, 900.0) + goal = (240.0, 1160.0) + '''场景4: 行人自由移动 robot到达指定目标''' # # TODO: autowalk=True仿真器会闪退 ??? # scene.clean_walker() diff --git a/robowaiter/llm_client/data/test_questions.txt b/robowaiter/llm_client/data/test_questions.txt index cc0bb66..22b4854 100644 --- a/robowaiter/llm_client/data/test_questions.txt +++ b/robowaiter/llm_client/data/test_questions.txt @@ -1 +1 @@ -{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "测试VLM：做一杯咖啡放到吧台上": {"Answer": "测试VLM：做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "测试VLN：前往2号桌": {"Answer": "测试VLN：前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关空调": {"Answer": "测试VLM：关空调", "Goal": "{\"Is(AC,Off)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：开大厅灯": {"Answer": "测试VLM：开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：开筒灯": {"Answer": "测试VLM：开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：开窗帘": {"Answer": "测试VLM：开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：整理椅子": {"Answer": "测试VLM：整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}} +{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "测试VLM：做一杯咖啡放到吧台上": {"Answer": "测试VLM：做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "测试VLN：前往2号桌": {"Answer": "测试VLN：前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关空调": {"Answer": "测试VLM：关空调", "Goal": "{\"Is(AC,Off)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：开大厅灯": {"Answer": "测试VLM：开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：开筒灯": {"Answer": "测试VLM：开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：开窗帘": {"Answer": "测试VLM：开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：整理椅子": {"Answer": "测试VLM：整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}} diff --git a/robowaiter/llm_client/data_raw/test_questions.csv b/robowaiter/llm_client/data_raw/test_questions.csv index 1a7573a..694d30c 100644 --- a/robowaiter/llm_client/data_raw/test_questions.csv +++ b/robowaiter/llm_client/data_raw/test_questions.csv @@ -16,3 +16,4 @@ Question,Answer,Goal VLM,VLM,"{""Is(Table1,Clean)""}" VLM,VLM,"{""Is(Chairs,Clean)""}" VLMѱŵTable2,VLMѱŵTable2,"{""On(BottledDrink,Table2)""}" +еȣܿյ,ȻԣھͿ,"{""Is(AC,On)""}" diff --git a/robowaiter/scene/tasks/Auto_tasks.py b/robowaiter/scene/tasks/Auto_tasks.py index 3a5bf43..b926cf2 100644 --- a/robowaiter/scene/tasks/Auto_tasks.py +++ b/robowaiter/scene/tasks/Auto_tasks.py @@ -20,7 +20,23 @@ class SceneAT(Scene): self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)]) def _run(self): + + self.move_task_area(7) + self.op_task_execute(7) + self.move_task_area(6) + self.op_task_execute(6) pass def _step(self): - pass \ No newline at end of file + pass + +if __name__ == '__main__': + import os + from robowaiter.robot.robot import Robot + + robot = Robot() + + # create task + task = SceneAT(robot) + task.reset() + task.run() diff --git a/robowaiter/scene/tasks/Open_tasks.py b/robowaiter/scene/tasks/Open_tasks.py index ca49491..ba3c8f3 100644 --- a/robowaiter/scene/tasks/Open_tasks.py +++ b/robowaiter/scene/tasks/Open_tasks.py @@ -19,16 +19,21 @@ class SceneOT(Scene): # 在这里加入场景中发生的事件 self.event_list = [ # (5,self.create_chat_event("给我一杯咖啡")) # (事件发生的时间，事件函数) - (5, self.create_chat_event("测试VLM：拖地")) # (事件发生的时间，事件函数) + (5, self.create_chat_event("我有点热，能开个空调吗？")) # (事件发生的时间，事件函数) ] def _reset(self): - pass - # self.add_walker(1085, 2630, 220) + scene = self.add_walker(0, 0, 0) + print("我有点热，能开个空调吗？") + # print("scene.walkers:",scene.walkers) + # cont = scene.walkers[0].name+":我有点热，能开个空调吗？" + # self.control_robot_action(0,3,cont) # self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)]) + pass def _run(self): + pass diff --git a/robowaiter/scene/tasks/VLM.py b/robowaiter/scene/tasks/VLM.py index 98b6a23..19938fc 100644 --- a/robowaiter/scene/tasks/VLM.py +++ b/robowaiter/scene/tasks/VLM.py @@ -13,7 +13,7 @@ class SceneVLM(Scene): self.event_list = [ # (5, self.create_chat_event("测试VLM：做一杯咖啡")), # (5, self.create_chat_event("测试VLM：倒一杯水")), - # (5, self.create_chat_event("测试VLM：开空调")), + (5, self.create_chat_event("测试VLM：开空调")), # (5, self.create_chat_event("测试VLM：关空调")), # (5, self.create_chat_event("测试VLM：开大厅灯")), # (5, self.create_chat_event("测试VLM：拖地")), @@ -22,7 +22,7 @@ class SceneVLM(Scene): # (5, self.create_chat_event("测试VLM：把冰红茶放到Table2")), # (5, self.create_chat_event("测试VLM：关大厅灯")) - (5, self.create_chat_event("测试VLM：做一杯咖啡放到吧台上")), + # (5, self.create_chat_event("测试VLM：做一杯咖啡放到吧台上")), ] def _reset(self): @@ -57,28 +57,28 @@ class SceneVLM(Scene): # 流程测试 # 抓握放置:抓吧台前生成的酸奶，放到抹布桌上 - # self.gen_obj() - # self.move_task_area(16, obj_id=0) - # self.op_task_execute(16, obj_id=0) - # pos = [340.0, 900.0, 99.0] - # self.move_task_area(17, release_pos=pos) - # self.op_task_execute(17, release_pos=pos) - # - # # 做咖啡:做完的咖啡放到水杯桌上 - # self.move_task_area(1) - # self.op_task_execute(1) - # - # self.find_obj("CoffeeCup") - # - # self.move_task_area(16, obj_id=275) - # self.op_task_execute(16, obj_id=275) - # pos = [-70.0, 500.0, 107] - # self.move_task_area(17, release_pos=pos) - # self.op_task_execute(17, release_pos=pos) - # - # # 倒水:倒完的水放到旁边桌子上 - # self.move_task_area(2) - # self.op_task_execute(2) + self.gen_obj() + self.move_task_area(16, obj_id=0) + self.op_task_execute(16, obj_id=0) + pos = [340.0, 900.0, 99.0] + self.move_task_area(17, release_pos=pos) + self.op_task_execute(17, release_pos=pos) + + # 做咖啡:做完的咖啡放到水杯桌上 + self.move_task_area(1) + self.op_task_execute(1) + + self.find_obj("CoffeeCup") + + self.move_task_area(16, obj_id=275) + self.op_task_execute(16, obj_id=275) + pos = [-70.0, 500.0, 107] + self.move_task_area(17, release_pos=pos) + self.op_task_execute(17, release_pos=pos) + + # 倒水:倒完的水放到旁边桌子上 + self.move_task_area(2) + self.op_task_execute(2) # # self.move_task_area(16, obj_id=190) # self.op_task_execute(16, obj_id=190)