From 229e81370054fc3e503eeb677a6f5f5ace5b6446 Mon Sep 17 00:00:00 2001 From: Caiyishuai <39987654+Caiyishuai@users.noreply.github.com> Date: Thu, 23 Nov 2023 12:05:49 +0800 Subject: [PATCH] =?UTF-8?q?=E5=9C=A8UI=E4=B8=AD=E8=A1=A5=E5=85=85=E4=BA=86?= =?UTF-8?q?=E5=87=A0=E4=B8=AA=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- robowaiter/llm_client/data/fix_questions.txt | 6 + robowaiter/proto/camera.py | 20 +-- robowaiter/scene/scene.py | 11 +- robowaiter/scene/ui/scene_ui.py | 137 ++++++++++++++++++- tasks_no_ui/VLM/VLM_1_order.py | 2 +- 6 files changed, 162 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index d8f6b72..62ec602 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ pip install -e . ### 安装UI 1. 安装 [graphviz-9.0.0](https://gitlab.com/api/v4/projects/4207231/packages/generic/graphviz-releases/9.0.0/windows_10_cmake_Release_graphviz-install-9.0.0-win64.exe) (详见[官网](https://www.graphviz.org/download/#windows)) - +2. 将软件安装目录的bin文件添加到系统环境中。如电脑是Windows系统，Graphviz安装在D:\Program Files (x86)\Graphviz2.38，该目录下有bin文件，将该路径添加到电脑系统环境变量path中，即D:\Program Files (x86)\Graphviz2.38\bin。 ### 快速入门 1. 安装UE及Harix插件，打开默认项目并运行 diff --git a/robowaiter/llm_client/data/fix_questions.txt b/robowaiter/llm_client/data/fix_questions.txt index f6da6eb..372673f 100644 --- a/robowaiter/llm_client/data/fix_questions.txt +++ b/robowaiter/llm_client/data/fix_questions.txt @@ -112,6 +112,12 @@ get_object_info 给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。 好的，请稍等。 create_sub_task +{"goal":"On(Chips,WaterTable),On(NFCJuice,WaterTable)"} + + +给我来份薯片和果汁，我坐在对面的桌子那儿。 +好的，请稍等。 +create_sub_task {"goal":"On(BottledDrink,WaterTable),On(Yogurt,WaterTable)"} diff --git a/robowaiter/proto/camera.py b/robowaiter/proto/camera.py index cb73d19..8161f06 100644 --- a/robowaiter/proto/camera.py +++ b/robowaiter/proto/camera.py @@ -342,7 +342,8 @@ def save_obj_info(img_data, objs_name): return objs_name -def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio): +# def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio): +def get_obstacle_point(sence, db, scene, cur_obstacle_world_points, map_ratio): cur_obstacle_pixel_points = [] object_pixels = {} obj_detect_count = 0 @@ -380,10 +381,11 @@ def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio): objs_id[251] = "walker" # plt.imshow(d_depth, cmap="gray" if "depth" in im_depth.name.lower() else None) # plt.show() - plt.subplot(2, 2, 1) + # plt.subplot(2, 2, 1) plt.imshow(d_segment, cmap="gray" if "depth" in im_segment.name.lower() else None) plt.axis("off") - plt.title("相机分割") + # plt.title("相机分割") + sence.send_img("img_label_seg") # plt.show() d_depth = np.transpose(d_depth, (1, 0, 2)) @@ -416,10 +418,11 @@ def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio): world_point = transform_co(img_data_depth, pixel[0], pixel[1], d_depth[pixel[0]][pixel[1]][0], scene) cur_obstacle_world_points.append([world_point[0], world_point[1]]) # print(f"{pixel}：{[world_point[0], world_point[1]]}") - plt.subplot(2, 2, 2) + # plt.subplot(2, 2, 2) plt.imshow(d_color, cmap="gray" if "depth" in im_depth.name.lower() else None) plt.axis('off') - plt.title("目标检测") + # plt.title("目标检测") + # sence.send_img("img_label_obj") # plt.tight_layout() for key, value in object_pixels.items(): @@ -472,10 +475,9 @@ def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio): # height = point2[0] - point1[0] # rect = patches.Rectangle((0, 255), 15, 30, linewidth=1, edgecolor='g', # facecolor='none') - - plt.subplot(2, 7, 14) # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图 - - plt.text(0, 0.7, f'检测物体数量：{obj_detect_count}', fontsize=10) + sence.send_img("img_label_obj") + # plt.subplot(2, 7, 14) # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图 + # plt.text(0, 0.7, f'检测物体数量：{obj_detect_count}', fontsize=10) # plt.show() return cur_obstacle_world_points, cur_objs_id diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py index 8e2b2b9..bb3294b 100644 --- a/robowaiter/scene/scene.py +++ b/robowaiter/scene/scene.py @@ -122,7 +122,7 @@ class Scene: # 是否展示UI self.show_ui = False # 图像分割 - + self.take_picture = False self.map_ratio = 5 self.map_map = np.zeros((math.ceil(950 / self.map_ratio), math.ceil(1850 / self.map_ratio))) self.db = DBSCAN(eps=self.map_ratio, min_samples=int(self.map_ratio / 2)) @@ -1035,8 +1035,13 @@ class Scene: # cur_objs, objs_name_set = camera.get_semantic_map(GrabSim_pb2.CameraName.Head_Segment, cur_objs, # objs_name_set) - cur_obstacle_world_points, cur_objs_id = camera.get_obstacle_point(plt, db, scene, + # cur_obstacle_world_points, cur_objs_id = camera.get_obstacle_point(plt, db, scene, + # cur_obstacle_world_points, map_ratio) + cur_obstacle_world_points, cur_objs_id = camera.get_obstacle_point(self, db, scene, cur_obstacle_world_points, map_ratio) + # cur_obstacle_world_points, cur_objs_id = self.get_obstacle_point(db, scene, map_ratio) + # # self.get_obstacle_point(db, scene, cur_obstacle_world_points, map_ratio) + # if scene.info == "Unreachable": print(scene.info) @@ -1433,8 +1438,6 @@ class Scene: self.send_img("img_label_obj") - - new_map = self.updateMap(cur_obstacle_world_points) self.draw_map(plt,new_map) diff --git a/robowaiter/scene/ui/scene_ui.py b/robowaiter/scene/ui/scene_ui.py index b9b4308..765cdec 100644 --- a/robowaiter/scene/ui/scene_ui.py +++ b/robowaiter/scene/ui/scene_ui.py @@ -2,10 +2,23 @@ UI场景 """ import sys +import json +import math +from matplotlib import pyplot as plt +from sklearn.cluster import DBSCAN +import pickle +import time +import os +plt.rcParams['font.sans-serif'] = ['SimHei'] # 用来正常显示中文标签 +plt.rcParams['axes.unicode_minus'] = False # 用来正常显示负号 +from robowaiter.utils import get_root_path +root_path = get_root_path() from robowaiter.scene.scene import Scene from robowaiter.utils.bt.draw import render_dot_tree + + class SceneUI(Scene): scene_queue = None ui_queue = None @@ -33,9 +46,89 @@ class SceneUI(Scene): while not self.stoped: self.step() - def run_AEM(self): + def _run(self): pass + def run_AEM(self): + print(len(self.status.objects)) + # 创建一个从白色（1）到灰色（0）的 colormap + objs = self.status.objects + cur_objs = [] + cur_obstacle_world_points = [] + visited_obstacle = set() + obj_json_data = [] + obj_count = 0 + added_info = 0 + map_ratio = self.map_ratio + db = DBSCAN(eps=map_ratio, min_samples=int(map_ratio / 2)) + file_name = os.path.join(root_path, 'robowaiter/proto/map_1.pkl') + if os.path.exists(file_name): + with open(file_name, 'rb') as file: + map = pickle.load(file) + print('------------ 自主探索 ------------') + while True: + walker_count = 0 + fig = plt.figure() + goal = self.explore(map, 120) + if goal is None: + break + # cur_obstacle_world_points, cur_objs_id = self.navigation_move(plt, cur_objs, cur_obstacle_world_points, + # [[goal[0], goal[1]]], map_ratio, db, 0, 11) + cur_obstacle_world_points, cur_objs_id = self.navigation_move(self, cur_objs, cur_obstacle_world_points, + [[goal[0], goal[1]]], map_ratio, db, 0, 11) + for point in cur_obstacle_world_points: + if point[0] < -350 or point[0] > 600 or point[1] < -400 or point[1] > 1450: + continue + self.map_map[math.floor((point[0] + 350) / map_ratio), math.floor((point[1] + 400) / map_ratio)] = 1 + visited_obstacle.add( + (math.floor((point[0] + 350) / map_ratio), math.floor((point[1] + 400) / map_ratio))) + for i in range(len(cur_objs_id)): + if cur_objs_id[i] == "walker": + walker_count += 1 + for obj in objs: + if obj.name == cur_objs_id[i] and obj not in cur_objs: + cur_objs.append(obj) + break + # plt.subplot(2, 1, 2) # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图 + # plt.imshow(self.map_map, cmap='binary', alpha=0.5, origin='lower', + # extent=(-400 / map_ratio, 1450 / map_ratio, + # -350 / map_ratio, 600 / map_ratio)) + new_map = self.updateMap(cur_obstacle_world_points) + self.draw_map(plt, new_map) + plt.axis("off") + self.send_img("img_label_map") + # plt.title("地图构建过程") + # self.send_img("img_label_map") + # plt.subplot(2, 7, 14) # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图 + new_add_info = len(cur_objs) - added_info + walker_count + # plt.text(0, 0.5, f'新增语义信息：{new_add_info}', fontsize=10) # 在图中添加文字，x和y坐标是在这个图片大小内的相对位置，fontsize是字体大小 + added_info += new_add_info + # plt.text(0, 0.3, f'已存语义信息：{added_info}', fontsize=10) # 在图中添加文字，x和y坐标是在这个图片大小内的相对位置，fontsize是字体大小 + self.infoCount = added_info + plt.axis("off") + # plt.show() + print("------------当前检测到的物品信息--------------") + print(cur_objs) + time.sleep(1) + + for i in range(len(cur_objs)): + if cur_objs[i].name == "Desk" or cur_objs[i].name == "Chair": + obj_json_data.append( + {"id": f"{i}", "name": f"{cur_objs[i].name}", "location": f"{cur_objs[i].location}", + "height": f"{cur_objs[i].location.Z * 2}"}) + + else: + obj_json_data.append( + {"id": f"{i}", "name": f"{cur_objs[i].name}", "location": f"{cur_objs[i].location}", + "height": f"{cur_objs[i].location.Z}"}) + + with open('../../robowaiter/proto/objs.json', 'w') as file: + json.dump(obj_json_data, file) + + print("已绘制完成地图！！！") + print("------------检测到的所有物品信息--------------") + print(obj_json_data) + def run_VLN(self): self.gen_obj() self.add_walkers([ @@ -66,15 +159,57 @@ class SceneUI(Scene): ] def run_VLM(self): + self.gen_obj() + self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]]) + self. control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True) + self.signal_event_list = [ + (3, self.add_walker, (20,0,700)), + (1, self.control_walker, (6, False,100, 60, 520,0)), + (1, self.customer_say, (6, "给我来份薯片和果汁，我坐在对面的桌子那儿。")), + (5, self.control_walker, (6, False, 100, -250, 480, 0)), + ] pass def run_GQA(self): + self.gen_obj() + self.add_walkers([ [16,250, 1200],[6,-55, 750],[10,70, -200],[47,-290, 400, 180],[26, 60,-320,90]]) + self.control_walker(1, True, 100, 60, 720, 0) + self.control_walker(4, True, 100, 60, -120, 0) + self.add_walkers([[31, 60,500,0], [15,60,550,0]]) + self.signal_event_list = [ + (5, self.customer_say, (6, "你好呀，你们这有啥好吃的？")), # 男 + (8, self.customer_say, (6, "听起来都好甜呀，我女朋友爱吃水果。")), + (15, self.customer_say, (6, "你们这人可真多。")), + (15, self.customer_say, (6, "我女朋友怕晒，有空余的阴凉位置嘛？")), + (20, self.customer_say, (6, "那还不错。")), + (15, self.customer_say, (5, "请问洗手间在哪呢？")), + (20, self.customer_say, (5, "我们还想一起下下棋,切磋切磋。")), + (20, self.customer_say, (6, "太棒啦，亲爱的。")), + (15, self.customer_say, (5, "那你知道附近最近的电影院在哪吗?")), + (20, self.customer_say, (6, "谢啦，那我们先去阴凉位置下个棋，等电影开始了就去看呢!")), + ] pass def run_OT(self): + self.gen_obj() + self.add_walkers([ [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]]) + self.control_walker(1, True, 100, 60, 720, 0) + self.control_walker(4, True, 100, 60, -120, 0) + self.add_walkers([[16,60, 520], [47,-40, 520]]) + self.signal_event_list = [ + (8, self.customer_say, (5, "给我来杯咖啡，哦对，再倒一杯水。")), + (1, self.control_walker_ls,([[[5, False, 100, -250, 480, 0],[6, False, 100, 60, 520, 0]]])), + (-1, self.customer_say, (5, "感谢，这些够啦，你去忙吧。")), + (10, self.customer_say, (6, "我想来份点心和酸奶。")), + (-1, self.customer_say, (6, "真美味啊！")), + ] pass def run_AT(self): + self.add_walker(23, 60, 520, 0) + self.signal_event_list = [ + (2, self.customer_say, (0,"可以关筒灯和关窗帘吗？")), + ] pass def run_reset(self): diff --git a/tasks_no_ui/VLM/VLM_1_order.py b/tasks_no_ui/VLM/VLM_1_order.py index 337ccac..4e8f5e5 100644 --- a/tasks_no_ui/VLM/VLM_1_order.py +++ b/tasks_no_ui/VLM/VLM_1_order.py @@ -13,7 +13,7 @@ class SceneVLM(Scene): self.signal_event_list = [ (3, self.add_walker, (20,0,700)), (1, self.control_walker, (6, False,100, 60, 520,0)), - (1, self.customer_say, (6, "给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。")), + (1, self.customer_say, (6, "给我来份薯片和果汁，我坐在对面的桌子那儿。")), #给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。 (5, self.control_walker, (6, False, 100, -250, 480, 0)), ]