在UI中补充了几个场景

README.md

@@ -14,7 +14,7 @@ pip install -e .
 
 ### 安装UI
 1. 安装 [graphviz-9.0.0](https://gitlab.com/api/v4/projects/4207231/packages/generic/graphviz-releases/9.0.0/windows_10_cmake_Release_graphviz-install-9.0.0-win64.exe) (详见[官网](https://www.graphviz.org/download/#windows))
-
+2. 将软件安装目录的bin文件添加到系统环境中。如电脑是Windows系统，Graphviz安装在D:\Program Files (x86)\Graphviz2.38，该目录下有bin文件，将该路径添加到电脑系统环境变量path中，即D:\Program Files (x86)\Graphviz2.38\bin。
 
 ### 快速入门
 1. 安装UE及Harix插件，打开默认项目并运行

robowaiter/llm_client/data/fix_questions.txt

@@ -112,6 +112,12 @@ get_object_info
 给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。
 好的，请稍等。
 create_sub_task
+{"goal":"On(Chips,WaterTable),On(NFCJuice,WaterTable)"}
+
+
+给我来份薯片和果汁，我坐在对面的桌子那儿。
+好的，请稍等。
+create_sub_task
 {"goal":"On(BottledDrink,WaterTable),On(Yogurt,WaterTable)"}
 
 

robowaiter/proto/camera.py

@@ -342,7 +342,8 @@ def save_obj_info(img_data, objs_name):
     return objs_name
 
 
-def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
+# def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
+def get_obstacle_point(sence, db, scene, cur_obstacle_world_points, map_ratio):
     cur_obstacle_pixel_points = []
     object_pixels = {}
     obj_detect_count = 0
@@ -380,10 +381,11 @@ def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
     objs_id[251] = "walker"
     # plt.imshow(d_depth, cmap="gray" if "depth" in im_depth.name.lower() else None)
     # plt.show()
-    plt.subplot(2, 2, 1)
+    # plt.subplot(2, 2, 1)
     plt.imshow(d_segment, cmap="gray" if "depth" in im_segment.name.lower() else None)
     plt.axis("off")
-    plt.title("相机分割")
+    # plt.title("相机分割")
+    sence.send_img("img_label_seg")
     # plt.show()
 
     d_depth = np.transpose(d_depth, (1, 0, 2))
@@ -416,10 +418,11 @@ def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
         world_point = transform_co(img_data_depth, pixel[0], pixel[1], d_depth[pixel[0]][pixel[1]][0], scene)
         cur_obstacle_world_points.append([world_point[0], world_point[1]])
         # print(f"{pixel}：{[world_point[0], world_point[1]]}")
-    plt.subplot(2, 2, 2)
+    # plt.subplot(2, 2, 2)
     plt.imshow(d_color, cmap="gray" if "depth" in im_depth.name.lower() else None)
     plt.axis('off')
-    plt.title("目标检测")
+    # plt.title("目标检测")
+    # sence.send_img("img_label_obj")
     # plt.tight_layout()
 
     for key, value in object_pixels.items():
@@ -472,10 +475,9 @@ def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
         # height = point2[0] - point1[0]
         # rect = patches.Rectangle((0, 255), 15, 30, linewidth=1, edgecolor='g',
         #                          facecolor='none')
-
+    sence.send_img("img_label_obj")
-    plt.subplot(2, 7, 14)  # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图
+    # plt.subplot(2, 7, 14)  # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图
-
+    # plt.text(0, 0.7, f'检测物体数量：{obj_detect_count}', fontsize=10)
-    plt.text(0, 0.7, f'检测物体数量：{obj_detect_count}', fontsize=10)
 
     # plt.show()
     return cur_obstacle_world_points, cur_objs_id

robowaiter/scene/scene.py

@@ -122,7 +122,7 @@ class Scene:
         # 是否展示UI
         self.show_ui = False
         # 图像分割
-
+        self.take_picture = False
         self.map_ratio = 5
         self.map_map = np.zeros((math.ceil(950 / self.map_ratio), math.ceil(1850 / self.map_ratio)))
         self.db = DBSCAN(eps=self.map_ratio, min_samples=int(self.map_ratio / 2))
@@ -1035,8 +1035,13 @@ class Scene:
             # cur_objs, objs_name_set = camera.get_semantic_map(GrabSim_pb2.CameraName.Head_Segment, cur_objs,
             #                                                   objs_name_set)
 
-            cur_obstacle_world_points, cur_objs_id = camera.get_obstacle_point(plt, db, scene,
+            # cur_obstacle_world_points, cur_objs_id = camera.get_obstacle_point(plt, db, scene,
+            #                                                                    cur_obstacle_world_points, map_ratio)
+            cur_obstacle_world_points, cur_objs_id = camera.get_obstacle_point(self, db, scene,
                                                                                cur_obstacle_world_points, map_ratio)
+            # cur_obstacle_world_points, cur_objs_id = self.get_obstacle_point(db, scene, map_ratio)
+            # # self.get_obstacle_point(db, scene, cur_obstacle_world_points, map_ratio)
+
 
             # if scene.info == "Unreachable":
             print(scene.info)
@@ -1433,8 +1438,6 @@ class Scene:
         self.send_img("img_label_obj")
 
 
-
-
         new_map = self.updateMap(cur_obstacle_world_points)
         self.draw_map(plt,new_map)
 

robowaiter/scene/ui/scene_ui.py

@@ -2,10 +2,23 @@
 UI场景
 """
 import sys
+import json
+import math
+from matplotlib import pyplot as plt
+from sklearn.cluster import DBSCAN
+import pickle
+import time
+import os
+plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
+plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
 
+from robowaiter.utils import get_root_path
+root_path = get_root_path()
 
 from robowaiter.scene.scene import Scene
 from robowaiter.utils.bt.draw import render_dot_tree
+
+
 class SceneUI(Scene):
     scene_queue = None
     ui_queue = None
@@ -33,9 +46,89 @@ class SceneUI(Scene):
         while not self.stoped:
             self.step()
 
-    def run_AEM(self):
+    def _run(self):
         pass
 
+    def run_AEM(self):
+        print(len(self.status.objects))
+        # 创建一个从白色（1）到灰色（0）的 colormap
+        objs = self.status.objects
+        cur_objs = []
+        cur_obstacle_world_points = []
+        visited_obstacle = set()
+        obj_json_data = []
+        obj_count = 0
+        added_info = 0
+        map_ratio = self.map_ratio
+        db = DBSCAN(eps=map_ratio, min_samples=int(map_ratio / 2))
+        file_name = os.path.join(root_path, 'robowaiter/proto/map_1.pkl')
+        if os.path.exists(file_name):
+            with open(file_name, 'rb') as file:
+                map = pickle.load(file)
+        print('------------ 自主探索 ------------')
+        while True:
+            walker_count = 0
+            fig = plt.figure()
+            goal = self.explore(map, 120)
+            if goal is None:
+                break
+            # cur_obstacle_world_points, cur_objs_id = self.navigation_move(plt, cur_objs, cur_obstacle_world_points,
+            #                                                               [[goal[0], goal[1]]], map_ratio, db, 0, 11)
+            cur_obstacle_world_points, cur_objs_id = self.navigation_move(self, cur_objs, cur_obstacle_world_points,
+                                                                          [[goal[0], goal[1]]], map_ratio, db, 0, 11)
+            for point in cur_obstacle_world_points:
+                if point[0] < -350 or point[0] > 600 or point[1] < -400 or point[1] > 1450:
+                    continue
+                self.map_map[math.floor((point[0] + 350) / map_ratio), math.floor((point[1] + 400) / map_ratio)] = 1
+                visited_obstacle.add(
+                    (math.floor((point[0] + 350) / map_ratio), math.floor((point[1] + 400) / map_ratio)))
+            for i in range(len(cur_objs_id)):
+                if cur_objs_id[i] == "walker":
+                    walker_count += 1
+                for obj in objs:
+                    if obj.name == cur_objs_id[i] and obj not in cur_objs:
+                        cur_objs.append(obj)
+                        break
+            # plt.subplot(2, 1, 2)  # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图
+            # plt.imshow(self.map_map, cmap='binary', alpha=0.5, origin='lower',
+            #            extent=(-400 / map_ratio, 1450 / map_ratio,
+            #                    -350 / map_ratio, 600 / map_ratio))
+            new_map = self.updateMap(cur_obstacle_world_points)
+            self.draw_map(plt, new_map)
+            plt.axis("off")
+            self.send_img("img_label_map")
+            # plt.title("地图构建过程")
+            # self.send_img("img_label_map")
+            # plt.subplot(2, 7, 14)  # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图
+            new_add_info = len(cur_objs) - added_info + walker_count
+            # plt.text(0, 0.5, f'新增语义信息：{new_add_info}', fontsize=10)  # 在图中添加文字，x和y坐标是在这个图片大小内的相对位置，fontsize是字体大小
+            added_info += new_add_info
+            # plt.text(0, 0.3, f'已存语义信息：{added_info}', fontsize=10)  # 在图中添加文字，x和y坐标是在这个图片大小内的相对位置，fontsize是字体大小
+            self.infoCount = added_info
+            plt.axis("off")
+            # plt.show()
+            print("------------当前检测到的物品信息--------------")
+            print(cur_objs)
+            time.sleep(1)
+
+        for i in range(len(cur_objs)):
+            if cur_objs[i].name == "Desk" or cur_objs[i].name == "Chair":
+                obj_json_data.append(
+                    {"id": f"{i}", "name": f"{cur_objs[i].name}", "location": f"{cur_objs[i].location}",
+                     "height": f"{cur_objs[i].location.Z * 2}"})
+
+            else:
+                obj_json_data.append(
+                    {"id": f"{i}", "name": f"{cur_objs[i].name}", "location": f"{cur_objs[i].location}",
+                     "height": f"{cur_objs[i].location.Z}"})
+
+        with open('../../robowaiter/proto/objs.json', 'w') as file:
+            json.dump(obj_json_data, file)
+
+        print("已绘制完成地图！！！")
+        print("------------检测到的所有物品信息--------------")
+        print(obj_json_data)
+
     def run_VLN(self):
         self.gen_obj()
         self.add_walkers([
@@ -66,15 +159,57 @@ class SceneUI(Scene):
         ]
 
     def run_VLM(self):
+        self.gen_obj()
+        self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
+        self. control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
+        self.signal_event_list = [
+            (3, self.add_walker,  (20,0,700)),
+            (1, self.control_walker, (6, False,100, 60, 520,0)),
+            (1, self.customer_say, (6, "给我来份薯片和果汁，我坐在对面的桌子那儿。")),
+            (5, self.control_walker, (6, False, 100, -250, 480, 0)),
+        ]
         pass
 
     def run_GQA(self):
+        self.gen_obj()
+        self.add_walkers([ [16,250, 1200],[6,-55, 750],[10,70, -200],[47,-290, 400, 180],[26, 60,-320,90]])
+        self.control_walker(1, True, 100, 60, 720, 0)
+        self.control_walker(4, True, 100, 60, -120, 0)
+        self.add_walkers([[31, 60,500,0], [15,60,550,0]])
+        self.signal_event_list = [
+            (5, self.customer_say, (6, "你好呀，你们这有啥好吃的？")), # 男
+            (8, self.customer_say, (6, "听起来都好甜呀，我女朋友爱吃水果。")),
+            (15, self.customer_say, (6, "你们这人可真多。")),
+            (15, self.customer_say, (6, "我女朋友怕晒，有空余的阴凉位置嘛？")),
+            (20, self.customer_say, (6, "那还不错。")),
+            (15, self.customer_say, (5, "请问洗手间在哪呢？")),
+            (20, self.customer_say, (5, "我们还想一起下下棋,切磋切磋。")),
+            (20, self.customer_say, (6, "太棒啦，亲爱的。")),
+            (15, self.customer_say, (5, "那你知道附近最近的电影院在哪吗?")),
+            (20, self.customer_say, (6, "谢啦，那我们先去阴凉位置下个棋，等电影开始了就去看呢!")),
+        ]
         pass
 
     def run_OT(self):
+        self.gen_obj()
+        self.add_walkers([ [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
+        self.control_walker(1, True, 100, 60, 720, 0)
+        self.control_walker(4, True, 100, 60, -120, 0)
+        self.add_walkers([[16,60, 520], [47,-40, 520]])
+        self.signal_event_list = [
+            (8, self.customer_say, (5, "给我来杯咖啡，哦对，再倒一杯水。")),
+            (1, self.control_walker_ls,([[[5, False, 100, -250, 480, 0],[6, False, 100, 60, 520, 0]]])),
+            (-1, self.customer_say, (5, "感谢，这些够啦，你去忙吧。")),
+            (10, self.customer_say, (6, "我想来份点心和酸奶。")),
+            (-1, self.customer_say, (6, "真美味啊！")),
+        ]
         pass
 
     def run_AT(self):
+        self.add_walker(23, 60, 520, 0)
+        self.signal_event_list = [
+            (2, self.customer_say, (0,"可以关筒灯和关窗帘吗？")),
+        ]
         pass
 
     def run_reset(self):

tasks_no_ui/VLM/VLM_1_order.py

@@ -13,7 +13,7 @@ class SceneVLM(Scene):
         self.signal_event_list = [
             (3, self.add_walker,  (20,0,700)),
             (1, self.control_walker, (6, False,100, 60, 520,0)),
-            (1, self.customer_say, (6, "给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。")),
+            (1, self.customer_say, (6, "给我来份薯片和果汁，我坐在对面的桌子那儿。")), #给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。
             (5, self.control_walker, (6, False, 100, -250, 480, 0)),
         ]