在UI中补充了几个场景
This commit is contained in:
Caiyishuai
parent
0f5300d829
commit
229e813700
|
|
@ -14,7 +14,7 @@ pip install -e .
|
|||
|
||||
### 安装UI
|
||||
1. 安装 [graphviz-9.0.0](https://gitlab.com/api/v4/projects/4207231/packages/generic/graphviz-releases/9.0.0/windows_10_cmake_Release_graphviz-install-9.0.0-win64.exe) (详见[官网](https://www.graphviz.org/download/#windows))
|
||||
|
||||
2. 将软件安装目录的bin文件添加到系统环境中。如电脑是Windows系统，Graphviz安装在D:\Program Files (x86)\Graphviz2.38，该目录下有bin文件，将该路径添加到电脑系统环境变量path中，即D:\Program Files (x86)\Graphviz2.38\bin。
|
||||
|
||||
### 快速入门
|
||||
1. 安装UE及Harix插件，打开默认项目并运行
|
||||
|
|
|
|||
|
|
@ -112,6 +112,12 @@ get_object_info
|
|||
给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。
|
||||
好的，请稍等。
|
||||
create_sub_task
|
||||
{"goal":"On(Chips,WaterTable),On(NFCJuice,WaterTable)"}
|
||||
|
||||
|
||||
给我来份薯片和果汁，我坐在对面的桌子那儿。
|
||||
好的，请稍等。
|
||||
create_sub_task
|
||||
{"goal":"On(BottledDrink,WaterTable),On(Yogurt,WaterTable)"}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -342,7 +342,8 @@ def save_obj_info(img_data, objs_name):
|
|||
return objs_name
|
||||
|
||||
|
||||
def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
|
||||
# def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
|
||||
def get_obstacle_point(sence, db, scene, cur_obstacle_world_points, map_ratio):
|
||||
cur_obstacle_pixel_points = []
|
||||
object_pixels = {}
|
||||
obj_detect_count = 0
|
||||
|
|
@ -380,10 +381,11 @@ def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
|
|||
objs_id[251] = "walker"
|
||||
# plt.imshow(d_depth, cmap="gray" if "depth" in im_depth.name.lower() else None)
|
||||
# plt.show()
|
||||
plt.subplot(2, 2, 1)
|
||||
# plt.subplot(2, 2, 1)
|
||||
plt.imshow(d_segment, cmap="gray" if "depth" in im_segment.name.lower() else None)
|
||||
plt.axis("off")
|
||||
plt.title("相机分割")
|
||||
# plt.title("相机分割")
|
||||
sence.send_img("img_label_seg")
|
||||
# plt.show()
|
||||
|
||||
d_depth = np.transpose(d_depth, (1, 0, 2))
|
||||
|
|
@ -416,10 +418,11 @@ def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
|
|||
world_point = transform_co(img_data_depth, pixel[0], pixel[1], d_depth[pixel[0]][pixel[1]][0], scene)
|
||||
cur_obstacle_world_points.append([world_point[0], world_point[1]])
|
||||
# print(f"{pixel}：{[world_point[0], world_point[1]]}")
|
||||
plt.subplot(2, 2, 2)
|
||||
# plt.subplot(2, 2, 2)
|
||||
plt.imshow(d_color, cmap="gray" if "depth" in im_depth.name.lower() else None)
|
||||
plt.axis('off')
|
||||
plt.title("目标检测")
|
||||
# plt.title("目标检测")
|
||||
# sence.send_img("img_label_obj")
|
||||
# plt.tight_layout()
|
||||
|
||||
for key, value in object_pixels.items():
|
||||
|
|
@ -472,10 +475,9 @@ def get_obstacle_point(plt, db, scene, cur_obstacle_world_points, map_ratio):
|
|||
# height = point2[0] - point1[0]
|
||||
# rect = patches.Rectangle((0, 255), 15, 30, linewidth=1, edgecolor='g',
|
||||
# facecolor='none')
|
||||
|
||||
plt.subplot(2, 7, 14) # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图
|
||||
|
||||
plt.text(0, 0.7, f'检测物体数量：{obj_detect_count}', fontsize=10)
|
||||
sence.send_img("img_label_obj")
|
||||
# plt.subplot(2, 7, 14) # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图
|
||||
# plt.text(0, 0.7, f'检测物体数量：{obj_detect_count}', fontsize=10)
|
||||
|
||||
# plt.show()
|
||||
return cur_obstacle_world_points, cur_objs_id
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ class Scene:
|
|||
# 是否展示UI
|
||||
self.show_ui = False
|
||||
# 图像分割
|
||||
|
||||
self.take_picture = False
|
||||
self.map_ratio = 5
|
||||
self.map_map = np.zeros((math.ceil(950 / self.map_ratio), math.ceil(1850 / self.map_ratio)))
|
||||
self.db = DBSCAN(eps=self.map_ratio, min_samples=int(self.map_ratio / 2))
|
||||
|
|
@ -1035,8 +1035,13 @@ class Scene:
|
|||
# cur_objs, objs_name_set = camera.get_semantic_map(GrabSim_pb2.CameraName.Head_Segment, cur_objs,
|
||||
# objs_name_set)
|
||||
|
||||
cur_obstacle_world_points, cur_objs_id = camera.get_obstacle_point(plt, db, scene,
|
||||
# cur_obstacle_world_points, cur_objs_id = camera.get_obstacle_point(plt, db, scene,
|
||||
# cur_obstacle_world_points, map_ratio)
|
||||
cur_obstacle_world_points, cur_objs_id = camera.get_obstacle_point(self, db, scene,
|
||||
cur_obstacle_world_points, map_ratio)
|
||||
# cur_obstacle_world_points, cur_objs_id = self.get_obstacle_point(db, scene, map_ratio)
|
||||
# # self.get_obstacle_point(db, scene, cur_obstacle_world_points, map_ratio)
|
||||
|
||||
|
||||
# if scene.info == "Unreachable":
|
||||
print(scene.info)
|
||||
|
|
@ -1433,8 +1438,6 @@ class Scene:
|
|||
self.send_img("img_label_obj")
|
||||
|
||||
|
||||
|
||||
|
||||
new_map = self.updateMap(cur_obstacle_world_points)
|
||||
self.draw_map(plt,new_map)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,23 @@
|
|||
UI场景
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import math
|
||||
from matplotlib import pyplot as plt
|
||||
from sklearn.cluster import DBSCAN
|
||||
import pickle
|
||||
import time
|
||||
import os
|
||||
plt.rcParams['font.sans-serif'] = ['SimHei'] # 用来正常显示中文标签
|
||||
plt.rcParams['axes.unicode_minus'] = False # 用来正常显示负号
|
||||
|
||||
from robowaiter.utils import get_root_path
|
||||
root_path = get_root_path()
|
||||
|
||||
from robowaiter.scene.scene import Scene
|
||||
from robowaiter.utils.bt.draw import render_dot_tree
|
||||
|
||||
|
||||
class SceneUI(Scene):
|
||||
scene_queue = None
|
||||
ui_queue = None
|
||||
|
|
@ -33,9 +46,89 @@ class SceneUI(Scene):
|
|||
while not self.stoped:
|
||||
self.step()
|
||||
|
||||
def run_AEM(self):
|
||||
def _run(self):
|
||||
pass
|
||||
|
||||
def run_AEM(self):
|
||||
print(len(self.status.objects))
|
||||
# 创建一个从白色（1）到灰色（0）的 colormap
|
||||
objs = self.status.objects
|
||||
cur_objs = []
|
||||
cur_obstacle_world_points = []
|
||||
visited_obstacle = set()
|
||||
obj_json_data = []
|
||||
obj_count = 0
|
||||
added_info = 0
|
||||
map_ratio = self.map_ratio
|
||||
db = DBSCAN(eps=map_ratio, min_samples=int(map_ratio / 2))
|
||||
file_name = os.path.join(root_path, 'robowaiter/proto/map_1.pkl')
|
||||
if os.path.exists(file_name):
|
||||
with open(file_name, 'rb') as file:
|
||||
map = pickle.load(file)
|
||||
print('------------ 自主探索 ------------')
|
||||
while True:
|
||||
walker_count = 0
|
||||
fig = plt.figure()
|
||||
goal = self.explore(map, 120)
|
||||
if goal is None:
|
||||
break
|
||||
# cur_obstacle_world_points, cur_objs_id = self.navigation_move(plt, cur_objs, cur_obstacle_world_points,
|
||||
# [[goal[0], goal[1]]], map_ratio, db, 0, 11)
|
||||
cur_obstacle_world_points, cur_objs_id = self.navigation_move(self, cur_objs, cur_obstacle_world_points,
|
||||
[[goal[0], goal[1]]], map_ratio, db, 0, 11)
|
||||
for point in cur_obstacle_world_points:
|
||||
if point[0] < -350 or point[0] > 600 or point[1] < -400 or point[1] > 1450:
|
||||
continue
|
||||
self.map_map[math.floor((point[0] + 350) / map_ratio), math.floor((point[1] + 400) / map_ratio)] = 1
|
||||
visited_obstacle.add(
|
||||
(math.floor((point[0] + 350) / map_ratio), math.floor((point[1] + 400) / map_ratio)))
|
||||
for i in range(len(cur_objs_id)):
|
||||
if cur_objs_id[i] == "walker":
|
||||
walker_count += 1
|
||||
for obj in objs:
|
||||
if obj.name == cur_objs_id[i] and obj not in cur_objs:
|
||||
cur_objs.append(obj)
|
||||
break
|
||||
# plt.subplot(2, 1, 2) # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图
|
||||
# plt.imshow(self.map_map, cmap='binary', alpha=0.5, origin='lower',
|
||||
# extent=(-400 / map_ratio, 1450 / map_ratio,
|
||||
# -350 / map_ratio, 600 / map_ratio))
|
||||
new_map = self.updateMap(cur_obstacle_world_points)
|
||||
self.draw_map(plt, new_map)
|
||||
plt.axis("off")
|
||||
self.send_img("img_label_map")
|
||||
# plt.title("地图构建过程")
|
||||
# self.send_img("img_label_map")
|
||||
# plt.subplot(2, 7, 14) # 这里的2,1表示总共2行，1列，2表示这个位置是第2个子图
|
||||
new_add_info = len(cur_objs) - added_info + walker_count
|
||||
# plt.text(0, 0.5, f'新增语义信息：{new_add_info}', fontsize=10) # 在图中添加文字，x和y坐标是在这个图片大小内的相对位置，fontsize是字体大小
|
||||
added_info += new_add_info
|
||||
# plt.text(0, 0.3, f'已存语义信息：{added_info}', fontsize=10) # 在图中添加文字，x和y坐标是在这个图片大小内的相对位置，fontsize是字体大小
|
||||
self.infoCount = added_info
|
||||
plt.axis("off")
|
||||
# plt.show()
|
||||
print("------------当前检测到的物品信息--------------")
|
||||
print(cur_objs)
|
||||
time.sleep(1)
|
||||
|
||||
for i in range(len(cur_objs)):
|
||||
if cur_objs[i].name == "Desk" or cur_objs[i].name == "Chair":
|
||||
obj_json_data.append(
|
||||
{"id": f"{i}", "name": f"{cur_objs[i].name}", "location": f"{cur_objs[i].location}",
|
||||
"height": f"{cur_objs[i].location.Z * 2}"})
|
||||
|
||||
else:
|
||||
obj_json_data.append(
|
||||
{"id": f"{i}", "name": f"{cur_objs[i].name}", "location": f"{cur_objs[i].location}",
|
||||
"height": f"{cur_objs[i].location.Z}"})
|
||||
|
||||
with open('../../robowaiter/proto/objs.json', 'w') as file:
|
||||
json.dump(obj_json_data, file)
|
||||
|
||||
print("已绘制完成地图！！！")
|
||||
print("------------检测到的所有物品信息--------------")
|
||||
print(obj_json_data)
|
||||
|
||||
def run_VLN(self):
|
||||
self.gen_obj()
|
||||
self.add_walkers([
|
||||
|
|
@ -66,15 +159,57 @@ class SceneUI(Scene):
|
|||
]
|
||||
|
||||
def run_VLM(self):
|
||||
self.gen_obj()
|
||||
self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
|
||||
self. control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
|
||||
self.signal_event_list = [
|
||||
(3, self.add_walker, (20,0,700)),
|
||||
(1, self.control_walker, (6, False,100, 60, 520,0)),
|
||||
(1, self.customer_say, (6, "给我来份薯片和果汁，我坐在对面的桌子那儿。")),
|
||||
(5, self.control_walker, (6, False, 100, -250, 480, 0)),
|
||||
]
|
||||
pass
|
||||
|
||||
def run_GQA(self):
|
||||
self.gen_obj()
|
||||
self.add_walkers([ [16,250, 1200],[6,-55, 750],[10,70, -200],[47,-290, 400, 180],[26, 60,-320,90]])
|
||||
self.control_walker(1, True, 100, 60, 720, 0)
|
||||
self.control_walker(4, True, 100, 60, -120, 0)
|
||||
self.add_walkers([[31, 60,500,0], [15,60,550,0]])
|
||||
self.signal_event_list = [
|
||||
(5, self.customer_say, (6, "你好呀，你们这有啥好吃的？")), # 男
|
||||
(8, self.customer_say, (6, "听起来都好甜呀，我女朋友爱吃水果。")),
|
||||
(15, self.customer_say, (6, "你们这人可真多。")),
|
||||
(15, self.customer_say, (6, "我女朋友怕晒，有空余的阴凉位置嘛？")),
|
||||
(20, self.customer_say, (6, "那还不错。")),
|
||||
(15, self.customer_say, (5, "请问洗手间在哪呢？")),
|
||||
(20, self.customer_say, (5, "我们还想一起下下棋,切磋切磋。")),
|
||||
(20, self.customer_say, (6, "太棒啦，亲爱的。")),
|
||||
(15, self.customer_say, (5, "那你知道附近最近的电影院在哪吗?")),
|
||||
(20, self.customer_say, (6, "谢啦，那我们先去阴凉位置下个棋，等电影开始了就去看呢!")),
|
||||
]
|
||||
pass
|
||||
|
||||
def run_OT(self):
|
||||
self.gen_obj()
|
||||
self.add_walkers([ [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
|
||||
self.control_walker(1, True, 100, 60, 720, 0)
|
||||
self.control_walker(4, True, 100, 60, -120, 0)
|
||||
self.add_walkers([[16,60, 520], [47,-40, 520]])
|
||||
self.signal_event_list = [
|
||||
(8, self.customer_say, (5, "给我来杯咖啡，哦对，再倒一杯水。")),
|
||||
(1, self.control_walker_ls,([[[5, False, 100, -250, 480, 0],[6, False, 100, 60, 520, 0]]])),
|
||||
(-1, self.customer_say, (5, "感谢，这些够啦，你去忙吧。")),
|
||||
(10, self.customer_say, (6, "我想来份点心和酸奶。")),
|
||||
(-1, self.customer_say, (6, "真美味啊！")),
|
||||
]
|
||||
pass
|
||||
|
||||
def run_AT(self):
|
||||
self.add_walker(23, 60, 520, 0)
|
||||
self.signal_event_list = [
|
||||
(2, self.customer_say, (0,"可以关筒灯和关窗帘吗？")),
|
||||
]
|
||||
pass
|
||||
|
||||
def run_reset(self):
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ class SceneVLM(Scene):
|
|||
self.signal_event_list = [
|
||||
(3, self.add_walker, (20,0,700)),
|
||||
(1, self.control_walker, (6, False,100, 60, 520,0)),
|
||||
(1, self.customer_say, (6, "给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。")),
|
||||
(1, self.customer_say, (6, "给我来份薯片和果汁，我坐在对面的桌子那儿。")), #给我来杯酸奶和冰红茶，我坐在对面的桌子那儿。
|
||||
(5, self.control_walker, (6, False, 100, -250, 480, 0)),
|
||||
]
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue