EmbodiedAI
/
RoboWaiter
mirror of https://github.com/HPCL-EI/RoboWaiter.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:HPCL-EI/RoboWaiter

This commit is contained in:
ChenXL97 2023-11-18 12:07:49 +08:00
parent 935c359317 c4373d86cb
commit ae314d58e3
3 changed files with 58 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
robowaiter/llm_client/find_obj_utils.py
View File

@ -35,7 +35,7 @@ all_loc_en = ['bar', 'Table', 'sofa', 'stove', 'Gate', 'light switch', 'aircondi
'cake display', 'ChargingStations', 'refrigerator', 'bookshelf'] 'cake display', 'ChargingStations', 'refrigerator', 'bookshelf']
loc_map_en = {'bar': {'工作台', '服务台', '收银台', '蛋糕柜'}, loc_map_en = {'bar': {'工作台', '服务台', '收银台', '蛋糕柜'},
'Table': {'沙发', '大门', '窗户', '休闲区', '墙角', '椅子', '书架'}, 'Table': {'大门', '休闲区', '墙角'},
'sofa': {'餐桌', '窗户', '音响', '休闲区', '墙角', '书架'}, 'sofa': {'餐桌', '窗户', '音响', '休闲区', '墙角', '书架'},
'stove': {'吧台', '橱柜', '工作台', '服务台', '收银台', '蛋糕柜', '冰箱'}, 'stove': {'吧台', '橱柜', '工作台', '服务台', '收银台', '蛋糕柜', '冰箱'},
'Gate': {'吧台', '灯开关', '空调开关', '卫生间', '墙角'}, 'Gate': {'吧台', '灯开关', '空调开关', '卫生间', '墙角'},

28
robowaiter/llm_client/tool_api.py
View File

@ -60,7 +60,7 @@ def run_conversation(query: str, stream=False, max_retry=5):
"content": tool_response, # 调用函数返回结果 "content": tool_response, # 调用函数返回结果
} }
) )
# del params["functions"] del params["functions"]
else: else:
reply = response["choices"][0]["message"]["content"] reply = response["choices"][0]["message"]["content"]
return { return {
@ -116,12 +116,24 @@ if __name__ == "__main__":
query = "卫生间在哪里" # query = "卫生间在哪里" #
print(run_conversation(query, stream=False)) print(run_conversation(query, stream=False))
query = "我想看看冰箱,请问哪里可以找到冰箱"
print(run_conversation(query, stream=False))
query = "我想找个充电的地方，你能告诉我在哪儿吗"
print(run_conversation(query, stream=False))
query = "我想找张桌子" # query = "我想找张桌子" #
print(run_conversation(query, stream=False)) print(run_conversation(query, stream=False))
# for query in
query = "我想看看休闲区,请问哪里可以找到休闲区"
print(run_conversation(query, stream=False))
#
# query = "我想找个充电的地方，你能告诉我在哪儿吗"
# print(run_conversation(query, stream=False))
query = "我想来一个面包"
print(run_conversation(query, stream=False))
query = "哪里有卫生纸"
print(run_conversation(query, stream=False))
query = "插座在哪里你知道吗"
print(run_conversation(query, stream=False))
query = "你们的咖啡厅里有香蕉吗"
print(run_conversation(query, stream=False))

75
robowaiter/scene/scene.py
View File

@ -96,21 +96,21 @@ class Scene:
robot.load_BT() robot.load_BT()
self.robot = robot self.robot = robot
# myx op
# 1-7 正常执行, 8-10 控灯操作移动到6, 11-12窗帘操作不需要移动, # 1-7 正常执行, 8-10 控灯操作移动到6, 11-12窗帘操作不需要移动,
self.op_dialog = ["","制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7 self.op_dialog = ["","制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7
"关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12 "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12
"调整空调开关","调高空调温度","调低空调温度", # 13-15 "调整空调开关","调高空调温度","调低空调温度", # 13-15
"抓握物体","放置物体"] # 16-17 "抓握物体","放置物体"] # 16-17
# 动画控制的执行步骤数
self.op_act_num = [0,3,4,6,3,2,0,1, self.op_act_num = [0,3,4,6,3,2,0,1,
0,0,0,0,0, 0,0,0,0,0,
0,0,0, 0,0,0,
0,0] 0,0]
# 动画控制的执行区域坐标
self.op_v_list = [[0.0,0.0],[250.0, 310.0],[-70.0, 480.0],[250.0, 630.0],[-70.0, 740.0],[260.0, 1120.0],[300.0, -220.0], self.op_v_list = [[0.0,0.0],[250.0, 310.0],[-70.0, 480.0],[250.0, 630.0],[-70.0, 740.0],[260.0, 1120.0],[300.0, -220.0],
[0.0, -70.0]] [0.0, -70.0]]
self.op_typeToAct = {8:[6,2],9:[6,3],10:[6,4],11:[8,1],12:[8,2]} self.op_typeToAct = {8:[6,2],9:[6,3],10:[6,4],11:[8,1],12:[8,2]} # 任务类型到行动的映射
# 空调面板位置 self.obj_loc = [300.5, -140.0,114] # 空调面板位置
self.obj_loc = [300.5, -140.0,114]
# AEM # AEM
self.visited = set() self.visited = set()
@ -489,33 +489,33 @@ class Scene:
# 移动到进行操作任务的指定地点 # 移动到进行操作任务的指定地点
def move_task_area(self,op_type,obj_id=0, release_pos=[247.0, 520.0, 100.0]): def move_task_area(self, op_type, obj_id=0, release_pos=[247.0, 520.0, 100.0]):
scene = self.status scene = self.status
cur_pos = [scene.location.X, scene.location.Y, scene.rotation.Yaw] cur_pos = [scene.location.X, scene.location.Y, scene.rotation.Yaw]
print("Current Position:", cur_pos, "开始任务:", self.op_dialog[op_type]) print("Current Position:", cur_pos, "开始任务:", self.op_dialog[op_type])
if op_type == 11 or op_type == 12: # 开关窗帘不需要移动
if op_type==11 or op_type==12: # 开关窗帘不需要移动
return return
print('------------------moveTo_Area----------------------') print('------------------moveTo_Area----------------------')
if op_type < 8: if op_type < 8: # 动画控制相关任务的移动目标
walk_v = self.op_v_list[op_type] + [scene.rotation.Yaw, 180, 0] # 动画控制 walk_v = self.op_v_list[op_type] + [scene.rotation.Yaw, 180, 0]
print("walk_v:",walk_v) if 8 <= op_type <= 10: # 控灯相关任务的移动目标
if op_type>=8 and op_type<=10: walk_v = self.op_v_list[6] + [scene.rotation.Yaw, 180, 0] # 控灯 walk_v = self.op_v_list[6] + [scene.rotation.Yaw, 180, 0]
if op_type in [13,14,15]: walk_v = [240, -140.0] + [0, 180, 0] # 空调 if op_type in [13,14,15]: # 空调相关任务的移动目标
if op_type==16: # 抓握物体，移动到物体周围的可达区域 walk_v = [240, -140.0] + [0, 180, 0]
if op_type == 16: # 抓握物体，移动到物体周围的可达区域
scene = self.status scene = self.status
obj_info = scene.objects[obj_id] obj_info = scene.objects[obj_id]
# Robot
obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z
walk_v = [obj_x + 50, obj_y] + [180, 180, 0] walk_v = [obj_x + 50, obj_y] + [180, 180, 0]
if obj_y >= 820 and obj_y <= 1200 and obj_x >= 240 and obj_x <= 500: # 物品位于斜的抹布桌上 ([240,500],[820,1200]) if 820 <= obj_y <= 1200 and 240 <= obj_x <= 500: # 物品位于斜的抹布桌上 ([240,500],[820,1200])
walk_v = [obj_x + 40, obj_y - 35, 130, 180, 0] walk_v = [obj_x + 40, obj_y - 35, 130, 180, 0]
obj_x += 3 obj_x += 3
obj_y += 2.5 obj_y += 2.5
if op_type==17: # 放置物体，移动到物体周围的可达区域 if op_type == 17: # 放置物体，移动到物体周围的可达区域
walk_v = release_pos[:-1] + [180, 180, 0] walk_v = release_pos[:-1] + [180, 180, 0]
if release_pos == [340.0, 900.0, 99.0]: if release_pos == [340.0, 900.0, 99.0]:
walk_v[2] = 130 walk_v[2] = 130
# 移动到目标位置
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v) action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v)
scene = stub.Do(action) scene = stub.Do(action)
print("After Walk Position:", [scene.location.X, scene.location.Y, scene.rotation.Yaw]) print("After Walk Position:", [scene.location.X, scene.location.Y, scene.rotation.Yaw])
@ -536,18 +536,16 @@ class Scene:
print(scene.info) print(scene.info)
return False return False
# 调整空调开关、温度
def adjust_kongtiao(self,op_type): def adjust_kongtiao(self,op_type):
print("self.obj_loc:",self.obj_loc)
obj_loc = self.obj_loc[:] obj_loc = self.obj_loc[:]
print("obj_loc:",obj_loc,"self.obj_loc:", self.obj_loc)
obj_loc[2] -= 5 obj_loc[2] -= 5
print("obj_loc:",obj_loc)
if op_type == 13: obj_loc[1] -= 2 if op_type == 13: obj_loc[1] -= 2
if op_type == 14: obj_loc[1] -= 0 if op_type == 14: obj_loc[1] -= 0
if op_type == 15: obj_loc[1] += 2 if op_type == 15: obj_loc[1] += 2
self.ik_control_joints(2, obj_loc[0], obj_loc[1], obj_loc[2]) self.ik_control_joints(2, obj_loc[0], obj_loc[1], obj_loc[2])
time.sleep(3.0) time.sleep(3.0)
self.robo_recover() self.robo_recover() # 恢复肢体关节
return True return True
def gen_obj(self,h=100): def gen_obj(self,h=100):
@ -561,7 +559,9 @@ class Scene:
scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID)) scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID))
time.sleep(1.0) time.sleep(1.0)
# 实现抓握操作
def grasp_obj(self,obj_id,hand_id=1): def grasp_obj(self,obj_id,hand_id=1):
print('------------------adjust_joints----------------------')
scene = self.status scene = self.status
obj_info = scene.objects[obj_id] obj_info = scene.objects[obj_id]
obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z
@ -570,7 +570,7 @@ class Scene:
# obj_y -= 1 # obj_y -= 1
# values = [0,0,0,0,0, 10,-25,-45,-45,-45] # values = [0,0,0,0,0, 10,-25,-45,-45,-45]
# values= [-6, 0, 0, 0, 0, -6, 0, 45, 45, 45] # values= [-6, 0, 0, 0, 0, -6, 0, 45, 45, 45]
stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values)) # stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values))
pass pass
if obj_info.name=="Glass": if obj_info.name=="Glass":
pass pass
@ -590,12 +590,13 @@ class Scene:
values=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) values=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
scene = stub.Do(action) scene = stub.Do(action)
# 恢复手指关节
def standard_finger(self): def standard_finger(self):
values = [0,0,0,0,0, 0,0,0,0,0] values = [0,0,0,0,0, 0,0,0,0,0]
stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values)) stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values))
time.sleep(1.0) time.sleep(1.0)
# 弯腰以及手掌与放置面平齐
def robo_stoop_parallel(self): def robo_stoop_parallel(self):
# 0-3是躯干，4-6是脖子和头，7-13是左胳膊，14-20是右胳膊 # 0-3是躯干，4-6是脖子和头，7-13是左胳膊，14-20是右胳膊
scene = self.status scene = self.status
@ -608,8 +609,9 @@ class Scene:
scene = stub.Do(action) scene = stub.Do(action)
time.sleep(1.0) time.sleep(1.0)
# 实现放置操作
def release_obj(self,release_pos): def release_obj(self,release_pos):
print("------------------release_obj----------------------") print("------------------adjust_joints----------------------")
if release_pos==[340.0, 900.0, 99.0]: if release_pos==[340.0, 900.0, 99.0]:
self.ik_control_joints(2, release_pos[0]-40, release_pos[1]+35, release_pos[2]) self.ik_control_joints(2, release_pos[0]-40, release_pos[1]+35, release_pos[2])
time.sleep(2.0) time.sleep(2.0)
@ -617,25 +619,26 @@ class Scene:
self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2]) self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2])
time.sleep(2.0) time.sleep(2.0)
self.robo_stoop_parallel() self.robo_stoop_parallel()
print("------------------release_obj----------------------")
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Release, values=[1]) action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Release, values=[1])
scene = stub.Do(action) scene = stub.Do(action)
time.sleep(2.0) time.sleep(2.0)
self.robo_recover() self.robo_recover() # 恢复肢体关节
self.standard_finger() self.standard_finger() # 恢复手指关节
return True return True
# 执行过程：输出"开始(任务名)" -> 按步骤数执行任务 -> Robot输出成功或失败的对话 # 执行过程: Robot输出"开始(任务名)" -> 按步骤数执行任务 -> Robot输出成功或失败的对话
def op_task_execute(self,op_type,obj_id=0,release_pos=[247.0, 520.0, 100.0]): def op_task_execute(self,op_type,obj_id=0,release_pos=[247.0, 520.0, 100.0]):
self.control_robot_action(0, 1, "开始"+self.op_dialog[op_type]) # 开始制作咖啡 self.control_robot_action(0, 1, "开始"+self.op_dialog[op_type]) # 输出正在执行的任务
if op_type<8: result = self.control_robot_action(op_type, 1) if op_type < 8:
if op_type>=8 and op_type<=12: result = self.control_robot_action(self.op_typeToAct[op_type][0], self.op_typeToAct[op_type][1]) result = self.control_robot_action(op_type, 1)
if op_type in [13,14,15]: # 调整空调:13代表按开关,14升温,15降温 if 8 <= op_type <= 12:
result = self.control_robot_action(self.op_typeToAct[op_type][0], self.op_typeToAct[op_type][1])
if op_type in [13, 14, 15]: # 调整空调:13代表按开关,14升温,15降温
result = self.adjust_kongtiao(op_type) result = self.adjust_kongtiao(op_type)
if op_type ==16: # 抓握物体 if op_type == 16: # 抓握物体, 需要传入物品id
result = self.grasp_obj(obj_id) result = self.grasp_obj(obj_id)
if op_type ==17: # 放置物体 if op_type == 17: # 放置物体, 放置物品, 需要传入放置地点
result = self.release_obj(release_pos) result = self.release_obj(release_pos)
self.control_robot_action(0, 2) self.control_robot_action(0, 2)
if result: if result:
@ -777,7 +780,3 @@ class Scene:
return True return True
else: else:
return False return False