diff --git a/robowaiter/llm_client/find_obj_utils.py b/robowaiter/llm_client/find_obj_utils.py index 2d940ae..98f965d 100644 --- a/robowaiter/llm_client/find_obj_utils.py +++ b/robowaiter/llm_client/find_obj_utils.py @@ -35,7 +35,7 @@ all_loc_en = ['bar', 'Table', 'sofa', 'stove', 'Gate', 'light switch', 'aircondi 'cake display', 'ChargingStations', 'refrigerator', 'bookshelf'] loc_map_en = {'bar': {'工作台', '服务台', '收银台', '蛋糕柜'}, - 'Table': {'沙发', '大门', '窗户', '休闲区', '墙角', '椅子', '书架'}, + 'Table': {'大门', '休闲区', '墙角'}, 'sofa': {'餐桌', '窗户', '音响', '休闲区', '墙角', '书架'}, 'stove': {'吧台', '橱柜', '工作台', '服务台', '收银台', '蛋糕柜', '冰箱'}, 'Gate': {'吧台', '灯开关', '空调开关', '卫生间', '墙角'}, diff --git a/robowaiter/llm_client/tool_api.py b/robowaiter/llm_client/tool_api.py index 51be613..f6f1bd5 100644 --- a/robowaiter/llm_client/tool_api.py +++ b/robowaiter/llm_client/tool_api.py @@ -60,7 +60,7 @@ def run_conversation(query: str, stream=False, max_retry=5): "content": tool_response, # 调用函数返回结果 } ) - # del params["functions"] + del params["functions"] else: reply = response["choices"][0]["message"]["content"] return { @@ -116,12 +116,24 @@ if __name__ == "__main__": query = "卫生间在哪里" # print(run_conversation(query, stream=False)) - query = "我想看看冰箱,请问哪里可以找到冰箱" - print(run_conversation(query, stream=False)) - - query = "我想找个充电的地方，你能告诉我在哪儿吗" - print(run_conversation(query, stream=False)) - query = "我想找张桌子" # print(run_conversation(query, stream=False)) - # for query in \ No newline at end of file + + query = "我想看看休闲区,请问哪里可以找到休闲区" + print(run_conversation(query, stream=False)) + # + # query = "我想找个充电的地方，你能告诉我在哪儿吗" + # print(run_conversation(query, stream=False)) + + + query = "我想来一个面包" + print(run_conversation(query, stream=False)) + + query = "哪里有卫生纸" + print(run_conversation(query, stream=False)) + + query = "插座在哪里你知道吗" + print(run_conversation(query, stream=False)) + + query = "你们的咖啡厅里有香蕉吗" + print(run_conversation(query, stream=False)) \ No newline at end of file diff --git a/robowaiter/scene/scene.py b/robowaiter/scene/scene.py index 7088e8f..0cc08b7 100644 --- a/robowaiter/scene/scene.py +++ b/robowaiter/scene/scene.py @@ -96,21 +96,21 @@ class Scene: robot.load_BT() self.robot = robot - # myx op # 1-7 正常执行, 8-10 控灯操作移动到6, 11-12窗帘操作不需要移动, self.op_dialog = ["","制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7 "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12 "调整空调开关","调高空调温度","调低空调温度", # 13-15 "抓握物体","放置物体"] # 16-17 + # 动画控制的执行步骤数 self.op_act_num = [0,3,4,6,3,2,0,1, 0,0,0,0,0, 0,0,0, 0,0] + # 动画控制的执行区域坐标 self.op_v_list = [[0.0,0.0],[250.0, 310.0],[-70.0, 480.0],[250.0, 630.0],[-70.0, 740.0],[260.0, 1120.0],[300.0, -220.0], [0.0, -70.0]] - self.op_typeToAct = {8:[6,2],9:[6,3],10:[6,4],11:[8,1],12:[8,2]} - # 空调面板位置 - self.obj_loc = [300.5, -140.0,114] + self.op_typeToAct = {8:[6,2],9:[6,3],10:[6,4],11:[8,1],12:[8,2]} # 任务类型到行动的映射 + self.obj_loc = [300.5, -140.0,114] # 空调面板位置 # AEM self.visited = set() @@ -489,33 +489,33 @@ class Scene: # 移动到进行操作任务的指定地点 - def move_task_area(self,op_type,obj_id=0, release_pos=[247.0, 520.0, 100.0]): + def move_task_area(self, op_type, obj_id=0, release_pos=[247.0, 520.0, 100.0]): scene = self.status cur_pos = [scene.location.X, scene.location.Y, scene.rotation.Yaw] print("Current Position:", cur_pos, "开始任务:", self.op_dialog[op_type]) - - if op_type==11 or op_type==12: # 开关窗帘不需要移动 + if op_type == 11 or op_type == 12: # 开关窗帘不需要移动 return print('------------------moveTo_Area----------------------') - if op_type < 8: - walk_v = self.op_v_list[op_type] + [scene.rotation.Yaw, 180, 0] # 动画控制 - print("walk_v:",walk_v) - if op_type>=8 and op_type<=10: walk_v = self.op_v_list[6] + [scene.rotation.Yaw, 180, 0] # 控灯 - if op_type in [13,14,15]: walk_v = [240, -140.0] + [0, 180, 0] # 空调 - if op_type==16: # 抓握物体，移动到物体周围的可达区域 + if op_type < 8: # 动画控制相关任务的移动目标 + walk_v = self.op_v_list[op_type] + [scene.rotation.Yaw, 180, 0] + if 8 <= op_type <= 10: # 控灯相关任务的移动目标 + walk_v = self.op_v_list[6] + [scene.rotation.Yaw, 180, 0] + if op_type in [13,14,15]: # 空调相关任务的移动目标 + walk_v = [240, -140.0] + [0, 180, 0] + if op_type == 16: # 抓握物体，移动到物体周围的可达区域 scene = self.status obj_info = scene.objects[obj_id] - # Robot obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z walk_v = [obj_x + 50, obj_y] + [180, 180, 0] - if obj_y >= 820 and obj_y <= 1200 and obj_x >= 240 and obj_x <= 500: # 物品位于斜的抹布桌上 ([240,500],[820,1200]) + if 820 <= obj_y <= 1200 and 240 <= obj_x <= 500: # 物品位于斜的抹布桌上 ([240,500],[820,1200]) walk_v = [obj_x + 40, obj_y - 35, 130, 180, 0] obj_x += 3 obj_y += 2.5 - if op_type==17: # 放置物体，移动到物体周围的可达区域 + if op_type == 17: # 放置物体，移动到物体周围的可达区域 walk_v = release_pos[:-1] + [180, 180, 0] if release_pos == [340.0, 900.0, 99.0]: walk_v[2] = 130 + # 移动到目标位置 action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v) scene = stub.Do(action) print("After Walk Position:", [scene.location.X, scene.location.Y, scene.rotation.Yaw]) @@ -536,18 +536,16 @@ class Scene: print(scene.info) return False + # 调整空调开关、温度 def adjust_kongtiao(self,op_type): - print("self.obj_loc:",self.obj_loc) obj_loc = self.obj_loc[:] - print("obj_loc:",obj_loc,"self.obj_loc:", self.obj_loc) obj_loc[2] -= 5 - print("obj_loc:",obj_loc) if op_type == 13: obj_loc[1] -= 2 if op_type == 14: obj_loc[1] -= 0 if op_type == 15: obj_loc[1] += 2 self.ik_control_joints(2, obj_loc[0], obj_loc[1], obj_loc[2]) time.sleep(3.0) - self.robo_recover() + self.robo_recover() # 恢复肢体关节 return True def gen_obj(self,h=100): @@ -561,7 +559,9 @@ class Scene: scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID)) time.sleep(1.0) + # 实现抓握操作 def grasp_obj(self,obj_id,hand_id=1): + print('------------------adjust_joints----------------------') scene = self.status obj_info = scene.objects[obj_id] obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z @@ -570,7 +570,7 @@ class Scene: # obj_y -= 1 # values = [0,0,0,0,0, 10,-25,-45,-45,-45] # values= [-6, 0, 0, 0, 0, -6, 0, 45, 45, 45] - stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values)) + # stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values)) pass if obj_info.name=="Glass": pass @@ -590,12 +590,13 @@ class Scene: values=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) scene = stub.Do(action) + # 恢复手指关节 def standard_finger(self): values = [0,0,0,0,0, 0,0,0,0,0] stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values)) time.sleep(1.0) - + # 弯腰以及手掌与放置面平齐 def robo_stoop_parallel(self): # 0-3是躯干，4-6是脖子和头，7-13是左胳膊，14-20是右胳膊 scene = self.status @@ -608,8 +609,9 @@ class Scene: scene = stub.Do(action) time.sleep(1.0) + # 实现放置操作 def release_obj(self,release_pos): - print("------------------release_obj----------------------") + print("------------------adjust_joints----------------------") if release_pos==[340.0, 900.0, 99.0]: self.ik_control_joints(2, release_pos[0]-40, release_pos[1]+35, release_pos[2]) time.sleep(2.0) @@ -617,25 +619,26 @@ class Scene: self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2]) time.sleep(2.0) self.robo_stoop_parallel() - + print("------------------release_obj----------------------") action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Release, values=[1]) scene = stub.Do(action) time.sleep(2.0) - self.robo_recover() - self.standard_finger() - + self.robo_recover() # 恢复肢体关节 + self.standard_finger() # 恢复手指关节 return True - # 执行过程：输出"开始(任务名)" -> 按步骤数执行任务 -> Robot输出成功或失败的对话 + # 执行过程: Robot输出"开始(任务名)" -> 按步骤数执行任务 -> Robot输出成功或失败的对话 def op_task_execute(self,op_type,obj_id=0,release_pos=[247.0, 520.0, 100.0]): - self.control_robot_action(0, 1, "开始"+self.op_dialog[op_type]) # 开始制作咖啡 - if op_type<8: result = self.control_robot_action(op_type, 1) - if op_type>=8 and op_type<=12: result = self.control_robot_action(self.op_typeToAct[op_type][0], self.op_typeToAct[op_type][1]) - if op_type in [13,14,15]: # 调整空调:13代表按开关,14升温,15降温 + self.control_robot_action(0, 1, "开始"+self.op_dialog[op_type]) # 输出正在执行的任务 + if op_type < 8: + result = self.control_robot_action(op_type, 1) + if 8 <= op_type <= 12: + result = self.control_robot_action(self.op_typeToAct[op_type][0], self.op_typeToAct[op_type][1]) + if op_type in [13, 14, 15]: # 调整空调:13代表按开关,14升温,15降温 result = self.adjust_kongtiao(op_type) - if op_type ==16: # 抓握物体 + if op_type == 16: # 抓握物体, 需要传入物品id result = self.grasp_obj(obj_id) - if op_type ==17: # 放置物体 + if op_type == 17: # 放置物体, 放置物品, 需要传入放置地点 result = self.release_obj(release_pos) self.control_robot_action(0, 2) if result: @@ -777,7 +780,3 @@ class Scene: return True else: return False - - - -