diff --git a/robowaiter/llm_client/tool_register.py b/robowaiter/llm_client/tool_register.py index b2f893e..36f204b 100644 --- a/robowaiter/llm_client/tool_register.py +++ b/robowaiter/llm_client/tool_register.py @@ -6,11 +6,11 @@ from types import GenericAlias from typing import get_origin, Annotated import robowaiter.llm_client.find_obj_utils as find_obj_utils import random -import spacy +# import spacy _TOOL_HOOKS = {} _TOOL_DESCRIPTIONS = {} -nlp = spacy.load('en_core_web_lg') +# nlp = spacy.load('en_core_web_lg') def register_tool(func: callable): @@ -173,30 +173,30 @@ def create_sub_task( # near_object = random.choices(near_ls,k=5) # return near_object -@register_tool -def find_location( - location: Annotated[str, '客人咨询的地点', True] -) -> str: - """" - 获取的location为英文 - 用户想找某个地点 - """ - near_location = None - query_token = nlp(location) - max_similarity = 0 - similar_word = None - for w in find_obj_utils.all_loc_en: - word_token = nlp(w) - similarity = query_token.similarity(word_token) - - if similarity > max_similarity: - max_similarity = similarity - similar_word = w - print("similarity:", max_similarity, "similar_word:", similar_word) - if similar_word: - mp = list(find_obj_utils.loc_map_en[similar_word]) - near_location = random.choice(mp) - return near_location +# @register_tool +# def find_location( +# location: Annotated[str, '客人咨询的地点', True] +# ) -> str: +# """" +# 获取的location为英文 +# 用户想找某个地点 +# """ +# near_location = None +# query_token = nlp(location) +# max_similarity = 0 +# similar_word = None +# for w in find_obj_utils.all_loc_en: +# word_token = nlp(w) +# similarity = query_token.similarity(word_token) +# +# if similarity > max_similarity: +# max_similarity = similarity +# similar_word = w +# print("similarity:", max_similarity, "similar_word:", similar_word) +# if similar_word: +# mp = list(find_obj_utils.loc_map_en[similar_word]) +# near_location = random.choice(mp) +# return near_location if __name__ == "__main__": print(dispatch_tool("get_weather", {"city_name": "beijing"})) diff --git a/robowaiter/scene/tasks/VLM/VLM2.py b/robowaiter/scene/tasks/VLM/VLM2.py index f83416c..1f23690 100644 --- a/robowaiter/scene/tasks/VLM/VLM2.py +++ b/robowaiter/scene/tasks/VLM/VLM2.py @@ -15,6 +15,8 @@ class SceneVLM(Scene): ] def _reset(self): + self.gen_obj() + self.add_walkers([[47, 920]]) pass def _run(self, op_type=10): @@ -24,61 +26,21 @@ class SceneVLM(Scene): # 带领行人去有太阳的地方 # 行人说 有点热 # 好的，这就去开空调 - - self.gen_obj() - self.add_walkers([[47, 920]]) - self.control_walker( - [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)]) - time.sleep(1) - cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。" - self.control_robot_action(0,3,cont) - - # time.sleep(3) - # self.event_list.append((5, self.set_goal("At(Robot,BrightTable1)"))) # "请问可以带我去空位上嘛？我想晒太阳" - # self.chat_bubble("没问题！请跟我来。") - - - # 跟随机器人 - # cont = self.status.walkers[0].name + "好的！" - # self.control_robot_action(0, 3, cont) - # - # start = [self.status.location.X, self.status.location.Y] - # time.sleep(0.2) - # end = [self.status.location.X, self.status.location.Y] - # while abs(start[0]-end[0])>=1 or abs(start[1]-end[1])>=1: - # self.control_walker( - # [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=0)]) - # - # cont = self.status.walkers[0].name+"谢谢！" - # self.control_robot_action(0,3,cont) - - # 共17个操作 - # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7 - # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12 - # "调整空调开关","调高空调温度","调低空调温度", # 13-15 - # "抓握物体","放置物体" # 16-17 - - # self.gen_obj() - # if op_type <=15: - # self.move_task_area(op_type) - # self.op_task_execute(op_type) - # if op_type == 16: # 16: 抓操作需要传入物品id - # self.move_task_area(op_type, obj_id=0) - # self.op_task_execute(op_type, obj_id=0) - # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107] - # # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0] # 桌子2:[-55.0, 150.0, 107], - # if op_type == 17: # 17: 放操作需要传入放置位置周围的可达区域 - # pos = [240.0, 40.0, 100.0] - # self.move_task_area(op_type, release_pos=pos) - # self.op_task_execute(op_type, release_pos=pos) # [325.0, 860.0, 100] pass def _step(self): + self.control_walker( + [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)]) + time.sleep(3) + cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。" + self.control_robot_action(0,3,cont) + + # 如果机器人不在 吧台 # if "At(Robot,Bar)" not in self.state['condition_set']: end = [self.status.location.X, self.status.location.Y] - print("end:",end) + # print("end:",end) if end[1]>=600 or end[1]<=450 or end[0]>=250: # if int(self.status.location.X)!=247 or int(self.status.location.X)!=520: self.control_walker( diff --git a/robowaiter/scene/tasks/VLM/VLM3.py b/robowaiter/scene/tasks/VLM/VLM3.py new file mode 100644 index 0000000..f83416c --- /dev/null +++ b/robowaiter/scene/tasks/VLM/VLM3.py @@ -0,0 +1,101 @@ +""" +视觉语言操作 +机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板 +""" + +import time +from robowaiter.scene.scene import Scene + +class SceneVLM(Scene): + def __init__(self, robot): + super().__init__(robot) + # 在这里加入场景中发生的事件， (事件发生的时间，事件函数) + self.event_list = [ + (5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")), + ] + + def _reset(self): + pass + + def _run(self, op_type=10): + # 一个行人从门口走到 吧台 + # 打招呼需要什么 + # 行人说 哪里有位置，想晒个太阳 + # 带领行人去有太阳的地方 + # 行人说 有点热 + # 好的，这就去开空调 + + self.gen_obj() + self.add_walkers([[47, 920]]) + self.control_walker( + [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)]) + time.sleep(1) + cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。" + self.control_robot_action(0,3,cont) + + # time.sleep(3) + # self.event_list.append((5, self.set_goal("At(Robot,BrightTable1)"))) # "请问可以带我去空位上嘛？我想晒太阳" + # self.chat_bubble("没问题！请跟我来。") + + + # 跟随机器人 + # cont = self.status.walkers[0].name + "好的！" + # self.control_robot_action(0, 3, cont) + # + # start = [self.status.location.X, self.status.location.Y] + # time.sleep(0.2) + # end = [self.status.location.X, self.status.location.Y] + # while abs(start[0]-end[0])>=1 or abs(start[1]-end[1])>=1: + # self.control_walker( + # [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=0)]) + # + # cont = self.status.walkers[0].name+"谢谢！" + # self.control_robot_action(0,3,cont) + + # 共17个操作 + # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7 + # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12 + # "调整空调开关","调高空调温度","调低空调温度", # 13-15 + # "抓握物体","放置物体" # 16-17 + + # self.gen_obj() + # if op_type <=15: + # self.move_task_area(op_type) + # self.op_task_execute(op_type) + # if op_type == 16: # 16: 抓操作需要传入物品id + # self.move_task_area(op_type, obj_id=0) + # self.op_task_execute(op_type, obj_id=0) + # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107] + # # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0] # 桌子2:[-55.0, 150.0, 107], + # if op_type == 17: # 17: 放操作需要传入放置位置周围的可达区域 + # pos = [240.0, 40.0, 100.0] + # self.move_task_area(op_type, release_pos=pos) + # self.op_task_execute(op_type, release_pos=pos) # [325.0, 860.0, 100] + pass + + def _step(self): + + # 如果机器人不在 吧台 + # if "At(Robot,Bar)" not in self.state['condition_set']: + end = [self.status.location.X, self.status.location.Y] + print("end:",end) + if end[1]>=600 or end[1]<=450 or end[0]>=250: + # if int(self.status.location.X)!=247 or int(self.status.location.X)!=520: + self.control_walker( + [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=-90)]) + + cont = self.status.walkers[0].name+"谢谢！" + self.control_robot_action(0,3,cont) + pass + + +if __name__ == '__main__': + import os + from robowaiter.robot.robot import Robot + + robot = Robot() + + # create task + task = SceneVLM(robot) + task.reset() + task.run()