注释了工具调用

robowaiter/llm_client/tool_register.py

@@ -6,11 +6,11 @@ from types import GenericAlias
 from typing import get_origin, Annotated
 import robowaiter.llm_client.find_obj_utils as find_obj_utils
 import random
-import spacy
+# import spacy
 
 _TOOL_HOOKS = {}
 _TOOL_DESCRIPTIONS = {}
-nlp = spacy.load('en_core_web_lg')
+# nlp = spacy.load('en_core_web_lg')
 
 
 def register_tool(func: callable):
@@ -173,30 +173,30 @@ def create_sub_task(
 #         near_object = random.choices(near_ls,k=5)
 #     return near_object
 
-@register_tool
+# @register_tool
-def find_location(
+# def find_location(
-        location: Annotated[str, '客人咨询的地点', True]
+#         location: Annotated[str, '客人咨询的地点', True]
-) -> str:
+# ) -> str:
-    """"
+#     """"
-    获取的location为英文
+#     获取的location为英文
-    用户想找某个地点
+#     用户想找某个地点
-    """
+#     """
-    near_location = None
+#     near_location = None
-    query_token = nlp(location)
+#     query_token = nlp(location)
-    max_similarity = 0
+#     max_similarity = 0
-    similar_word = None
+#     similar_word = None
-    for w in find_obj_utils.all_loc_en:
+#     for w in find_obj_utils.all_loc_en:
-        word_token = nlp(w)
+#         word_token = nlp(w)
-        similarity = query_token.similarity(word_token)
+#         similarity = query_token.similarity(word_token)
-
+#
-        if similarity > max_similarity:
+#         if similarity > max_similarity:
-            max_similarity = similarity
+#             max_similarity = similarity
-            similar_word = w
+#             similar_word = w
-    print("similarity:", max_similarity, "similar_word:", similar_word)
+#     print("similarity:", max_similarity, "similar_word:", similar_word)
-    if similar_word:
+#     if similar_word:
-        mp = list(find_obj_utils.loc_map_en[similar_word])
+#         mp = list(find_obj_utils.loc_map_en[similar_word])
-        near_location = random.choice(mp)
+#         near_location = random.choice(mp)
-    return near_location
+#     return near_location
 
 if __name__ == "__main__":
     print(dispatch_tool("get_weather", {"city_name": "beijing"}))

robowaiter/scene/tasks/VLM/VLM2.py

@@ -15,6 +15,8 @@ class SceneVLM(Scene):
         ]
 
     def _reset(self):
+        self.gen_obj()
+        self.add_walkers([[47, 920]])
         pass
 
     def _run(self, op_type=10):
@@ -24,61 +26,21 @@ class SceneVLM(Scene):
         # 带领行人去有太阳的地方
         # 行人说 有点热
         # 好的，这就去开空调
-
-        self.gen_obj()
-        self.add_walkers([[47, 920]])
-        self.control_walker(
-            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
-        time.sleep(1)
-        cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
-        self.control_robot_action(0,3,cont)
-
-        # time.sleep(3)
-        # self.event_list.append((5, self.set_goal("At(Robot,BrightTable1)"))) # "请问可以带我去空位上嘛？我想晒太阳"
-        # self.chat_bubble("没问题！请跟我来。")
-
-
-        # 跟随机器人
-        # cont = self.status.walkers[0].name + "好的！"
-        # self.control_robot_action(0, 3, cont)
-        #
-        # start = [self.status.location.X, self.status.location.Y]
-        # time.sleep(0.2)
-        # end = [self.status.location.X, self.status.location.Y]
-        # while abs(start[0]-end[0])>=1 or abs(start[1]-end[1])>=1:
-        #     self.control_walker(
-        #         [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=0)])
-        #
-        # cont = self.status.walkers[0].name+"谢谢！"
-        # self.control_robot_action(0,3,cont)
-
-        # 共17个操作
-        # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子",    # 1-7
-        # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘",            # 8-12
-        # "调整空调开关","调高空调温度","调低空调温度",                   # 13-15
-        # "抓握物体","放置物体"                                       # 16-17
-
-        # self.gen_obj()
-        # if op_type <=15:
-        #     self.move_task_area(op_type)
-        #     self.op_task_execute(op_type)
-        # if op_type == 16:   # 16: 抓操作需要传入物品id
-        #     self.move_task_area(op_type, obj_id=0)
-        #     self.op_task_execute(op_type, obj_id=0)
-        # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107]
-        # # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0]   # 桌子2:[-55.0, 150.0, 107],
-        # if op_type == 17:   # 17: 放操作需要传入放置位置周围的可达区域
-        #     pos = [240.0, 40.0, 100.0]
-        #     self.move_task_area(op_type, release_pos=pos)
-        #     self.op_task_execute(op_type, release_pos=pos)   # [325.0, 860.0, 100]
         pass
     
     def _step(self):
 
+        self.control_walker(
+            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
+        time.sleep(3)
+        cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
+        self.control_robot_action(0,3,cont)
+
+
         # 如果机器人不在 吧台
         # if "At(Robot,Bar)" not in self.state['condition_set']:
         end = [self.status.location.X, self.status.location.Y]
-        print("end:",end)
+        # print("end:",end)
         if end[1]>=600 or end[1]<=450 or end[0]>=250:
         # if int(self.status.location.X)!=247 or  int(self.status.location.X)!=520:
             self.control_walker(

robowaiter/scene/tasks/VLM/VLM3.py

@@ -0,0 +1,101 @@
+"""
+视觉语言操作
+机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板
+"""
+
+import time
+from robowaiter.scene.scene import Scene
+
+class SceneVLM(Scene):
+    def __init__(self, robot):
+        super().__init__(robot)
+        # 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
+        self.event_list = [
+            (5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
+        ]
+
+    def _reset(self):
+        pass
+
+    def _run(self, op_type=10):
+        # 一个行人从门口走到 吧台
+        # 打招呼需要什么
+        # 行人说 哪里有位置，想晒个太阳
+        # 带领行人去有太阳的地方
+        # 行人说 有点热
+        # 好的，这就去开空调
+
+        self.gen_obj()
+        self.add_walkers([[47, 920]])
+        self.control_walker(
+            [self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
+        time.sleep(1)
+        cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
+        self.control_robot_action(0,3,cont)
+
+        # time.sleep(3)
+        # self.event_list.append((5, self.set_goal("At(Robot,BrightTable1)"))) # "请问可以带我去空位上嘛？我想晒太阳"
+        # self.chat_bubble("没问题！请跟我来。")
+
+
+        # 跟随机器人
+        # cont = self.status.walkers[0].name + "好的！"
+        # self.control_robot_action(0, 3, cont)
+        #
+        # start = [self.status.location.X, self.status.location.Y]
+        # time.sleep(0.2)
+        # end = [self.status.location.X, self.status.location.Y]
+        # while abs(start[0]-end[0])>=1 or abs(start[1]-end[1])>=1:
+        #     self.control_walker(
+        #         [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=0)])
+        #
+        # cont = self.status.walkers[0].name+"谢谢！"
+        # self.control_robot_action(0,3,cont)
+
+        # 共17个操作
+        # "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子",    # 1-7
+        # "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘",            # 8-12
+        # "调整空调开关","调高空调温度","调低空调温度",                   # 13-15
+        # "抓握物体","放置物体"                                       # 16-17
+
+        # self.gen_obj()
+        # if op_type <=15:
+        #     self.move_task_area(op_type)
+        #     self.op_task_execute(op_type)
+        # if op_type == 16:   # 16: 抓操作需要传入物品id
+        #     self.move_task_area(op_type, obj_id=0)
+        #     self.op_task_execute(op_type, obj_id=0)
+        # # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107]
+        # # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0]   # 桌子2:[-55.0, 150.0, 107],
+        # if op_type == 17:   # 17: 放操作需要传入放置位置周围的可达区域
+        #     pos = [240.0, 40.0, 100.0]
+        #     self.move_task_area(op_type, release_pos=pos)
+        #     self.op_task_execute(op_type, release_pos=pos)   # [325.0, 860.0, 100]
+        pass
+    
+    def _step(self):
+
+        # 如果机器人不在 吧台
+        # if "At(Robot,Bar)" not in self.state['condition_set']:
+        end = [self.status.location.X, self.status.location.Y]
+        print("end:",end)
+        if end[1]>=600 or end[1]<=450 or end[0]>=250:
+        # if int(self.status.location.X)!=247 or  int(self.status.location.X)!=520:
+            self.control_walker(
+                    [self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=-90)])
+
+            cont = self.status.walkers[0].name+"谢谢！"
+            self.control_robot_action(0,3,cont)
+        pass
+
+
+if __name__ == '__main__':
+    import os
+    from robowaiter.robot.robot import Robot
+
+    robot = Robot()
+
+    # create task
+    task = SceneVLM(robot)
+    task.reset()
+    task.run()