diff --git a/README.md b/README.md index 68975c1..0a3eb56 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,65 @@ # RoboWaiter 大模型具身智能比赛-机器人控制端 -### 机器人控制 -1. 加载场景 -```python -from scene_utils import control -control.init_world(scene_num=1, mapID=3) -``` -当前只有一个咖啡馆场景。加载操作只需要执行一遍,当引擎进入相应场景后,可以用`control.reset()`重置场景。 +# 项目安装(必看) +## 环境要求 +Python=3.10 -2. 物品类别 +### 安装步骤 +```shell +cd RoboWaiter +pip install -e . +``` +以上步骤将完成robowaiter项目以及相关依赖库的安装 + +### 快速入门 +1. 安装UE及Harix插件,打开默认项目并运行 +2. 运行 run_robowaiter.py 文件即可实现机器人控制端与仿真器的交互 + + +# 运行流程介绍 +run_robowaiter.py 入口文件如下: +```python +import os +from robowaiter import Robot, task_map + +TASK_NAME = 'GQA' + +# create robot +project_path = "./robowaiter" +ptml_path = os.path.join(project_path, 'robot/Default.ptml') +behavior_lib_path = os.path.join(project_path, 'behavior_lib') + +robot = Robot(ptml_path,behavior_lib_path) + +# create task +task = task_map[TASK_NAME](robot) +task.reset() +task.run() +``` + +## Robot +Robot是机器人类,包括从ptml加载行为树的方法,以及执行行为树的方法等 + + +## task_map +task_map是任务字典,通过任务缩写来返回相应的场景类。 + +| 缩写 | 任务 | +|----|---------| +| AEM | 主动探索和记忆 | +| GQA | 具身多轮对话 | +| VLN | 视觉语言导航 | +| VLM | 视觉语言操作 | +| OT | 复杂开放任务 | +| AT | 自主任务 | + + +## Scene +Scene是场景基类,task_map返回的任务场景都继承于Scene。 +该类实现了一些通用的场景操作接口。 + +### 场景中物品类别 | ID | Item | |-----|----------------------| @@ -115,4 +165,11 @@ control.init_world(scene_num=1, mapID=3) | 252 | Floor | | 253 | Roof | | 254 | Wall | -注意:78及以后无法使用add_object方法生成 \ No newline at end of file +注意:78及以后无法使用add_object方法生成 + +# 调用大模型接口 +运行llm_client.py文件调用大模型进行多轮对话。 +```shell +python llm_client.py +``` +输入字符即可等待回答,输入end表示对话结束。 diff --git a/llm_clients.py b/llm_clients.py new file mode 100644 index 0000000..a81b50c --- /dev/null +++ b/llm_clients.py @@ -0,0 +1,29 @@ +import requests +import urllib3 + +######################################## +# 该文件实现了与大模型的简单通信、多轮对话,输入end表示对话结束 +######################################## + +# 忽略https的安全性警告 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +url = "https://45.125.46.134:25344/v1/chat/completions" +headers = {"Content-Type": "application/json"} + +#在这里输入你的问题 +k=input() +data_memory=[] +n=1 +while k!='end': + question_now=k + user_dict={"role": "user","content":question_now} + data_memory.append(user_dict) + #print(data_memory) + response = requests.post(url, headers=headers, json={"messages":data_memory, "repetition_penalty": 1.0}, verify=False) + answer=response.json()['choices'][n]['message']['content'] + print(answer) + assistant_dict={"role": "assistant","content":answer} + data_memory.append(assistant_dict) + n=n+2 + k=input() diff --git a/llm_comm/llm_client.py b/llm_comm/llm_client.py deleted file mode 100644 index 449615c..0000000 --- a/llm_comm/llm_client.py +++ /dev/null @@ -1,28 +0,0 @@ -import requests -import urllib3 - -######################################## -# 该文件实现了与大模型的简单通信 -######################################## - -# 忽略https的安全性警告 -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -#在这里输入你的问题 -question = "假设你是一个咖啡厅的机器人服务员,有一个顾客的请求是'请给我一杯咖啡',请生成对应的行为树来控制机器人完成该动作" - -url = "https://45.125.46.134:25344/v1/completions" -headers = {"Content-Type": "application/json"} -data = { - "prompt": question -} - -response = requests.post(url, headers=headers, json=data, verify=False) - -if response.status_code == 200: - result = response.json() - print(f'问题:{question}') - print('回答:' + result['choices'][0]['text']) -else: - print("请求失败:", response.status_code) - diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7711d13 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +antlr4-python3-runtime +py_trees +shortuuid +protobuf==3.20.0 +gym==0.21.0 +grpcio==1.53.0 +requests +urllib3 diff --git a/robowaiter/__init__.py b/robowaiter/__init__.py new file mode 100644 index 0000000..fe4de55 --- /dev/null +++ b/robowaiter/__init__.py @@ -0,0 +1,3 @@ + +from robowaiter.robot.robot import Robot +from robowaiter.scene import task_map \ No newline at end of file diff --git a/robowaiter/behavior_lib/Behavior.py b/robowaiter/behavior_lib/Behavior.py new file mode 100644 index 0000000..9a4732f --- /dev/null +++ b/robowaiter/behavior_lib/Behavior.py @@ -0,0 +1,22 @@ +import py_trees as ptree +from typing import Any + +# base Behavior +class Bahavior(ptree.behaviour.Behaviour): + scene = None + def __init__(self, name: str, scene): + super().__init__(name) + self.scene = scene + + def setup(self, **kwargs: Any) -> None: + return super().setup(**kwargs) + + def initialise(self) -> None: + return super().initialise() + + def update(self) -> ptree.common.Status: + print("this is a base behavior node.") + return ptree.common.Status.SUCCESS + + def terminate(self, new_status: ptree.common.Status) -> None: + return super().terminate(new_status) diff --git a/ptml/behaviour_lib/CoffeeCupFound.py b/robowaiter/behavior_lib/CoffeeCupFound.py similarity index 79% rename from ptml/behaviour_lib/CoffeeCupFound.py rename to robowaiter/behavior_lib/CoffeeCupFound.py index 0c73896..3357a01 100644 --- a/ptml/behaviour_lib/CoffeeCupFound.py +++ b/robowaiter/behavior_lib/CoffeeCupFound.py @@ -1,10 +1,10 @@ import py_trees as ptree from typing import Any +from robowaiter.behavior_lib.Behavior import Bahavior - -class CoffeeCupFound(ptree.behaviour.Behaviour): +class CoffeeCupFound(Bahavior): def __init__(self, name: str, scene): - super().__init__(name) + super().__init__(name, scene) def setup(self, **kwargs: Any) -> None: return super().setup(**kwargs) diff --git a/ptml/behaviour_lib/CoffeeCupGrasped.py b/robowaiter/behavior_lib/CoffeeCupGrasped.py similarity index 79% rename from ptml/behaviour_lib/CoffeeCupGrasped.py rename to robowaiter/behavior_lib/CoffeeCupGrasped.py index 9485f40..a2e00b8 100644 --- a/ptml/behaviour_lib/CoffeeCupGrasped.py +++ b/robowaiter/behavior_lib/CoffeeCupGrasped.py @@ -1,10 +1,11 @@ import py_trees as ptree from typing import Any +from robowaiter.behavior_lib.Behavior import Bahavior -class CoffeeCupGrasped(ptree.behaviour.Behaviour): - +class CoffeeCupGrasped(Bahavior): def __init__(self, name: str, scene): - super().__init__(name) + super().__init__(name, scene) + def setup(self, **kwargs: Any) -> None: return super().setup(**kwargs) diff --git a/ptml/behaviour_lib/CoffeeCupPlaced.py b/robowaiter/behavior_lib/CoffeeCupPlaced.py similarity index 100% rename from ptml/behaviour_lib/CoffeeCupPlaced.py rename to robowaiter/behavior_lib/CoffeeCupPlaced.py diff --git a/ptml/behaviour_lib/DestinationAReached.py b/robowaiter/behavior_lib/DestinationAReached.py similarity index 100% rename from ptml/behaviour_lib/DestinationAReached.py rename to robowaiter/behavior_lib/DestinationAReached.py diff --git a/ptml/behaviour_lib/FindCoffeeCup.py b/robowaiter/behavior_lib/FindCoffeeCup.py similarity index 100% rename from ptml/behaviour_lib/FindCoffeeCup.py rename to robowaiter/behavior_lib/FindCoffeeCup.py diff --git a/ptml/behaviour_lib/Grasp.py b/robowaiter/behavior_lib/Grasp.py similarity index 100% rename from ptml/behaviour_lib/Grasp.py rename to robowaiter/behavior_lib/Grasp.py diff --git a/ptml/behaviour_lib/GraspCoffeeCup.py b/robowaiter/behavior_lib/GraspCoffeeCup.py similarity index 100% rename from ptml/behaviour_lib/GraspCoffeeCup.py rename to robowaiter/behavior_lib/GraspCoffeeCup.py diff --git a/robowaiter/behavior_lib/IsChatting.py b/robowaiter/behavior_lib/IsChatting.py new file mode 100644 index 0000000..4dd0ce3 --- /dev/null +++ b/robowaiter/behavior_lib/IsChatting.py @@ -0,0 +1,20 @@ +import py_trees as ptree +from typing import Any +from robowaiter.behavior_lib.Behavior import Bahavior + +class IsChatting(Bahavior): + def __init__(self, name: str, scene): + super().__init__(name, scene) + + def setup(self, **kwargs: Any) -> None: + return super().setup(**kwargs) + + def initialise(self) -> None: + return super().initialise() + + def update(self) -> ptree.common.Status: + # if self.scene.status? + return ptree.common.Status.SUCCESS + + def terminate(self, new_status: ptree.common.Status) -> None: + return super().terminate(new_status) diff --git a/ptml/behaviour_lib/Istask.py b/robowaiter/behavior_lib/Istask.py similarity index 100% rename from ptml/behaviour_lib/Istask.py rename to robowaiter/behavior_lib/Istask.py diff --git a/ptml/behaviour_lib/Move.py b/robowaiter/behavior_lib/Move.py similarity index 100% rename from ptml/behaviour_lib/Move.py rename to robowaiter/behavior_lib/Move.py diff --git a/ptml/behaviour_lib/PlaceCoffeeCup.py b/robowaiter/behavior_lib/PlaceCoffeeCup.py similarity index 100% rename from ptml/behaviour_lib/PlaceCoffeeCup.py rename to robowaiter/behavior_lib/PlaceCoffeeCup.py diff --git a/ptml/behaviour_lib/ReachDestinationA.py b/robowaiter/behavior_lib/ReachDestinationA.py similarity index 100% rename from ptml/behaviour_lib/ReachDestinationA.py rename to robowaiter/behavior_lib/ReachDestinationA.py diff --git a/ptml/behaviour_lib/SeqTest.py b/robowaiter/behavior_lib/SeqTest.py similarity index 100% rename from ptml/behaviour_lib/SeqTest.py rename to robowaiter/behavior_lib/SeqTest.py diff --git a/ptml/behaviour_lib/TestTask.py b/robowaiter/behavior_lib/TestTask.py similarity index 100% rename from ptml/behaviour_lib/TestTask.py rename to robowaiter/behavior_lib/TestTask.py diff --git a/behavior_tree/__init__.py b/robowaiter/behavior_lib/__init__.py similarity index 100% rename from behavior_tree/__init__.py rename to robowaiter/behavior_lib/__init__.py diff --git a/robowaiter/behavior_tree/__init__.py b/robowaiter/behavior_tree/__init__.py new file mode 100644 index 0000000..dac458f --- /dev/null +++ b/robowaiter/behavior_tree/__init__.py @@ -0,0 +1,3 @@ + +# from robowaiter.behavior_tree.behavior_tree import BehaviorTree +from robowaiter.behavior_tree.behavior_tree import load_bt_from_ptml \ No newline at end of file diff --git a/robowaiter/behavior_tree/behavior_tree.py b/robowaiter/behavior_tree/behavior_tree.py new file mode 100644 index 0000000..3867407 --- /dev/null +++ b/robowaiter/behavior_tree/behavior_tree.py @@ -0,0 +1,19 @@ +import py_trees as ptree +from robowaiter.behavior_tree.ptml import ptmlCompiler + + +def load_bt_from_ptml(scene, ptml_path, behavior_lib_path): + ptml_bt = ptmlCompiler.load(scene, ptml_path, behavior_lib_path) + bt = ptree.trees.BehaviourTree(ptml_bt) + + with open(ptml_path, 'r') as f: + ptml = f.read() + + print(f'BT loaded: \n {ptml}') + + # print(ptree.display.unicode_tree(root=bt.root, show_status=True)) + return bt + +# class BehaviorTree(ptree): +# def __init__(self): +# super().__init__() \ No newline at end of file diff --git a/ptml/CoffeeDelivery.ptml b/robowaiter/behavior_tree/ptml/CoffeeDelivery.ptml similarity index 95% rename from ptml/CoffeeDelivery.ptml rename to robowaiter/behavior_tree/ptml/CoffeeDelivery.ptml index b21e38b..7f1b11f 100644 --- a/ptml/CoffeeDelivery.ptml +++ b/robowaiter/behavior_tree/ptml/CoffeeDelivery.ptml @@ -1,35 +1,35 @@ -//sequence: -// act action1() -// act action2(2, 2.3, True) -// -// parallel 2: -// act action3(int a, float b) -// act action4() - -sequence{ - selector{ - cond CoffeeCupFound() - task FindCoffeeCup() - sequence{ - cond SeqTest() - task Move(1.2, 2, 2.3, True) - task Grasp() - parallel 3 { - cond Istask() - task TestTask() - } - } - } - selector{ - cond CoffeeCupGrasped() - task GraspCoffeeCup() - } - selector{ - cond DestinationAReached() - task ReachDestinationA() - } - selector{ - cond CoffeeCupPlaced() - task PlaceCoffeeCup() - } -} +//sequence: +// act action1() +// act action2(2, 2.3, True) +// +// parallel 2: +// act action3(int a, float b) +// act action4() + +sequence{ + selector{ + cond CoffeeCupFound() + task FindCoffeeCup() + sequence{ + cond SeqTest() + task Move(1.2, 2, 2.3, True) + task Grasp() + parallel 3 { + cond Istask() + task TestTask() + } + } + } + selector{ + cond CoffeeCupGrasped() + task GraspCoffeeCup() + } + selector{ + cond DestinationAReached() + task ReachDestinationA() + } + selector{ + cond CoffeeCupPlaced() + task PlaceCoffeeCup() + } +} diff --git a/ptml/__init__.py b/robowaiter/behavior_tree/ptml/__init__.py similarity index 100% rename from ptml/__init__.py rename to robowaiter/behavior_tree/ptml/__init__.py diff --git a/ptml/ptml.g4 b/robowaiter/behavior_tree/ptml/ptml.g4 similarity index 100% rename from ptml/ptml.g4 rename to robowaiter/behavior_tree/ptml/ptml.g4 diff --git a/ptml/ptml.interp b/robowaiter/behavior_tree/ptml/ptml.interp similarity index 100% rename from ptml/ptml.interp rename to robowaiter/behavior_tree/ptml/ptml.interp diff --git a/ptml/ptml.tokens b/robowaiter/behavior_tree/ptml/ptml.tokens similarity index 100% rename from ptml/ptml.tokens rename to robowaiter/behavior_tree/ptml/ptml.tokens diff --git a/ptml/ptmlCompiler.py b/robowaiter/behavior_tree/ptml/ptmlCompiler.py similarity index 100% rename from ptml/ptmlCompiler.py rename to robowaiter/behavior_tree/ptml/ptmlCompiler.py diff --git a/ptml/ptmlLexer.interp b/robowaiter/behavior_tree/ptml/ptmlLexer.interp similarity index 100% rename from ptml/ptmlLexer.interp rename to robowaiter/behavior_tree/ptml/ptmlLexer.interp diff --git a/ptml/ptmlLexer.py b/robowaiter/behavior_tree/ptml/ptmlLexer.py similarity index 100% rename from ptml/ptmlLexer.py rename to robowaiter/behavior_tree/ptml/ptmlLexer.py diff --git a/ptml/ptmlLexer.tokens b/robowaiter/behavior_tree/ptml/ptmlLexer.tokens similarity index 100% rename from ptml/ptmlLexer.tokens rename to robowaiter/behavior_tree/ptml/ptmlLexer.tokens diff --git a/ptml/ptmlListener.py b/robowaiter/behavior_tree/ptml/ptmlListener.py similarity index 100% rename from ptml/ptmlListener.py rename to robowaiter/behavior_tree/ptml/ptmlListener.py diff --git a/ptml/ptmlParser.py b/robowaiter/behavior_tree/ptml/ptmlParser.py similarity index 100% rename from ptml/ptmlParser.py rename to robowaiter/behavior_tree/ptml/ptmlParser.py diff --git a/ptml/ptmlTranslator.py b/robowaiter/behavior_tree/ptml/ptmlTranslator.py similarity index 100% rename from ptml/ptmlTranslator.py rename to robowaiter/behavior_tree/ptml/ptmlTranslator.py diff --git a/ptml/ptml_test.py b/robowaiter/behavior_tree/ptml/ptml_test.py similarity index 72% rename from ptml/ptml_test.py rename to robowaiter/behavior_tree/ptml/ptml_test.py index ba18089..bdcd3d2 100644 --- a/ptml/ptml_test.py +++ b/robowaiter/behavior_tree/ptml/ptml_test.py @@ -1,16 +1,16 @@ import os import py_trees as ptree -from scene_utils import control +from scene import scene from ptmlCompiler import load if __name__ == '__main__': - project_path = "." - + project_path = "" + ptml_path = os.path.join(project_path, 'CoffeeDelivery.ptml') - behavior_lib_path = os.path.join(project_path, 'behaviour_lib') + behavior_lib_path = os.path.join(project_path, '../../behavior_lib') scene = control.Scene(sceneID=0) # load diff --git a/ptml/sequence_oufbg8fg.dot b/robowaiter/behavior_tree/ptml/sequence_oufbg8fg.dot similarity index 100% rename from ptml/sequence_oufbg8fg.dot rename to robowaiter/behavior_tree/ptml/sequence_oufbg8fg.dot diff --git a/ptml/behaviour_lib/__init__.py b/robowaiter/llm_client/__init__.py similarity index 100% rename from ptml/behaviour_lib/__init__.py rename to robowaiter/llm_client/__init__.py diff --git a/robowaiter/llm_client/ask_llm.py b/robowaiter/llm_client/ask_llm.py new file mode 100644 index 0000000..15172fd --- /dev/null +++ b/robowaiter/llm_client/ask_llm.py @@ -0,0 +1,31 @@ + +import requests +import urllib3 + +######################################## +# 该文件实现了与大模型的简单通信 +######################################## + +# 忽略https的安全性警告 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +def ask_llm(question): + url = "https://45.125.46.134:25344/v1/completions" + headers = {"Content-Type": "application/json"} + data = { + "prompt": question + } + + response = requests.post(url, headers=headers, json=data, verify=False) + + if response.status_code == 200: + result = response.json() + return result['choices'][0]['text'] + else: + return "大模型请求失败:", response.status_code + + +if __name__ == '__main__': + question = "假设你是一个咖啡厅的机器人服务员,有一个顾客的请求是'请给我一杯咖啡',请生成对应的行为树来控制机器人完成该动作" + + print(ask_llm(question)) \ No newline at end of file diff --git a/proto/GrabSim_pb2.pyd b/robowaiter/proto/GrabSim_pb2.pyd similarity index 100% rename from proto/GrabSim_pb2.pyd rename to robowaiter/proto/GrabSim_pb2.pyd diff --git a/proto/GrabSim_pb2_grpc.pyd b/robowaiter/proto/GrabSim_pb2_grpc.pyd similarity index 100% rename from proto/GrabSim_pb2_grpc.pyd rename to robowaiter/proto/GrabSim_pb2_grpc.pyd diff --git a/proto/__init__.py b/robowaiter/proto/__init__.py similarity index 100% rename from proto/__init__.py rename to robowaiter/proto/__init__.py diff --git a/robowaiter/robot/Default.ptml b/robowaiter/robot/Default.ptml new file mode 100644 index 0000000..c955552 --- /dev/null +++ b/robowaiter/robot/Default.ptml @@ -0,0 +1,105 @@ +//sequence: +// act action1() +// act action2(2, 2.3, True) +// +// parallel 2: +// act action3(int a, float b) +// act action4() + +sequence{ + selector{ + cond CoffeeCupFound() + task FindCoffeeCup() + sequence{ + cond SeqTest() + task Move(1.2, 2, 2.3, True) + task Grasp() + parallel 3 { + cond Istask() + task TestTask() + } + } + } + selector{ + cond CoffeeCupGrasped() + task GraspCoffeeCup() + } + selector{ + cond DestinationAReached() + task ReachDestinationA() + } + selector{ + cond CoffeeCupPlaced() + task PlaceCoffeeCup() + } +} + + + + + + +sequence{ + cond NotAt(Coffee,Table) + selector{ + cond Holding(Coffee) + sequence{ + selector{ + cond At(Bar) + task MoveTo(Bar) + } + task Grasp(Coffee) + } + } + selector{ + cond At(Table) + task MoveTo(Table) + } + Put(Coffee) +} + + +selector{ + sequence{ + cond At(Table) + cond Holding(Coffee) + task Put(Table) + } + sequence{ + cond NotHolding(Coffee) + task MoveTo(Table) + } + sequence{ + cond At(Bar) + task Grasp(Coffee) + } + task MoveTo(Table) + + + + sequence{ + cond Coffee() + task FindCoffeeCup() + sequence{ + cond SeqTest() + task Move(1.2, 2, 2.3, True) + task Grasp() + parallel 3 { + cond Istask() + task TestTask() + } + } + } + selector{ + cond At(Table,Coffee) + task GraspCoffeeCup() + } + selector{ + cond DestinationAReached() + task ReachDestinationA() + } + selector{ + cond CoffeeCupPlaced() + task PlaceCoffeeCup() + } +} diff --git a/robowaiter/robot/__init__.py b/robowaiter/robot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/robowaiter/robot/robot.py b/robowaiter/robot/robot.py new file mode 100644 index 0000000..3cd0667 --- /dev/null +++ b/robowaiter/robot/robot.py @@ -0,0 +1,20 @@ +import os +import py_trees as ptree + +from robowaiter.behavior_tree import load_bt_from_ptml + +class Robot(object): + scene = None + def __init__(self,ptml_path,behavior_lib_path): + self.ptml_path = ptml_path + self.behavior_lib_path = behavior_lib_path + + def set_scene(self,scene): + self.scene = scene + + def load_BT(self): + self.bt = load_bt_from_ptml(self.scene, self.ptml_path,self.behavior_lib_path) + + +if __name__ == '__main__': + pass \ No newline at end of file diff --git a/robowaiter/scene/__init__.py b/robowaiter/scene/__init__.py new file mode 100644 index 0000000..f3578e9 --- /dev/null +++ b/robowaiter/scene/__init__.py @@ -0,0 +1,17 @@ + +from .scene import Scene +from robowaiter.scene.tasks.AEM import SceneAEM +from robowaiter.scene.tasks.GQA import SceneGQA +from robowaiter.scene.tasks.VLN import SceneVLN +from robowaiter.scene.tasks.VLM import SceneVLM +from robowaiter.scene.tasks.Open_tasks import SceneOT +from robowaiter.scene.tasks.Auto_tasks import SceneAT + +task_map = { + "AEM": SceneAEM, + "GQA": SceneGQA, + "VLN": SceneVLN, + "VLM": SceneVLM, + "OT": SceneOT, + "AT": SceneAT, +} \ No newline at end of file diff --git a/scene_utils/control.py b/robowaiter/scene/scene.py similarity index 90% rename from scene_utils/control.py rename to robowaiter/scene/scene.py index ad6e52f..943cfb1 100644 --- a/scene_utils/control.py +++ b/robowaiter/scene/scene.py @@ -1,12 +1,10 @@ import time -import gym import grpc import numpy as np -from proto import GrabSim_pb2 -from proto import GrabSim_pb2_grpc +from robowaiter.proto import GrabSim_pb2 +from robowaiter.proto import GrabSim_pb2_grpc -from ptml import ptmlCompiler channel = grpc.insecure_channel( "localhost:30001", @@ -21,7 +19,7 @@ animation_step = [4, 5, 7, 3, 3] loc_offset = [-700, -1400] -def init_world(scene_num, mapID): +def init_world(scene_num=1, mapID=3): stub.SetWorld(GrabSim_pb2.BatchMap(count=scene_num, mapID=mapID)) time.sleep(3) # wait for the map to load @@ -34,7 +32,21 @@ def image_extract(camera_data): class Scene: + robot = None + state = {} """ + # 当前场景的状态 + state: { + "chat_pool": [ #未处理的顾客的对话池 + { + "pos": 顾客的位置, + "chat": 顾客对话的内容 + } + ], + + "status": # 仿真器中的观测信息,见下方详细解释 + } + status: location: Dict[X: float, Y: float] rotation: Dict[Yaw: float] @@ -47,18 +59,40 @@ class Scene: collision: str, info: str """ - def __init__(self, sceneID=0): + def __init__(self,robot, sceneID=0): self.sceneID = sceneID self.use_offset = True - self.BT = None + + # init robot + robot.set_scene(self) + robot.load_BT() + self.robot = robot + + + def run(self): + pass + @property def status(self): return stub.Observe(GrabSim_pb2.SceneID(value=self.sceneID)) - def reset(self): + def reset_sim(self): stub.Reset(GrabSim_pb2.ResetParams(scene=self.sceneID)) + # reset world + init_world() + + # reset state + self.state = { + "chatting_list": [] + } + + + + def reset(self): + self.reset_sim() + def walker_control_generator(self, walkerID, autowalk, speed, X, Y, Yaw): if self.use_offset: X, Y = X + loc_offset[0], Y + loc_offset[1] @@ -239,5 +273,3 @@ class Scene: def animation_reset(self): stub.ControlRobot(GrabSim_pb2.ControlInfo(scene=self.sceneID, type=0, action=0)) - def load_BT(self, ptml_path, behaviour_lib_path): - self.BT = ptmlCompiler.load(self, ptml_path, behaviour_lib_path) diff --git a/robowaiter/scene/tasks/AEM.py b/robowaiter/scene/tasks/AEM.py new file mode 100644 index 0000000..39a94c7 --- /dev/null +++ b/robowaiter/scene/tasks/AEM.py @@ -0,0 +1,26 @@ +""" +环境主动探索和记忆 +要求输出探索结果(语义地图)对环境重点信息记忆。生成环境的语义拓扑地图,和不少于10个环境物品的识别和位置记忆,可以是图片或者文字或者格式化数据。 +""" + +from robowaiter.scene.scene import Scene + +class SceneAEM(Scene): + def __init__(self, robot): + super().__init__(robot) + + # control.init_world(1, 3) + + def reset(self): + self.reset_sim() + + self.add_object(0, 570, 1600, 85.5) # type与物品编号对应,具体参考README.md + self.add_object(1, 570, 1630, 85.5) + self.add_object(2, 570, 1660, 85.5) + self.add_object(3, 580, 1680, 85.5) + + # todo: 探索并获得语义地图 + print(self.status.objects) # 全部的物品信息,包括名称、位置等,与获得的语义地图进行对比 + + def run(self): + pass \ No newline at end of file diff --git a/robowaiter/scene/tasks/Auto_tasks.py b/robowaiter/scene/tasks/Auto_tasks.py new file mode 100644 index 0000000..dbc4c55 --- /dev/null +++ b/robowaiter/scene/tasks/Auto_tasks.py @@ -0,0 +1,26 @@ +""" +在特定环境下,机器人发现目标,可自主完成任务 +1. 打扫地面:地面有垃圾,机器人主动扫地、清理地面垃圾 +2. 收拾桌子:桌子上的污渍,机器人主动擦桌子 +3. 摆椅子:椅子不正,机器人主动摆正椅子 +4. 开灯:室内光线暗,机器人主动打开房屋的灯 +""" + +# todo: 通过行为树控制自动任务 + +from robowaiter.scene.scene import Scene + + +class SceneAT(Scene): + def __init__(self, robot): + super().__init__(robot) + + def reset(self): + self.reset_sim() + + self.add_walker(1085, 2630, 220) + self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)]) + + def run(self): + self.chat_bubble("顾客说:请给我一杯咖啡") + diff --git a/tasks/GQA.py b/robowaiter/scene/tasks/GQA.py similarity index 50% rename from tasks/GQA.py rename to robowaiter/scene/tasks/GQA.py index 48a0b3c..0fd5ed3 100644 --- a/tasks/GQA.py +++ b/robowaiter/scene/tasks/GQA.py @@ -1,5 +1,5 @@ """ -具身多轮对话 +具身多轮对话 GQA 点餐(order)的对话,咖啡厅服务员可以为客人(NPC)完成点餐基本对话 场景对话(GQA)结合场景:询问卫生间、附近娱乐场所(数据来源自主定义) 开始条件:顾客NPC发出点餐指令 @@ -8,3 +8,21 @@ # todo: 使用大模型进行对话,获得指令信息,适时结束对话 # order = {...} + +from robowaiter.scene.scene import Scene + +class SceneGQA(Scene): + def __init__(self, robot): + super().__init__(robot) + + def reset(self): + self.reset_sim() + + self.add_walker(1085, 2630, 220) + self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)]) + + + def run(self): + self.chat_bubble("顾客说:123546567") + + diff --git a/tasks/Multi_tasks.py b/robowaiter/scene/tasks/Open_tasks.py similarity index 57% rename from tasks/Multi_tasks.py rename to robowaiter/scene/tasks/Open_tasks.py index 9ac2b94..cdcc6c7 100644 --- a/tasks/Multi_tasks.py +++ b/robowaiter/scene/tasks/Open_tasks.py @@ -8,12 +8,24 @@ """ -from scene_utils import control - -# control.init_world(1, 3) - -scene = control.Scene(sceneID=0) - -scene.reset() - # todo: 接收点单信息,大模型生成任务规划 + +from robowaiter.scene.scene import Scene + +class SceneOT(Scene): + def __init__(self, robot): + super().__init__(robot) + + def reset(self): + self.reset_sim() + + self.add_walker(1085, 2630, 220) + self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)]) + + + def run(self): + self.chat_bubble("顾客说:请给我一杯咖啡") + + + + diff --git a/robowaiter/scene/tasks/VLM.py b/robowaiter/scene/tasks/VLM.py new file mode 100644 index 0000000..af5d246 --- /dev/null +++ b/robowaiter/scene/tasks/VLM.py @@ -0,0 +1,36 @@ +""" +视觉语言操作 +机器人根据指令人的指令调节空调,自主探索环境导航到目标点,通过手臂的运动规划能力操作空调,比如开关按钮、调温按钮、显示面板 +""" + +import time +from robowaiter.scene.scene import Scene + + +class SceneVLM(Scene): + def __init__(self, robot): + super().__init__(robot) + + def reset(self): + self.reset_sim() + + self.add_walker(1085, 2630, 220) + self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)]) + + def run(self): + # 空调操作 + self.walk_to(950, 1260, 90) # 没法转向? + # todo: 手臂操作 + time.sleep(5) + self.walk_to(947, 1900, 0) + + # 物品挪动 + # todo: 视觉导航至目标点,操作手臂至可抓位置 + """ + scene.grasp(1, your_objectID) + """ + + # todo: 视觉导航至目标点,找准释放位置 + """ + scene.release(1) + """ diff --git a/robowaiter/scene/tasks/VLN.py b/robowaiter/scene/tasks/VLN.py new file mode 100644 index 0000000..d7793d4 --- /dev/null +++ b/robowaiter/scene/tasks/VLN.py @@ -0,0 +1,39 @@ +""" +视觉语言导航 +识别顾客(NPC)靠近、打招呼、对话、领位导航到适合人数的空闲餐桌 +开始条件:监测到顾客靠近 +结束条件:完成领位,语音:“请问您想喝点什么?”,并等待下一步指令 +""" + +from robowaiter.scene.scene import Scene + + +class SceneVLN(Scene): + def __init__(self, robot): + super().__init__(robot) + + def reset(self): + self.reset_sim() + + self.add_walker(1085, 2630, 220) + self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)]) + + def run(self): + # 实现单顾客领位 + self.add_walker(1085, 2630, 220) + self.control_walker([self.walker_control_generator(0, False, 100, 755, 1900, 180)]) + + # todo: 监测到顾客靠近,打招呼,对话,识别获取空闲餐桌位置 + # 可以使用scene.chat_bubble(message)函数实现对话 + + """ + scene.walk_to(your_free_table_location) + time.sleep(5) + scene.control_walker([scene.walker_control_generator(your_free_table_location)]) + """ + + reach = True + if reach: + self.chat_bubble("请问您想喝点什么?") + + print(self.status.walkers) diff --git a/robowaiter/scene/tasks/__init__.py b/robowaiter/scene/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/run_robowaiter.py b/run_robowaiter.py new file mode 100644 index 0000000..afef9f9 --- /dev/null +++ b/run_robowaiter.py @@ -0,0 +1,16 @@ +import os +from robowaiter import Robot, task_map + +TASK_NAME = 'GQA' + +# create robot +project_path = "./robowaiter" +ptml_path = os.path.join(project_path, 'robot/Default.ptml') +behavior_lib_path = os.path.join(project_path, 'behavior_lib') + +robot = Robot(ptml_path,behavior_lib_path) + +# create task +task = task_map[TASK_NAME](robot) +task.reset() +task.run() \ No newline at end of file diff --git a/scene_utils/__init__.py b/scene_utils/__init__.py deleted file mode 100644 index d22ac13..0000000 --- a/scene_utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import control diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d38c0b1 --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +from setuptools import setup, find_packages +import os + +with open('requirements.txt') as f: + required = f.read().splitlines() + +setup( + name='robowaiter', + version='0.1.0', + packages=['robowaiter'], + install_requires=required, + author='HPCL-EI', + author_email='hpcl_ei@163.com', + description='RoboWaiter', + url='https://github.com/HPCL-EI/RoboWaiter', + classifiers=[ + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + ], +) + diff --git a/tasks/AEM.py b/tasks/AEM.py deleted file mode 100644 index 18baed8..0000000 --- a/tasks/AEM.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -环境主动探索和记忆 -要求输出探索结果(语义地图)对环境重点信息记忆。生成环境的语义拓扑地图,和不少于10个环境物品的识别和位置记忆,可以是图片或者文字或者格式化数据。 -""" - -from scene_utils import control - -# control.init_world(1, 3) - -scene = control.Scene(sceneID=0) - -scene.reset() -scene.add_object(0, 570, 1600, 85.5) # type与物品编号对应,具体参考README.md -scene.add_object(1, 570, 1630, 85.5) -scene.add_object(2, 570, 1660, 85.5) -scene.add_object(3, 580, 1680, 85.5) - -# todo: 探索并获得语义地图 - -print(scene.status.objects) # 全部的物品信息,包括名称、位置等,与获得的语义地图进行对比 diff --git a/tasks/Auto_tasks.py b/tasks/Auto_tasks.py deleted file mode 100644 index f5e1cf9..0000000 --- a/tasks/Auto_tasks.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -在特定环境下,机器人发现目标,可自主完成任务 -1. 打扫地面:地面有垃圾,机器人主动扫地、清理地面垃圾 -2. 收拾桌子:桌子上的污渍,机器人主动擦桌子 -3. 摆椅子:椅子不正,机器人主动摆正椅子 -4. 开灯:室内光线暗,机器人主动打开房屋的灯 -""" - -# todo: 通过行为树控制自动任务 diff --git a/tasks/VLM.py b/tasks/VLM.py deleted file mode 100644 index d9d2af5..0000000 --- a/tasks/VLM.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -视觉语言操作 -机器人根据指令人的指令调节空调,自主探索环境导航到目标点,通过手臂的运动规划能力操作空调,比如开关按钮、调温按钮、显示面板 -""" - -import time -from scene_utils import control - -# control.init_world(1, 3) - -scene = control.Scene(sceneID=0) - -scene.reset() - -# 空调操作 -scene.walk_to(950, 1260, 90) # 没法转向? -# todo: 手臂操作 -time.sleep(5) -scene.walk_to(947, 1900, 0) - -# 物品挪动 -# todo: 视觉导航至目标点,操作手臂至可抓位置 -""" -scene.grasp(1, your_objectID) -""" -# todo: 视觉导航至目标点,找准释放位置 -""" -scene.release(1) -""" diff --git a/tasks/VLN.py b/tasks/VLN.py deleted file mode 100644 index 7c58d49..0000000 --- a/tasks/VLN.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -视觉语言导航 -识别顾客(NPC)靠近、打招呼、对话、领位导航到适合人数的空闲餐桌 -开始条件:监测到顾客靠近 -结束条件:完成领位,语音:“请问您想喝点什么?”,并等待下一步指令 -""" - -from scene_utils import control - -# control.init_world(1, 3) - -scene = control.Scene(sceneID=0) - -# 实现单顾客领位 -scene.reset() -scene.add_walker(1085, 2630, 220) -scene.control_walker([scene.walker_control_generator(0, False, 100, 755, 1900, 180)]) - -# todo: 监测到顾客靠近,打招呼,对话,识别获取空闲餐桌位置 -# 可以使用scene.chat_bubble(message)函数实现对话 - -""" -scene.walk_to(your_free_table_location) -time.sleep(5) -scene.control_walker([scene.walker_control_generator(your_free_table_location)]) -""" - -reach = True -if reach: - scene.chat_bubble("请问您想喝点什么?") - -print(scene.status.walkers) diff --git a/zoo/__init__.py b/zoo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zoo/behavior_tree/__init__.py b/zoo/behavior_tree/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/behavior_tree/behavior_library.py b/zoo/behavior_tree/behavior_library.py similarity index 100% rename from behavior_tree/behavior_library.py rename to zoo/behavior_tree/behavior_library.py diff --git a/behavior_tree/main.py b/zoo/behavior_tree/main.py similarity index 100% rename from behavior_tree/main.py rename to zoo/behavior_tree/main.py diff --git a/zoo/demo/__init__.py b/zoo/demo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/demo/关节控制.py b/zoo/demo/关节控制.py similarity index 100% rename from demo/关节控制.py rename to zoo/demo/关节控制.py diff --git a/demo/动画控制.py b/zoo/demo/动画控制.py similarity index 100% rename from demo/动画控制.py rename to zoo/demo/动画控制.py diff --git a/demo/场景操作.py b/zoo/demo/场景操作.py similarity index 100% rename from demo/场景操作.py rename to zoo/demo/场景操作.py diff --git a/demo/导航寻路.py b/zoo/demo/导航寻路.py similarity index 100% rename from demo/导航寻路.py rename to zoo/demo/导航寻路.py diff --git a/demo/文字冒泡.py b/zoo/demo/文字冒泡.py similarity index 100% rename from demo/文字冒泡.py rename to zoo/demo/文字冒泡.py diff --git a/demo/物品操作.py b/zoo/demo/物品操作.py similarity index 100% rename from demo/物品操作.py rename to zoo/demo/物品操作.py diff --git a/demo/相机操作.py b/zoo/demo/相机操作.py similarity index 100% rename from demo/相机操作.py rename to zoo/demo/相机操作.py diff --git a/demo/行人控制.py b/zoo/demo/行人控制.py similarity index 100% rename from demo/行人控制.py rename to zoo/demo/行人控制.py