Merge branch 'main' of https://github.com/HPCL-EI/RoboWaiter
This commit is contained in:
liwang_zhang
commit
356c99a138
|
|
@ -8,10 +8,6 @@ requests~=2.31.0
|
|||
urllib3~=2.0.7
|
||||
tabulate~=0.9.0
|
||||
autopep8
|
||||
pytorch==1.11.0
|
||||
torchvision==0.12.0
|
||||
torchaudio==0.11.0
|
||||
cudatoolkit=11.3
|
||||
loguru~=0.5.3
|
||||
matplotlib~=3.8.0
|
||||
numpy~=1.26.0
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ class Bahavior(ptree.behaviour.Behaviour):
|
|||
# all_place=set()
|
||||
|
||||
place_xyz_dic={
|
||||
'Bar': (247.0, 520.0, 100.0),
|
||||
'Bar': (247.0, 520.0, 180.0), #(247.0, 520.0, 100.0)
|
||||
'Bar2': (240.0, 40.0, 70.0),
|
||||
'WaterTable':(-70.0, 500.0, 107),
|
||||
'CoffeeTable':(250.0, 310.0, 100.0),
|
||||
|
|
|
|||
|
|
@ -1,48 +1,72 @@
|
|||
import py_trees as ptree
|
||||
from robowaiter.behavior_lib._base.Act import Act
|
||||
from robowaiter.llm_client.ask_llm import ask_llm
|
||||
|
||||
from robowaiter.llm_client.multi_rounds import ask_llm,new_history
|
||||
|
||||
class DealChat(Act):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.chat_history = ""
|
||||
self.function_success = False
|
||||
self.func_map = {
|
||||
"create_sub_task": self.create_sub_task,
|
||||
"get_object_info": self.get_object_info
|
||||
}
|
||||
|
||||
def _update(self) -> ptree.common.Status:
|
||||
# if self.scene.status?
|
||||
name,sentence = self.scene.state['chat_list'][0]
|
||||
name,sentence = self.scene.state['chat_list'].pop(0)
|
||||
|
||||
|
||||
chat = self.scene.state['chat_list'].pop()
|
||||
if isinstance(chat,set):
|
||||
self.create_sub_task(chat)
|
||||
if name == "Goal":
|
||||
self.create_sub_task(goal=sentence)
|
||||
return ptree.common.Status.RUNNING
|
||||
|
||||
if name not in self.scene.state["chat_history"]:
|
||||
self.scene.state["chat_history"][name] = new_history()
|
||||
|
||||
self.chat_history += chat + '\n'
|
||||
history = self.scene.state["chat_history"][name]
|
||||
self.scene.state["attention"]["customer"] = name
|
||||
self.scene.state["serve_state"] = {
|
||||
"last_chat_time": self.scene.time,
|
||||
}
|
||||
|
||||
res_dict = ask_llm(chat)
|
||||
answer = res_dict["Answer"]
|
||||
self.scene.chat_bubble(answer) # 机器人输出对话
|
||||
self.chat_history += answer + '\n'
|
||||
function_call, response = ask_llm(sentence,history,func_map=self.func_map)
|
||||
|
||||
goal = res_dict["Goal"]
|
||||
if goal:
|
||||
if "{" not in goal:
|
||||
goal = {str(goal)}
|
||||
else:
|
||||
goal=eval(goal)
|
||||
|
||||
if goal is not None:
|
||||
print(f'goal：{goal}')
|
||||
|
||||
self.create_sub_task(goal)
|
||||
|
||||
if self.scene.show_bubble:
|
||||
self.scene.chat_bubble(f"{answer}")
|
||||
self.scene.chat_bubble(response) # 机器人输出对话
|
||||
|
||||
return ptree.common.Status.RUNNING
|
||||
|
||||
|
||||
def create_sub_task(self,goal):
|
||||
self.scene.robot.expand_sub_task_tree(goal)
|
||||
def create_sub_task(self,**args):
|
||||
try:
|
||||
goal = args['goal']
|
||||
|
||||
w = goal.split(")")
|
||||
goal_set = set()
|
||||
goal_set.add(w[0] + ")")
|
||||
if len(w)>1:
|
||||
for x in w[1:]:
|
||||
if x != "":
|
||||
goal_set.add(x[1:] + ")")
|
||||
self.function_success = True
|
||||
except:
|
||||
print("参数解析错误")
|
||||
|
||||
self.scene.robot.expand_sub_task_tree(goal_set)
|
||||
|
||||
|
||||
def get_object_info(self,**args):
|
||||
try:
|
||||
obj = args['obj']
|
||||
|
||||
self.function_success = True
|
||||
except:
|
||||
obj = None
|
||||
print("参数解析错误")
|
||||
|
||||
near_object = "None"
|
||||
if obj == "洗手间":
|
||||
near_object = "大门"
|
||||
|
||||
return near_object
|
||||
|
|
@ -9,6 +9,8 @@ class DelSubTree(Act):
|
|||
super().__init__(*args)
|
||||
|
||||
def _update(self) -> ptree.common.Status:
|
||||
self.scene.state["attention"] = {}
|
||||
|
||||
sub_task_tree = self.parent
|
||||
self.scene.sub_task_seq.children.remove(sub_task_tree)
|
||||
return Status.RUNNING
|
||||
|
|
@ -23,9 +23,12 @@ class GreetCustomer(Act):
|
|||
|
||||
goal = Act.place_xyz_dic['Bar']
|
||||
self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0)
|
||||
self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？")
|
||||
# self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？")
|
||||
self.scene.chat_bubble("欢迎光临！")
|
||||
|
||||
customer_name = self.scene.state['attention']['customer']
|
||||
self.scene.state['greeted_customers'].add(customer_name)
|
||||
|
||||
|
||||
|
||||
return ptree.common.Status.RUNNING
|
||||
|
|
|
|||
|
|
@ -3,9 +3,6 @@ from robowaiter.behavior_lib._base.Act import Act
|
|||
from robowaiter.algos.navigator.navigate import Navigator
|
||||
|
||||
class ServeCustomer(Act):
|
||||
can_be_expanded = False
|
||||
num_args = 0
|
||||
valid_args = ()
|
||||
|
||||
def __init__(self, *args):
|
||||
super().__init__(*args)
|
||||
|
|
@ -20,6 +17,8 @@ class ServeCustomer(Act):
|
|||
return info
|
||||
|
||||
def _update(self) -> ptree.common.Status:
|
||||
# if self.scene.time - self.scene.state["serve_state"]["last_chat_time"] > 10:
|
||||
# self.chat_bubble
|
||||
|
||||
goal = Act.place_xyz_dic['Bar']
|
||||
self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
import py_trees as ptree
|
||||
from typing import Any
|
||||
from robowaiter.behavior_lib._base.Cond import Cond
|
||||
|
||||
class CustomerChatting(Cond):
|
||||
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
|
||||
def _update(self) -> ptree.common.Status:
|
||||
# if self.scene.status?
|
||||
if self.scene.state['chat_list'] == []:
|
||||
return ptree.common.Status.FAILURE
|
||||
|
||||
name,sentence = self.scene.state['chat_list'][0]
|
||||
if "customer" in self.scene.state["attention"]:
|
||||
attention_customer = self.scene.state["attention"]["customer"]
|
||||
if name == attention_customer:
|
||||
return ptree.common.Status.SUCCESS
|
||||
else:
|
||||
return ptree.common.Status.FAILURE
|
||||
else:
|
||||
self.scene.state["attention"]["customer"] = name
|
||||
return ptree.common.Status.SUCCESS
|
||||
|
|
@ -2,8 +2,7 @@ import py_trees as ptree
|
|||
from typing import Any
|
||||
from robowaiter.behavior_lib._base.Cond import Cond
|
||||
|
||||
class Chatting(Cond):
|
||||
|
||||
class FocusingCustomer(Cond):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
|
@ -11,7 +10,7 @@ class Chatting(Cond):
|
|||
|
||||
def _update(self) -> ptree.common.Status:
|
||||
# if self.scene.status?
|
||||
if self.scene.state['chat_list'] == []:
|
||||
return ptree.common.Status.FAILURE
|
||||
else:
|
||||
if "customer" in self.scene.state['attention']:
|
||||
return ptree.common.Status.SUCCESS
|
||||
else:
|
||||
return ptree.common.Status.FAILURE
|
||||
|
|
@ -3,7 +3,7 @@ from typing import Any
|
|||
from robowaiter.behavior_lib._base.Cond import Cond
|
||||
import itertools
|
||||
|
||||
class DetectCustomer(Cond):
|
||||
class NewCustomerComing(Cond):
|
||||
can_be_expanded = False
|
||||
num_params = 0
|
||||
valid_args = ()
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
你好
|
||||
您好，我是这家咖啡厅的服务员，请问您要点什么？
|
||||
|
||||
做一杯咖啡
|
||||
好的，我马上做咖啡
|
||||
create_sub_task
|
||||
{"goal":"On(Coffee,CoffeeTable)"}
|
||||
|
||||
不用了
|
||||
好的，您有需要再跟我说
|
||||
stop_serve
|
||||
{}
|
||||
|
||||
来一号桌
|
||||
好的，我马上来一号桌
|
||||
create_sub_task
|
||||
{"goal":"At(Robot,Table1)"}
|
||||
|
||||
|
||||
早上好呀，我想找个能晒太阳的地方。
|
||||
没问题，您右手边就有能晒太阳的位置呢。
|
||||
|
||||
|
||||
可以带我过去嘛？
|
||||
当然，请跟我来！
|
||||
create_sub_task
|
||||
{"goal":"At(Robot,BrightTable1)"}
|
||||
|
||||
|
||||
来杯酸奶吧。
|
||||
好的没问题，请稍等！
|
||||
create_sub_task
|
||||
{"goal":"On(Yogurt,WaterTable)"}
|
||||
|
||||
好热呀！太阳也好大！
|
||||
抱歉，我这就去开空调和关窗帘。
|
||||
create_sub_task
|
||||
{"goal":"Is(AC,On),Is(Curtain,Off)"}
|
||||
|
||||
|
||||
|
||||
请问洗手间在哪里？
|
||||
洗手间在这附近
|
||||
get_object_info
|
||||
{"obj":"洗手间"}
|
||||
|
|
@ -1 +1 @@
|
|||
{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "请问可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}
|
||||
{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！别忘了打扫卫生。": {"Answer": "收到！下班啦，我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}, "给我来杯酸奶和咖啡，哦对，再倒一杯水。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,WaterTable)\",\"On(Water,WaterTable)\"}"}, "好热呀！太阳也好大！": {"Answer": "抱歉！我这就去开空调和关窗帘！", "Goal": "{\"Is(AC,On)\",\"Is(Curtain,Off)\"}"}, "来杯酸奶吧。": {"Answer": "好嘞，请稍等！", "Goal": "{\"On(Yogurt,WaterTable)\"}"}}
|
||||
|
|
|
|||
|
|
@ -20,5 +20,8 @@ Question,Answer,Goal
|
|||
我有点热，能开个空调吗？,当然可以，我现在就开！,"{""Is(AC,On)""}"
|
||||
可以带我去吗,当然可以，前往一号桌,"{""At(Robot,Table1)""}"
|
||||
把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。,明白，我这就去办！,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
|
||||
下班啦！打扫卫生，关灯关空调关窗帘。,太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
|
||||
请问可以带我去空位上嘛？我想晒太阳。,没问题！请跟我来。,"{""At(Robot,BrightTable1)""}"
|
||||
下班啦！别忘了打扫卫生。,收到！下班啦，我这就去打扫卫生，关灯关空调关窗帘。,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
|
||||
可以带我去空位上嘛？我想晒太阳。,没问题！请跟我来。,"{""At(Robot,BrightTable1)""}"
|
||||
给我来杯酸奶和咖啡，哦对，再倒一杯水。,好嘞，请稍等！,"{""On(Coffee,WaterTable)"",""On(Yogurt,WaterTable)"",""On(Water,WaterTable)""}"
|
||||
好热呀！太阳也好大！,抱歉！我这就去开空调和关窗帘！,"{""Is(AC,On)"",""Is(Curtain,Off)""}"
|
||||
来杯酸奶吧。,好嘞，请稍等！,"{""On(Yogurt,WaterTable)""}"
|
||||
|
|
|
|||
|
|
|
@ -1,29 +1,183 @@
|
|||
import json
|
||||
|
||||
import openai
|
||||
from colorama import init, Fore
|
||||
from loguru import logger
|
||||
import json
|
||||
from robowaiter.llm_client.tool_register import get_tools, dispatch_tool
|
||||
import requests
|
||||
import json
|
||||
from collections import deque
|
||||
|
||||
import urllib3
|
||||
from robowaiter.llm_client.tool_api import run_conversation
|
||||
import copy
|
||||
init(autoreset=True)
|
||||
from robowaiter.utils import get_root_path
|
||||
import os
|
||||
import re
|
||||
from robowaiter.llm_client.single_round import single_round
|
||||
########################################
|
||||
# 该文件实现了与大模型的简单通信、多轮对话，输入end表示对话结束
|
||||
# 该文件实现了与大模型的通信以及工具调用
|
||||
########################################
|
||||
|
||||
# 忽略https的安全性警告
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
url = "https://45.125.46.134:25344/v1/chat/completions"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址
|
||||
|
||||
#在这里输入你的问题
|
||||
k=input()
|
||||
data_memory=[]
|
||||
n=1
|
||||
while k!='end':
|
||||
question_now=k
|
||||
user_dict={"role": "user","content":question_now}
|
||||
data_memory.append(user_dict)
|
||||
#print(data_memory)
|
||||
response = run_conversation(str(data_memory))
|
||||
answer=str(response)
|
||||
print(answer)
|
||||
assistant_dict={"role": "assistant","content":answer}
|
||||
data_memory.append(assistant_dict)
|
||||
n=n+2
|
||||
k=input()
|
||||
root_path = get_root_path()
|
||||
# load test questions
|
||||
file_path = os.path.join(root_path,"robowaiter/llm_client/data/fix_questions.txt")
|
||||
|
||||
functions = get_tools()
|
||||
|
||||
fix_questions_dict = {}
|
||||
no_reply_functions = ["create_sub_task"]
|
||||
|
||||
with open(file_path,'r',encoding="utf-8") as f:
|
||||
#读取所有行
|
||||
lines = f.read().strip()
|
||||
sections = re.split(r'\n\s*\n', lines)
|
||||
for s in sections:
|
||||
x = s.split()
|
||||
if len(x) == 2:
|
||||
fix_questions_dict[x[0]] = {
|
||||
"answer": x[1],
|
||||
"function": None
|
||||
}
|
||||
else:
|
||||
fix_questions_dict[x[0]] = {
|
||||
"answer": x[1],
|
||||
"function": x[2],
|
||||
"args": x[3]
|
||||
}
|
||||
|
||||
|
||||
role_system = [{
|
||||
"role": "system",
|
||||
"content": "你是RoboWaiter,一个由HPCL团队开发的机器人服务员，你在咖啡厅工作。接受顾客的指令并调用工具函数来完成各种服务任务。如果顾客问你们这里有什么，或者想要点单，你说我们咖啡厅提供咖啡，水，点心，酸奶等食物。如果顾客不需要你了，你就回到吧台招待。如果顾客叫你去做某事，你回复：好的，我马上去做这件事。",
|
||||
}]
|
||||
|
||||
def new_history(max_length=7):
|
||||
history = deque(maxlen=max_length)
|
||||
|
||||
return history
|
||||
|
||||
def new_response():
|
||||
return {'choices': [{'index': 0, 'message':{} }]}
|
||||
|
||||
def parse_fix_question(question):
|
||||
response = new_response()
|
||||
fix_ans = fix_questions_dict[question]
|
||||
if not fix_ans['function']: #简单对话
|
||||
message = {'role': 'assistant', 'content': fix_ans["answer"], 'name': None,
|
||||
'function_call': None}
|
||||
else:
|
||||
func = fix_ans["function"]
|
||||
args = fix_ans["args"]
|
||||
# tool_response = dispatch_tool(function_call["name"], json.loads(args))
|
||||
# logger.info(f"Tool Call Response: {tool_response}")
|
||||
message = {'role': 'assistant',
|
||||
'content': f"\n <|assistant|> {func}({args})\n ```python\ntool_call(goal={args})\n```",
|
||||
'name': None,
|
||||
'function_call': {'name': func, 'arguments': args}}
|
||||
|
||||
response["choices"][0]["message"] = message
|
||||
return response
|
||||
|
||||
def get_response(sentence, history, allow_function_call = True):
|
||||
if sentence:
|
||||
history.append({"role": "user", "content": sentence})
|
||||
|
||||
if sentence in fix_questions_dict:
|
||||
return parse_fix_question(sentence)
|
||||
|
||||
params = dict(model="RoboWaiter")
|
||||
params['messages'] = role_system + list(history)
|
||||
if allow_function_call:
|
||||
params["functions"] = functions
|
||||
|
||||
|
||||
response = requests.post(f"{base_url}/v1/chat/completions", json=params, stream=False, verify=False)
|
||||
decoded_line = response.json()
|
||||
return decoded_line
|
||||
|
||||
def deal_response(response, history, func_map=None ):
|
||||
if response["choices"][0]["message"].get("function_call"):
|
||||
function_call = response["choices"][0]["message"]["function_call"]
|
||||
logger.info(f"Function Call Response: {function_call}")
|
||||
|
||||
function_name = function_call["name"]
|
||||
function_args = json.loads(function_call["arguments"])
|
||||
if func_map:
|
||||
tool_response = func_map[function_name](**function_args)
|
||||
else:
|
||||
try:
|
||||
tool_response = dispatch_tool(function_call["name"], function_args)
|
||||
logger.info(f"Tool Call Response: {tool_response}")
|
||||
except:
|
||||
logger.info(f"重试工具调用")
|
||||
# tool_response = dispatch_tool(function_call["name"], function_args)
|
||||
return function_name,None
|
||||
|
||||
return_message = response["choices"][0]["message"]
|
||||
|
||||
history.append(return_message)
|
||||
t = {
|
||||
"role": "function",
|
||||
"name": function_call["name"],
|
||||
"content": str(tool_response), # 调用函数返回结果
|
||||
}
|
||||
|
||||
history.append(t)
|
||||
return function_call["name"], tool_response
|
||||
|
||||
else:
|
||||
return_message = response["choices"][0]["message"]
|
||||
reply = return_message["content"]
|
||||
|
||||
history.append(return_message)
|
||||
logger.info(f"Final Reply: \n{reply}")
|
||||
|
||||
return False, reply
|
||||
|
||||
|
||||
def ask_llm(question,history, func_map=None, retry=3):
|
||||
response = get_response(question, history)
|
||||
|
||||
function_call,result = deal_response(response, history, func_map)
|
||||
if function_call:
|
||||
if question in fix_questions_dict:
|
||||
if fix_questions_dict[question]['function'] in no_reply_functions:
|
||||
reply = fix_questions_dict[question]["answer"]
|
||||
result = single_round(reply,
|
||||
"你是机器人服务员，请把以下句子换一种表述方式对顾客说，但是意思不变，尽量简短：\n")
|
||||
else:
|
||||
reply = fix_questions_dict[question]["answer"]
|
||||
result = single_round(f"你是机器人服务员，顾客想知道{question}, 你的具身场景查询返回的是{result},请把按照以下句子对顾客说，{reply}, 尽量简短。\n")
|
||||
|
||||
message = {'role': 'assistant', 'content': result, 'name': None,
|
||||
'function_call': None}
|
||||
history.append(message)
|
||||
|
||||
else:
|
||||
response = get_response(None, history,allow_function_call=False)
|
||||
_,result = deal_response(response, history, func_map)
|
||||
|
||||
|
||||
print(f'{len(history)}条历史记录:')
|
||||
for x in history:
|
||||
print(x)
|
||||
return function_call, result
|
||||
|
||||
if __name__ == "__main__":
|
||||
question = input("\n顾客：")
|
||||
history = new_history()
|
||||
n = 1
|
||||
max_retry = 2
|
||||
|
||||
while question != 'end':
|
||||
function_call, return_message = ask_llm(question,history)
|
||||
|
||||
|
||||
question = input("\n顾客：")
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import urllib3
|
|||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
|
||||
def single_round(question):
|
||||
def single_round(question,prefix=""):
|
||||
url = "https://45.125.46.134:25344/v1/chat/completions"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
data = {
|
||||
|
|
@ -21,7 +21,7 @@ def single_round(question):
|
|||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": question
|
||||
"content": prefix + question
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -147,31 +147,42 @@ def create_sub_task(
|
|||
|
||||
return goal
|
||||
|
||||
# @register_tool
|
||||
# def get_object_info(
|
||||
# obj: Annotated[str, '需要获取信息的物体名称', True]
|
||||
# ) -> str:
|
||||
# """
|
||||
# 获取场景中指定物体 `object` 在哪里，不涉及到具体的执行任务
|
||||
# 如果`object` 是一个地点，例如洗手间，则输出大门。
|
||||
# 如果`object`是咖啡，则输出桌子，咖啡在桌子上。
|
||||
# 如果`object` 是空桌子，则输出一号桌
|
||||
# """
|
||||
# near_object = None
|
||||
# # if obj == "Table":
|
||||
# # near_object = "Bar"
|
||||
# # if obj == "洗手间":
|
||||
# # near_object = "大门"
|
||||
# # if obj == "空桌子":
|
||||
# # near_object = "一号桌"
|
||||
# if obj in find_obj_utils.all_loc: # object是一个地点
|
||||
# mp = list(find_obj_utils.loc_map[obj])
|
||||
# # near_object = random.choice(mp)
|
||||
# near_object = mp
|
||||
# if obj in find_obj_utils.all_obj: # object是一个物品
|
||||
# near_ls = find_obj_utils.all_loc + find_obj_utils.all_obj
|
||||
# near_object = random.choices(near_ls,k=5)
|
||||
# return near_object
|
||||
|
||||
@register_tool
|
||||
def stop_serve(
|
||||
) -> bool:
|
||||
"""
|
||||
当顾客通过任何形式表示不再需要服务时，调用该函数
|
||||
"""
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@register_tool
|
||||
def get_object_info(
|
||||
obj: Annotated[str, '需要获取信息的物体名称', True]
|
||||
) -> str:
|
||||
"""
|
||||
获取场景中指定物体 `object` 在哪里，不涉及到具体的执行任务
|
||||
如果`object` 是一个地点，例如洗手间，则输出大门。
|
||||
如果`object`是咖啡，则输出桌子，咖啡在桌子上。
|
||||
如果`object` 是空桌子，则输出一号桌
|
||||
"""
|
||||
near_object = None
|
||||
# if obj == "Table":
|
||||
# near_object = "Bar"
|
||||
if obj == "洗手间":
|
||||
near_object = "大门"
|
||||
# if obj == "空桌子":
|
||||
# near_object = "一号桌"
|
||||
if obj in find_obj_utils.all_loc: # object是一个地点
|
||||
mp = list(find_obj_utils.loc_map[obj])
|
||||
# near_object = random.choice(mp)
|
||||
near_object = mp
|
||||
if obj in find_obj_utils.all_obj: # object是一个物品
|
||||
near_ls = find_obj_utils.all_loc + find_obj_utils.all_obj
|
||||
near_object = random.choices(near_ls,k=5)
|
||||
return near_object
|
||||
|
||||
# @register_tool
|
||||
# def find_location(
|
||||
|
|
|
|||
|
|
@ -2,9 +2,8 @@ selector
|
|||
{
|
||||
sequence
|
||||
{
|
||||
cond Chatting()
|
||||
cond CustomerChatting()
|
||||
act DealChat()
|
||||
|
||||
}
|
||||
sequence
|
||||
{
|
||||
|
|
@ -16,7 +15,12 @@ selector
|
|||
}
|
||||
sequence
|
||||
{
|
||||
cond DetectCustomer()
|
||||
cond FocusingCustomer()
|
||||
act ServeCustomer()
|
||||
}
|
||||
sequence
|
||||
{
|
||||
cond NewCustomerComing()
|
||||
selector
|
||||
{
|
||||
cond At(Robot,Bar)
|
||||
|
|
@ -24,4 +28,5 @@ selector
|
|||
}
|
||||
act GreetCustomer()
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import io
|
||||
import contextlib
|
||||
import os
|
||||
|
|
@ -41,7 +42,6 @@ class Robot(object):
|
|||
def load_BT(self):
|
||||
self.bt = load_bt_from_ptml(self.scene, self.ptml_path,self.behavior_lib_path)
|
||||
sub_task_place_holder = find_node_by_name(self.bt.root,"SubTaskPlaceHolder()")
|
||||
print(sub_task_place_holder)
|
||||
if sub_task_place_holder:
|
||||
sub_task_seq = sub_task_place_holder.parent
|
||||
sub_task_seq.children.pop()
|
||||
|
|
@ -53,7 +53,7 @@ class Robot(object):
|
|||
|
||||
def expand_sub_task_tree(self,goal):
|
||||
if self.action_list is None:
|
||||
print("\n--------------------")
|
||||
print("\n\n--------------------")
|
||||
print(f"首次运行行为树扩展算法")
|
||||
self.action_list = self.collect_action_nodes()
|
||||
print(f"共收集到{len(self.action_list)}个实例化动作:")
|
||||
|
|
@ -63,9 +63,12 @@ class Robot(object):
|
|||
print("--------------------\n")
|
||||
|
||||
# 如果目标是下班，规划的时候就直接快捷导入？
|
||||
# end_goal = {"Is(Floor,Clean)","Is(Table1,Clean)","Is(Chairs,Clean)","Is(AC,Off)","Is(HallLight,Off)","Is(TubeLight,Off)","Is(Curtain,Off)"}
|
||||
# if goal & end_goal == goal
|
||||
# else:
|
||||
end_goal = {"Is(Floor,Clean)","Is(Table1,Clean)","Is(Chairs,Clean)","Is(AC,Off)","Is(HallLight,Off)","Is(TubeLight,Off)","Is(Curtain,Off)"}
|
||||
if goal & end_goal == goal:
|
||||
tmp_list = copy.deepcopy(self.action_list)
|
||||
self.action_list=[]
|
||||
self.action_list = [action for action in tmp_list if "Turn" in action.name or "Clean" in action.name]
|
||||
|
||||
algo = BTOptExpInterface(self.action_list,self.scene)
|
||||
|
||||
ptml_string = algo.process(goal)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from robowaiter.algos.navigator.navigate import Navigator
|
|||
import math
|
||||
from robowaiter.proto import GrabSim_pb2
|
||||
from robowaiter.proto import GrabSim_pb2_grpc
|
||||
|
||||
import copy
|
||||
import os
|
||||
from robowaiter.utils import get_root_path
|
||||
root_path = get_root_path()
|
||||
|
|
@ -69,7 +69,9 @@ class Scene:
|
|||
"customer_mem":{},
|
||||
"served_mem":{},
|
||||
"greeted_customers":set(),
|
||||
"attention":{}
|
||||
"attention":{},
|
||||
"serve_state":{},
|
||||
"chat_history":{}
|
||||
}
|
||||
"""
|
||||
status:
|
||||
|
|
@ -128,7 +130,7 @@ class Scene:
|
|||
self.reset_sim()
|
||||
|
||||
# reset state
|
||||
self.state = self.default_state
|
||||
self.state = copy.deepcopy(self.default_state)
|
||||
|
||||
|
||||
|
||||
|
|
@ -182,16 +184,18 @@ class Scene:
|
|||
|
||||
return customer_say
|
||||
|
||||
|
||||
|
||||
def set_goal(self,goal):
|
||||
g = eval("{'" + goal + "'}")
|
||||
def set_sub_task():
|
||||
self.state['chat_list'].append(g)
|
||||
self.state['chat_list'].append(("Goal",g))
|
||||
|
||||
return set_sub_task
|
||||
|
||||
def new_set_goal(self,goal):
|
||||
g = eval("{'" + goal + "'}")
|
||||
self.state['chat_list'].append(g)
|
||||
self.state['chat_list'].append(("Goal",g))
|
||||
|
||||
|
||||
@property
|
||||
|
|
@ -283,15 +287,15 @@ class Scene:
|
|||
|
||||
def add_walkers(self,walker_loc=[[0, 880], [250, 1200], [-55, 750], [70, -200]]):
|
||||
print('------------------add_walkers----------------------')
|
||||
for id,walker in enumerate(walker_loc):
|
||||
for i,walker in enumerate(walker_loc):
|
||||
if len(walker)==2:
|
||||
self.add_walker(id,walker[0],walker[1])
|
||||
self.add_walker(i,walker[0],walker[1])
|
||||
elif len(walker)==3:
|
||||
self.add_walker(id, walker[0], walker[1],walker[2])
|
||||
self.add_walker(walker[0], walker[1], walker[2])
|
||||
elif len(walker) == 4:
|
||||
self.add_walker(id, walker[0], walker[1], walker[2], walker[3])
|
||||
self.add_walker(walker[0], walker[1], walker[2], walker[3])
|
||||
elif len(walker) == 5:
|
||||
self.add_walker(id, walker[0], walker[1], walker[2], walker[3], walker[4])
|
||||
self.add_walker(walker[0], walker[1], walker[2], walker[3], walker[4])
|
||||
|
||||
def remove_walker(self, *args): # take single walkerID or a list of walkerIDs
|
||||
remove_list = []
|
||||
|
|
@ -315,11 +319,33 @@ class Scene:
|
|||
def clean_walker(self):
|
||||
stub.CleanWalkers(GrabSim_pb2.SceneID(value=self.sceneID))
|
||||
|
||||
def control_walker(self, control_list):
|
||||
stub.ControlWalkers(
|
||||
def control_walker(self, walkerID,autowalk,speed,X,Y,Yaw=0):
|
||||
pose = GrabSim_pb2.Pose(X=X, Y=Y, Yaw=Yaw)
|
||||
scene = stub.ControlWalkers(
|
||||
GrabSim_pb2.WalkerControls(controls=[GrabSim_pb2.WalkerControls.WControl(id=walkerID, autowalk=autowalk, speed=speed, pose=pose)], scene=self.sceneID)
|
||||
)
|
||||
return scene
|
||||
# stub.ControlWalkers(
|
||||
# GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
|
||||
# )
|
||||
|
||||
def control_walkers_and_say(self, control_list_ls):
|
||||
""" 同时处理行人的行走和对话
|
||||
control_list_ls =[walkerID,autowalk,speed,X,Y,Yaw,cont]
|
||||
"""
|
||||
control_list= []
|
||||
for control in control_list_ls:
|
||||
if control[-1]!= None:
|
||||
walkerID = control[0]
|
||||
# cont = self.status.walkers[walkerID].name + ":"+control[-1]
|
||||
# self.control_robot_action(control[walkerID], 3, cont)
|
||||
self.customer_say(walkerID,control[-1])
|
||||
control_list.append(self.walker_control_generator(walkerID=control[0], autowalk=control[1], speed=control[2], X=control[3], Y=control[4], Yaw=control[5]))
|
||||
# 收集没有对话的统一控制
|
||||
scene = stub.ControlWalkers(
|
||||
GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
|
||||
)
|
||||
|
||||
return scene
|
||||
|
||||
def control_walkers(self,walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True):
|
||||
"""pose:表示行人的终止位置姿态"""
|
||||
|
|
@ -332,7 +358,7 @@ class Scene:
|
|||
pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180)
|
||||
controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=80, pose=pose))
|
||||
scene = stub.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=self.sceneID))
|
||||
|
||||
return scene
|
||||
|
||||
def control_joints(self, angles):
|
||||
stub.Do(
|
||||
|
|
@ -421,7 +447,7 @@ class Scene:
|
|||
def chat_bubble(self, message):
|
||||
stub.ControlRobot(
|
||||
GrabSim_pb2.ControlInfo(
|
||||
scene=self.sceneID, type=0, action=1, content=message
|
||||
scene=self.sceneID, type=0, action=1, content=message.strip()
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -430,6 +456,9 @@ class Scene:
|
|||
self.control_robot_action(0, 3, talk_content)
|
||||
|
||||
def customer_say(self,name,sentence,show_bubble=True):
|
||||
if isinstance(name,int):
|
||||
name = self.walker_index2mem(name)
|
||||
|
||||
print(f'{name} say: {sentence}')
|
||||
if show_bubble:
|
||||
self.walker_bubble(name,sentence)
|
||||
|
|
|
|||
|
|
@ -15,9 +15,8 @@ class SceneGQA(Scene):
|
|||
def __init__(self, robot):
|
||||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
|
||||
self.event_list = [
|
||||
(5, self.create_chat_event("哪里有空桌子")),
|
||||
(12, self.create_chat_event("可以带我去吗")),
|
||||
self.new_event_list = [
|
||||
(3, self.customer_say, ("System","请问洗手间在哪里？"))
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
|
|
@ -18,28 +18,14 @@ class SceneOT(Scene):
|
|||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件
|
||||
self.new_event_list = [
|
||||
# (9,self.add_walkers,([[0, 880]],)),
|
||||
# (10,self.walker_walk_to,(2,50,500))
|
||||
# (5, self.set_goal("On(Yogurt,Table4)"))
|
||||
(3, self.customer_say, ("System","来一号桌"))
|
||||
# (5, self.set_goal("At(Robot,BrightTable4)"))
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
# self.add_walkers([[0, 880], [250, 1200]])
|
||||
pass
|
||||
|
||||
# 展示顾客，前8个id是小孩，后面都是大人
|
||||
for i in range(4):
|
||||
self.add_walker(i,50,300 + i * 50)
|
||||
name1 = self.walker_index2mem(1)
|
||||
name2 = self.walker_index2mem(3)
|
||||
|
||||
self.remove_walker(0,2)
|
||||
|
||||
index1 = self.state["customer_mem"][name1]
|
||||
index2 = self.state["customer_mem"][name2]
|
||||
|
||||
self.walker_bubble(name1,f"我是第{index1}个")
|
||||
self.walker_bubble(name2,f"我是第{index2}个")
|
||||
|
||||
def _run(self):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
"""
|
||||
人提出请求，机器人完成任务
|
||||
1. 做咖啡（固定动画）：接收到做咖啡指令、走到咖啡机、拿杯子、操作咖啡机、取杯子、送到客人桌子上
|
||||
2. 倒水
|
||||
3. 夹点心
|
||||
|
||||
具体描述：设计一套点单规则（如菜单包含咖啡、水、点心等），按照规则拟造随机的订单。在收到订单后，通过大模型让机器人输出合理的备餐计划，并尝试在模拟环境中按照这个规划实现任务。
|
||||
|
||||
"""
|
||||
|
||||
# todo: 接收点单信息，大模型生成任务规划
|
||||
|
||||
from robowaiter.scene.scene import Scene
|
||||
|
||||
class SceneOT(Scene):
|
||||
|
||||
def __init__(self, robot):
|
||||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件
|
||||
self.new_event_list = [
|
||||
# (9,self.add_walkers,([[0, 880]],)),
|
||||
# (10,self.walker_walk_to,(2,50,500))
|
||||
# (5, self.set_goal("On(Yogurt,Table4)"))
|
||||
# (5, self.set_goal("At(Robot,BrightTable4)"))
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
# self.add_walkers([[0, 880], [250, 1200]])
|
||||
|
||||
# 展示顾客，前8个id是小孩，后面都是大人
|
||||
for i in range(4):
|
||||
self.add_walker(i,50,300 + i * 50)
|
||||
name1 = self.walker_index2mem(1)
|
||||
name2 = self.walker_index2mem(3)
|
||||
|
||||
self.remove_walker(0,2)
|
||||
|
||||
index1 = self.state["customer_mem"][name1]
|
||||
index2 = self.state["customer_mem"][name2]
|
||||
|
||||
self.walker_bubble(name1,f"我是第{index1}个")
|
||||
self.walker_bubble(name2,f"我是第{index2}个")
|
||||
|
||||
def _run(self):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import os
|
||||
from robowaiter.robot.robot import Robot
|
||||
|
||||
robot = Robot()
|
||||
|
||||
# create task
|
||||
task = SceneOT(robot)
|
||||
task.reset()
|
||||
task.run()
|
||||
|
|
@ -11,7 +11,7 @@ class SceneVLM(Scene):
|
|||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
|
||||
self.event_list = [
|
||||
(5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
|
||||
# (5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
|
||||
# (10, self.create_chat_event("开空调")),
|
||||
|
||||
# (15, self.create_chat_event("下班啦！打扫卫生，关灯关空调关窗帘。")),
|
||||
|
|
@ -37,13 +37,25 @@ class SceneVLM(Scene):
|
|||
|
||||
def _reset(self):
|
||||
self.gen_obj()
|
||||
|
||||
|
||||
self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)',
|
||||
'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)',
|
||||
'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
|
||||
'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
|
||||
|
||||
self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
|
||||
# [3,1, 880] 1号桌旁边小女孩
|
||||
# [31,250, 1200] 最角落QuietTable1女红色
|
||||
# [6,-55, 750] 1号桌附近小男孩
|
||||
# [10,70, -200] 另一边角落 QuietTable2 男黄色
|
||||
# [27,-290, 400, 180] 中间 BrightTable4 女灰
|
||||
# [26, 60,-320,90] 另一边角落 BrightTable5 红胖男
|
||||
# self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
|
||||
|
||||
# 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220)
|
||||
self.add_walkers([[0, 880], [250, 1200], [-55, 750], [70, -200],[-290, 400, 0],[20, -150,180]])
|
||||
self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
|
||||
# self.add_walkers([[0, 880], [250, 1200], [-55, 750], [70, -200],[-290, 400, 0],[20, -150,180]])
|
||||
# self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
|
||||
|
||||
|
||||
# self.control_walkers(walker_loc=[[-55, 750]],is_autowalk = False)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ class SceneVLM(Scene):
|
|||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
|
||||
self.event_list = [
|
||||
(5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
|
||||
(5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
"""
|
||||
视觉语言操作
|
||||
机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板
|
||||
"""
|
||||
|
||||
import time
|
||||
from robowaiter.scene.scene import Scene
|
||||
|
||||
class SceneVLM(Scene):
|
||||
def __init__(self, robot):
|
||||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
|
||||
# 出现员工和机器人说关门了(机器人可能手上还拿着东西)
|
||||
self.new_event_list = [
|
||||
(3, self.add_walker, (17, 60, 1000)),
|
||||
(6, self.control_walkers_and_say, ([[[6, False, 500, 60, 520, 0, "下班啦！别忘了打扫卫生。"]]])),
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
self.gen_obj()
|
||||
self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)',
|
||||
'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)',
|
||||
'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
|
||||
'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
|
||||
# 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220)
|
||||
self.add_walkers([[3,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
|
||||
# [3,1, 880] 1号桌旁边小女孩
|
||||
# [31,250, 1200] 最角落QuietTable1女红色
|
||||
# [6,-55, 750] 1号桌附近小男孩
|
||||
# [10,70, -200] 另一边角落 QuietTable2 男黄色
|
||||
# [27,-290, 400, 180] 中间 BrightTable4 女灰
|
||||
# [26, 60,-320,90] 另一边角落 BrightTable5 红胖男
|
||||
self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
|
||||
|
||||
# 0-3男孩 4-7女孩 8-26男
|
||||
# 3男孩 31女红 32女灰 10黄色衣服男瘦 9男灰瘦 26红胖男
|
||||
# 17 是员工 police
|
||||
# [0, -150,180]
|
||||
|
||||
pass
|
||||
|
||||
def _run(self, op_type=10):
|
||||
|
||||
pass
|
||||
|
||||
def _step(self):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import os
|
||||
from robowaiter.robot.robot import Robot
|
||||
|
||||
robot = Robot()
|
||||
|
||||
# create task
|
||||
task = SceneVLM(robot)
|
||||
task.reset()
|
||||
task.run()
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
"""
|
||||
视觉语言操作
|
||||
机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板
|
||||
"""
|
||||
|
||||
import time
|
||||
from robowaiter.scene.scene import Scene
|
||||
|
||||
class SceneVLM(Scene):
|
||||
def __init__(self, robot):
|
||||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
|
||||
self.new_event_list = [
|
||||
(3, self.add_walker, (20,0,700)),
|
||||
# (5, self.control_walker, (6, False,100, 60, 520,0)), #[walkerID,autowalk,speed,X,Y,Yaw]
|
||||
# # (10, self.customer_say, (6,"给我来杯酸奶和咖啡，哦对，再倒一杯水。")),
|
||||
# (8, self.customer_say, (6, "来杯酸奶吧。")),
|
||||
# (7, self.control_walker, (6, False, 100, -250, 480, 0)), #(-100,600)
|
||||
|
||||
# 有人提出要开空调和关窗帘
|
||||
# bar (60, 520)
|
||||
# (28, self.add_walker, (0, 0, 0)),
|
||||
# (33, self.control_walker, (7, False, 100, 60, 520, 180)),
|
||||
# (35, self.customer_say, (7,"好热呀！太阳也好大！")),
|
||||
# (45, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])),
|
||||
|
||||
(5, self.add_walker, (0, 0, 0)),
|
||||
(6, self.control_walker, (7, False, 100, 60, 520, 180)),
|
||||
(7, self.customer_say, (7,"好热呀！太阳也好大！")),
|
||||
(8, self.control_walkers_and_say, ([[[7, False, 100, 270, -240, -65, "谢谢，这下凉快了！"]]])),
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
self.gen_obj()
|
||||
self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)',
|
||||
'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)',
|
||||
'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
|
||||
'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
|
||||
# 随机生成4个自由行走，一个在 BrightTable4,BrightTable5(-20,220)
|
||||
self.add_walkers([[4,1, 880], [31,250, 1200],[6,-55, 750],[10,70, -200],[27,-290, 400, 180],[26, 60,-320,90]])
|
||||
# [3,1, 880] 1号桌旁边小女孩
|
||||
# [31,250, 1200] 最角落QuietTable1女红色
|
||||
# [6,-55, 750] 1号桌附近小男孩
|
||||
# [10,70, -200] 另一边角落 QuietTable2 男黄色
|
||||
# [27,-290, 400, 180] 中间 BrightTable4 女灰
|
||||
# [26, 60,-320,90] 另一边角落 BrightTable5 红胖男
|
||||
self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
|
||||
|
||||
# 0-3男孩 4-7女孩 8-26男
|
||||
# 3男孩 31女红 32女灰 10黄色衣服男瘦 9男灰瘦 26红胖男
|
||||
# 17 是员工 police
|
||||
# [0, -150,180]
|
||||
|
||||
|
||||
# self.control_walkers(walker_loc=[[-55, 750]],is_autowalk = False)
|
||||
# 在场景中随机增加一堆行人。
|
||||
# walker_loc = [[-55, 750], [70, -200], [250, 1200], [0, 880]]
|
||||
# controls = []
|
||||
# for i in range(len(s.walkers)):
|
||||
# loc = walker_loc[i]
|
||||
# is_autowalk = False
|
||||
# pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180)
|
||||
# controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=200, pose=pose))
|
||||
# scene = sim_client.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=scene_id))
|
||||
|
||||
# self.gen_obj(type=5)
|
||||
# self.gen_obj(type=9)
|
||||
# self.op_task_execute(op_type=16, obj_id=0)
|
||||
# self.move_task_area(op_type=4)
|
||||
pass
|
||||
|
||||
def _run(self, op_type=10):
|
||||
# 共17个操作
|
||||
# "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7
|
||||
# "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12
|
||||
# "调整空调开关","调高空调温度","调低空调温度", # 13-15
|
||||
# "抓握物体","放置物体" # 16-17
|
||||
#
|
||||
# self.gen_obj()
|
||||
# if op_type <=15:
|
||||
# self.move_task_area(op_type)
|
||||
# self.op_task_execute(op_type)
|
||||
# if op_type == 16: # 16: 抓操作需要传入物品id
|
||||
# self.move_task_area(op_type, obj_id=0)
|
||||
# self.op_task_execute(op_type, obj_id=0)
|
||||
# # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107]
|
||||
# # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0] # 桌子2:[-55.0, 150.0, 107],
|
||||
# if op_type == 17: # 17: 放操作需要传入放置位置周围的可达区域
|
||||
# pos = [240.0, 40.0, 100.0]
|
||||
# self.move_task_area(op_type, release_pos=pos)
|
||||
# self.op_task_execute(op_type, release_pos=pos) # [325.0, 860.0, 100]
|
||||
#
|
||||
# 流程测试
|
||||
# 抓握放置:抓吧台前生成的酸奶，放到抹布桌上
|
||||
# self.gen_obj()
|
||||
# self.move_task_area(16, obj_id=0)
|
||||
# self.op_task_execute(16, obj_id=0)
|
||||
# pos = [340.0, 900.0, 99.0]
|
||||
# self.move_task_area(17, release_pos=pos)
|
||||
# self.op_task_execute(17, release_pos=pos)
|
||||
#
|
||||
# # 做咖啡:做完的咖啡放到水杯桌上
|
||||
# self.move_task_area(1)
|
||||
# self.op_task_execute(1)
|
||||
#
|
||||
# self.find_obj("CoffeeCup")
|
||||
#
|
||||
# self.move_task_area(16, obj_id=275)
|
||||
# self.op_task_execute(16, obj_id=275)
|
||||
# pos = [-70.0, 500.0, 107]
|
||||
# self.move_task_area(17, release_pos=pos)
|
||||
# self.op_task_execute(17, release_pos=pos)
|
||||
#
|
||||
# # 倒水:倒完的水放到旁边桌子上
|
||||
# self.move_task_area(2)
|
||||
# self.op_task_execute(2)
|
||||
#
|
||||
#
|
||||
# self.move_task_area(16, obj_id=190)
|
||||
# self.op_task_execute(16, obj_id=190)
|
||||
# pos = [-55.0, 0.0, 107]
|
||||
# self.move_task_area(17, release_pos=pos)
|
||||
# self.op_task_execute(17, release_pos=pos)
|
||||
#
|
||||
# self.test_yaw()
|
||||
|
||||
pass
|
||||
|
||||
def _step(self):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import os
|
||||
from robowaiter.robot.robot import Robot
|
||||
|
||||
robot = Robot()
|
||||
|
||||
# create task
|
||||
task = SceneVLM(robot)
|
||||
task.reset()
|
||||
task.run()
|
||||
|
|
@ -10,13 +10,15 @@ class SceneVLM(Scene):
|
|||
def __init__(self, robot):
|
||||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
|
||||
self.event_list = [
|
||||
(5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
|
||||
self.new_event_list = [
|
||||
(3, self.add_walker, (5, 230, 1200)),
|
||||
(5, self.control_walkers_and_say, ([[[0, False, 200, 60, 520, 0, "早上好呀，我想找个能晒太阳的地方。"]]])),# (0, 60, 520)),
|
||||
(6, self.customer_say, (0,"可以带我过去嘛？")),
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
self.gen_obj()
|
||||
self.add_walkers([[47, 920]])
|
||||
# self.add_walkers([[47, 920]])
|
||||
pass
|
||||
|
||||
def _run(self, op_type=10):
|
||||
|
|
@ -26,29 +28,23 @@ class SceneVLM(Scene):
|
|||
# 带领行人去有太阳的地方
|
||||
# 行人说 有点热
|
||||
# 好的，这就去开空调
|
||||
self.walker_followed = False
|
||||
pass
|
||||
|
||||
def _step(self):
|
||||
|
||||
self.control_walker(
|
||||
[self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
|
||||
time.sleep(3)
|
||||
cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
|
||||
self.control_robot_action(0,3,cont)
|
||||
|
||||
|
||||
# 如果机器人不在 吧台
|
||||
# if "At(Robot,Bar)" not in self.state['condition_set']:
|
||||
if self.walker_followed:
|
||||
return
|
||||
|
||||
end = [self.status.location.X, self.status.location.Y]
|
||||
# print("end:",end)
|
||||
if end[1]>=600 or end[1]<=450 or end[0]>=250:
|
||||
# if int(self.status.location.X)!=247 or int(self.status.location.X)!=520:
|
||||
self.control_walker(
|
||||
[self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=-90)])
|
||||
|
||||
cont = self.status.walkers[0].name+"谢谢！"
|
||||
self.control_robot_action(0,3,cont)
|
||||
pass
|
||||
self.walker_followed = True
|
||||
self.control_walkers_and_say([[0,False,300,end[0],end[1],90,"谢谢！"]])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
Loading…
Reference in New Issue