Merge remote-tracking branch 'origin/main'
# Conflicts: # robowaiter/scene/scene.py
This commit is contained in:
liwang zhang
commit
0da334d8b4
|
|
@ -1,15 +1,20 @@
|
|||
antlr4-python3-runtime
|
||||
py_trees
|
||||
shortuuid
|
||||
shortuuid~=1.0.11
|
||||
protobuf==3.20.0
|
||||
gym==0.21.0
|
||||
grpcio==1.53.0
|
||||
requests
|
||||
urllib3
|
||||
tabulate
|
||||
requests~=2.31.0
|
||||
urllib3~=2.0.7
|
||||
tabulate~=0.9.0
|
||||
autopep8
|
||||
pytorch==1.11.0
|
||||
torchvision==0.12.0
|
||||
torchaudio==0.11.0
|
||||
cudatoolkit=11.3
|
||||
loguru
|
||||
loguru~=0.5.3
|
||||
matplotlib~=3.8.0
|
||||
numpy~=1.26.0
|
||||
setuptools~=68.0.0
|
||||
pydot~=1.4.2
|
||||
colorama~=0.4.6
|
||||
|
|
@ -13,18 +13,23 @@ class Bahavior(ptree.behaviour.Behaviour):
|
|||
'''
|
||||
scene = None
|
||||
print_name_prefix = ""
|
||||
# all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'}
|
||||
# all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk',
|
||||
# 'VacuumCup'}
|
||||
all_place = {'Bar', 'WaterTable', 'CoffeeTable'}
|
||||
all_place = {'Bar', 'Bar2', 'WaterTable', 'CoffeeTable', 'Table1', 'Table2', 'Table3'}
|
||||
all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'BottledDrink', 'Yogurt', 'ADMilk', 'MilkDrink', 'Milk',
|
||||
'VacuumCup'}
|
||||
|
||||
# all_place = {'Bar', 'WaterTable', 'CoffeeTable'}
|
||||
# all_object = {'Coffee', 'Water', 'Dessert', 'Softdrink', 'Yogurt'}
|
||||
all_object = {'Coffee', 'Water'}
|
||||
# all_object = {'Coffee', 'Water'}
|
||||
|
||||
# all_object = set()
|
||||
# all_place=set()
|
||||
|
||||
place_xyz_dic={
|
||||
'Bar': (247.0, 520.0, 100.0),
|
||||
'Bar2': (240.0, 40.0, 70.0),
|
||||
'WaterTable':(-70.0, 500.0, 107),
|
||||
'CoffeeTable':(250.0, 310.0, 100.0),
|
||||
'Table1': (340.0, 900.0, 98.0),
|
||||
'Table1': (340.0, 900.0, 99.0),
|
||||
'Table2': (-55.0, 0.0, 107),
|
||||
'Table3':(-55.0, 150.0, 107)
|
||||
}
|
||||
|
|
@ -33,6 +38,8 @@ class Bahavior(ptree.behaviour.Behaviour):
|
|||
'Water': 'Glass',
|
||||
'Dessert':'Plate'
|
||||
}
|
||||
|
||||
|
||||
|
||||
@classmethod
|
||||
def get_ins_name(cls,*args):
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class Clean(Act):
|
|||
@classmethod
|
||||
def get_info(cls,arg):
|
||||
info = {}
|
||||
info["pre"]= {f'Holding(Nothing)'}
|
||||
info["pre"]= {f'Holding(Nothing)',f'Is(HallLight,On)'}
|
||||
if arg == "Table1":
|
||||
info["add"]= {f'Is(Table1,Clean)'}
|
||||
info["del_set"] = {f'Is(Table1,Dirty)'}
|
||||
|
|
@ -44,4 +44,6 @@ class Clean(Act):
|
|||
|
||||
self.scene.state["condition_set"] |= (self.info["add"])
|
||||
self.scene.state["condition_set"] -= self.info["del_set"]
|
||||
return Status.SUCCESS
|
||||
|
||||
# print("After Clean condition_set:",self.scene.state["condition_set"] )
|
||||
return Status.RUNNING
|
||||
|
|
@ -11,6 +11,11 @@ class DealChat(Act):
|
|||
def _update(self) -> ptree.common.Status:
|
||||
# if self.scene.status?
|
||||
chat = self.scene.state['chat_list'].pop()
|
||||
if isinstance(chat,set):
|
||||
self.create_sub_task(chat)
|
||||
return ptree.common.Status.RUNNING
|
||||
|
||||
|
||||
self.chat_history += chat + '\n'
|
||||
|
||||
res_dict = ask_llm(chat)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ from typing import Any
|
|||
from robowaiter.behavior_lib._base.Act import Act
|
||||
|
||||
class ExploreEnv(Act):
|
||||
can_be_expanded = True
|
||||
# can_be_expanded = True
|
||||
can_be_expanded = False
|
||||
num_args=0
|
||||
valid_args=()
|
||||
|
||||
|
|
@ -20,8 +21,6 @@ class ExploreEnv(Act):
|
|||
|
||||
def _update(self) -> ptree.common.Status:
|
||||
# explore algorithm
|
||||
|
||||
|
||||
self.scene.state["condition_set"]|= self.info["add"]
|
||||
|
||||
return ptree.common.Status.RUNNING
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
import py_trees as ptree
|
||||
from robowaiter.behavior_lib._base.Act import Act
|
||||
from robowaiter.algos.navigator.navigate import Navigator
|
||||
|
||||
class GreatCustomer(Act):
|
||||
can_be_expanded = False
|
||||
num_args = 0
|
||||
valid_args = ()
|
||||
|
||||
def __init__(self, *args):
|
||||
super().__init__(*args)
|
||||
|
||||
@classmethod
|
||||
def get_info(cls):
|
||||
info = {}
|
||||
info['pre'] = set()
|
||||
info["add"] = set()
|
||||
info["del_set"] = set()
|
||||
info['cost']=0
|
||||
return info
|
||||
|
||||
def _update(self) -> ptree.common.Status:
|
||||
|
||||
goal = Act.place_xyz_dic['Bar']
|
||||
self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0)
|
||||
self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？")
|
||||
return ptree.common.Status.RUNNING
|
||||
|
|
@ -34,6 +34,7 @@ class Make(Act):
|
|||
info["add"] |= {f'On({arg},WaterTable)'}
|
||||
elif arg == cls.valid_args[2]:
|
||||
info["add"] |= {f'On({arg},Bar)'}
|
||||
info['cost'] = 10
|
||||
return info
|
||||
|
||||
def _update(self) -> ptree.common.Status:
|
||||
|
|
@ -57,6 +58,6 @@ class Make(Act):
|
|||
self.scene.state["condition_set"] |= (self.info["add"])
|
||||
self.scene.state["condition_set"] -= self.info["del_set"]
|
||||
|
||||
print("condition_set:",self.scene.state["condition_set"])
|
||||
# print("condition_set:",self.scene.state["condition_set"])
|
||||
|
||||
return Status.RUNNING
|
||||
|
|
@ -53,8 +53,6 @@ class MoveTo(Act):
|
|||
if obj.name == target_name:
|
||||
obj_info = obj_dict[id]
|
||||
dis = self.scene.cal_distance_to_robot(obj_info.location.X, obj_info.location.Y, obj_info.location.Z)
|
||||
if id==275:
|
||||
print("275'dis:",dis)
|
||||
if dis<min_dis:
|
||||
min_dis = dis
|
||||
obj_id = id
|
||||
|
|
@ -64,7 +62,7 @@ class MoveTo(Act):
|
|||
if obj_id == -1:
|
||||
return ptree.common.Status.FAILURE
|
||||
|
||||
print("self.target_place:",self.target_place,"id:",obj_id,"dis:",min_dis)
|
||||
# print("self.target_place:",self.target_place,"id:",obj_id,"dis:",min_dis)
|
||||
self.scene.move_to_obj(obj_id=obj_id)
|
||||
|
||||
# 为了演示，写死咖啡位置
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ class PutDown(Act):
|
|||
info["pre"] = {f'Holding({arg[0]})',f'At(Robot,{arg[1]})'}
|
||||
info["add"] = {f'Holding(Nothing)',f'On({arg[0]},{arg[1]})'}
|
||||
info["del_set"] = {f'Holding({arg[0]})'}
|
||||
|
||||
info['cost'] = 1
|
||||
return info
|
||||
|
||||
|
||||
|
|
@ -36,4 +38,6 @@ class PutDown(Act):
|
|||
|
||||
self.scene.state["condition_set"] |= (self.info["add"])
|
||||
self.scene.state["condition_set"] -= self.info["del_set"]
|
||||
|
||||
# print("After PutDown condition_set:",self.scene.state["condition_set"])
|
||||
return Status.RUNNING
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ class Turn(Act):
|
|||
self.scene.move_task_area(self.op_type)
|
||||
self.scene.op_task_execute(self.op_type)
|
||||
|
||||
self.scene.state["condition_set"].union(self.info["add"])
|
||||
self.scene.state["condition_set"] |= (self.info["add"])
|
||||
self.scene.state["condition_set"] -= self.info["del_set"]
|
||||
|
||||
return Status.RUNNING
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import py_trees as ptree
|
||||
from typing import Any
|
||||
from robowaiter.behavior_lib._base.Cond import Cond
|
||||
import itertools
|
||||
|
||||
class DetectCustomer(Cond):
|
||||
can_be_expanded = False
|
||||
num_params = 0
|
||||
valid_args = ()
|
||||
|
||||
def __init__(self,*args):
|
||||
super().__init__(*args)
|
||||
|
||||
|
||||
def _update(self) -> ptree.common.Status:
|
||||
|
||||
# 获取customer的位置
|
||||
# bar (247.0, 520.0, 100.0)
|
||||
close_to_bar = False
|
||||
scene = self.scene.status
|
||||
for walker in scene.walkers:
|
||||
x, y, yaw = walker.pose.X, walker.pose.Y, walker.pose.Yaw
|
||||
# 到达一定区域就打招呼
|
||||
if y >= 450 and y <= 620 and x >= 40 and x <= 100 and yaw>=-10 and yaw <=10:
|
||||
close_to_bar = True
|
||||
break
|
||||
|
||||
if close_to_bar:
|
||||
return ptree.common.Status.SUCCESS
|
||||
else:
|
||||
return ptree.common.Status.FAILURE
|
||||
|
|
@ -16,6 +16,9 @@ class On(Cond):
|
|||
def _update(self) -> ptree.common.Status:
|
||||
# if self.scene.status?
|
||||
|
||||
# print("self.name:",self.name)
|
||||
# print("On: condition_set:",self.scene.state["condition_set"])
|
||||
|
||||
if self.name in self.scene.state["condition_set"]:
|
||||
return ptree.common.Status.SUCCESS
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -214,5 +214,93 @@ def enumerate_goal_states(total: int):
|
|||
file.write(i + '\n')
|
||||
|
||||
|
||||
def translate_zero_one(i: str) -> str:
|
||||
if 'ACTemperature' in i:
|
||||
i = re.sub('0\)', '调高', i)
|
||||
i = re.sub('1\)', '调低', i)
|
||||
elif 'AC' in i or 'HallLight' in i or 'TubeLight' in i or 'Curtain' in i:
|
||||
i = re.sub('0\)', '关闭', i)
|
||||
i = re.sub('1\)', '打开', i)
|
||||
elif 'Chairs' in i or 'Floor' in i or 'Table' in i:
|
||||
i = re.sub('0\)', '脏', i)
|
||||
i = re.sub('1\)', '打扫干净', i)
|
||||
|
||||
return i
|
||||
|
||||
|
||||
def enumerate_goal_states_with_describe() -> str:
|
||||
with open(os.path.join('./goal_states_with_description.txt'), 'w', encoding='utf-8') as file:
|
||||
# vln
|
||||
count, res = enumerate_predict(['Robot'], Place, 'at')
|
||||
print(count)
|
||||
for i in range(count):
|
||||
tmp = '#' + res[i].split(',')[-1][:-1]
|
||||
file.write(f'{res[i]}\t请你来一下{tmp}。\n')
|
||||
file.write(f'{res[i]}\t请你去一下{tmp}。\n')
|
||||
file.write(f'{res[i]}\t你能去{tmp}那个位置吗？\n')
|
||||
|
||||
# vlm, on
|
||||
count, res = enumerate_predict(Object, Place, 'on')
|
||||
print(count)
|
||||
for i in range(count):
|
||||
tmp = res[i].split(',')
|
||||
obj = '#' + tmp[0][3:]
|
||||
pla = '#' + tmp[-1][:-1]
|
||||
file.write(f'{res[i]}\t请你把{obj}放到{pla}那个位置。\n')
|
||||
file.write(f'{res[i]}\t请你拿一下{obj}到{pla}位置。\n')
|
||||
|
||||
# vlm, is
|
||||
count, res = enumerate_predict(Operable, ['0', '1'], 'is')
|
||||
print(count)
|
||||
for i in res:
|
||||
tmp = i.split(',')
|
||||
thing, op = '#' + tmp[0][3:], '#' + tmp[-1]
|
||||
file.write('%s\t%s\n' % (i, translate_zero_one(f'你能把{thing}{op}一下吗？')))
|
||||
|
||||
# vlm, holding
|
||||
count, res = enumerate_predict(Object + ['Nothing'], None, 'hold')
|
||||
print(count)
|
||||
for i in res:
|
||||
tmp = '#' + i.split('(')[-1][:-1]
|
||||
if tmp == 'Nothing':
|
||||
file.write(f'{i}\t你手里是没有东西的吗？\n')
|
||||
continue
|
||||
file.write(f'{i}\t你能把{tmp}抓在手里吗？\n')
|
||||
file.write(f'{i}\t你能一直拿着{tmp}吗？\n')
|
||||
|
||||
count, res = enumerate_predict(Cookable, Place, 'on')
|
||||
print(count)
|
||||
for i in res:
|
||||
tmp = i.split(',')
|
||||
thing, pla = '#' + tmp[0][3:], '#' + tmp[-1][:-1]
|
||||
|
||||
file.write(f'{i}\t你能制作{thing}并把它端到{pla}这里来吗？\n')
|
||||
file.write(f'{i}\t给我来点{thing}，并把它端到{pla}这里来。\n')
|
||||
return './goal_states_with_description.txt'
|
||||
|
||||
|
||||
from copy import deepcopy
|
||||
def mutex(path: str):
|
||||
with open(os.path.join(path), 'r', encoding='utf-8') as file:
|
||||
lines = "".join(file.readlines())
|
||||
new_line = deepcopy(lines)
|
||||
|
||||
check = ['#Bar2', '#WaterTable', '#CoffeeTable', '#Bar', '#Table1', '#Table2', '#Table3', '#Coffee', '#Water',
|
||||
'#Dessert', '#Softdrink', '#BottledDrink', '#Yogurt', '#ADMilk', '#MilkDrink', '#Milk', '#VacuumCup', '#AC',
|
||||
'#ACTemperature', '#HallLight', '#TubeLight', '#Curtain', '#Chairs', '#Floor', '#Table1']
|
||||
repla = ['#另一个吧台', '#茶水桌', '#咖啡桌', '#吧台', '#第一张桌子', '#第二张桌子', '#第三张桌子', '#咖啡', '#水',
|
||||
'#点心或者甜品', '#软饮料', '#瓶装饮料', '#酸奶', '#AD钙奶', '#牛奶饮料', '#牛奶', '#保温杯', '#空调',
|
||||
'#空调温度', '#大厅灯', '#筒灯', '#窗帘', '#椅子', '#地板', '#第一张桌子']
|
||||
|
||||
for i, j in zip(check, repla):
|
||||
new_line = re.sub(i, j, new_line)
|
||||
new_line = re.sub('#', '', new_line)
|
||||
lines = re.sub('#', '', lines)
|
||||
|
||||
with open(os.path.join(path), 'a', encoding='utf-8') as file:
|
||||
file.write(new_line*13 + lines * 13)
|
||||
|
||||
|
||||
# generate_goal_states(30, 6, 6)
|
||||
enumerate_goal_states(5000)
|
||||
# enumerate_goal_states(5000)
|
||||
mutex(enumerate_goal_states_with_describe())
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -138,14 +138,14 @@ class OptBTExpAlgorithm:
|
|||
break
|
||||
|
||||
if valid:
|
||||
# 把符合条件的动作节点都放到列表里
|
||||
if self.verbose:
|
||||
print("———— -- %s 符合条件放入列表" % actions[i].name)
|
||||
c_attr_node = Leaf(type='cond', content=c_attr, mincost=current_mincost + actions[i].cost)
|
||||
a_attr_node = Leaf(type='act', content=actions[i], mincost=current_mincost + actions[i].cost)
|
||||
cond_anc_pair = CondActPair(cond_leaf=c_attr_node, act_leaf=a_attr_node)
|
||||
self.nodes.append(copy.deepcopy(cond_anc_pair)) # condition node list
|
||||
self.traversed.append(c_attr) # 重点 the set of expanded conditions
|
||||
# 把符合条件的动作节点都放到列表里
|
||||
if self.verbose:
|
||||
print("———— -- %s 符合条件放入列表,对应的c为 %s" % (actions[i].name,c_attr))
|
||||
|
||||
if self.verbose:
|
||||
print("算法结束！\n")
|
||||
|
|
@ -188,13 +188,23 @@ class OptBTExpAlgorithm:
|
|||
|
||||
|
||||
# 树的dfs
|
||||
def dfs_ptml(self,parnode):
|
||||
def dfs_ptml(self,parnode,is_root=False):
|
||||
for child in parnode.children:
|
||||
if isinstance(child, Leaf):
|
||||
if child.type == 'cond':
|
||||
self.ptml_string += "cond "
|
||||
c_set_str = '\n cond '.join(map(str, child.content)) + "\n"
|
||||
self.ptml_string += c_set_str
|
||||
|
||||
if is_root and len(child.content) > 1:
|
||||
# 把多个 cond 串起来
|
||||
self.ptml_string += "sequence{\n"
|
||||
self.ptml_string += "cond "
|
||||
c_set_str = '\n cond '.join(map(str, child.content)) + "\n"
|
||||
self.ptml_string += c_set_str
|
||||
self.ptml_string += '}\n'
|
||||
else:
|
||||
self.ptml_string += "cond "
|
||||
c_set_str = '\n cond '.join(map(str, child.content)) + "\n"
|
||||
self.ptml_string += c_set_str
|
||||
|
||||
elif child.type == 'act':
|
||||
if '(' not in child.content.name:
|
||||
self.ptml_string += 'act ' + child.content.name + "()\n"
|
||||
|
|
@ -212,7 +222,7 @@ class OptBTExpAlgorithm:
|
|||
|
||||
def get_ptml(self):
|
||||
self.ptml_string = "selector{\n"
|
||||
self.dfs_ptml(self.bt.children[0])
|
||||
self.dfs_ptml(self.bt.children[0],is_root=True)
|
||||
self.ptml_string += '}\n'
|
||||
return self.ptml_string
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ class BTOptExpInterface:
|
|||
:return: A PTML string representing the outcome of the behavior tree.
|
||||
"""
|
||||
self.goal = goal
|
||||
self.algo = OptBTExpAlgorithm(verbose=True)
|
||||
self.algo = OptBTExpAlgorithm(verbose=False)
|
||||
self.algo.clear()
|
||||
self.algo.run_algorithm(self.goal, self.actions,self.scene) # 调用算法得到行为树保存至 algo.bt
|
||||
self.ptml_string = self.algo.get_ptml()
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"测试VLM：做一杯咖啡": {"Answer": "测试VLM：做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "测试VLM：做一杯咖啡放到吧台上": {"Answer": "测试VLM：做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "测试VLM：做一杯咖啡放到水杯桌上并倒水": {"Answer": "测试VLM：做一杯咖啡放到水杯桌上并倒水", "Goal": "{\"On(Coffee,WaterTable)\"}"}, "测试VLN：前往2号桌": {"Answer": "测试VLN：前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "测试VLM：倒一杯水": {"Answer": "测试VLM：倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "测试VLM：开空调": {"Answer": "测试VLM：开空调", "Goal": "{\"Is(AC,On)\"}"}, "测试VLM：关空调": {"Answer": "测试VLM：关空调", "Goal": "{\"Is(AC,Off)\"}"}, "测试VLM：关大厅灯": {"Answer": "测试VLM：关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "测试VLM：开大厅灯": {"Answer": "测试VLM：开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "测试VLM：关筒灯": {"Answer": "测试VLM：关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "测试VLM：开筒灯": {"Answer": "测试VLM：开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "测试VLM：关窗帘": {"Answer": "测试VLM：关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "测试VLM：开窗帘": {"Answer": "测试VLM：开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "测试VLM：拖地": {"Answer": "测试VLM：拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "测试VLM：擦桌子": {"Answer": "测试VLM：擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "测试VLM：整理椅子": {"Answer": "测试VLM：整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "测试VLM：把冰红茶放到Table2": {"Answer": "测试VLM：把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}}
|
||||
{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,23 @@
|
|||
Question,Answer,Goal
|
||||
测试VLM：做一杯咖啡,测试VLM：做一杯咖啡,"{""On(Coffee,CoffeeTable)""}"
|
||||
测试VLM：做一杯咖啡放到吧台上,测试VLM：做一杯咖啡放到吧台上,"{""On(Coffee,Bar)""}"
|
||||
测试VLM：做一杯咖啡放到水杯桌上并倒水,测试VLM：做一杯咖啡放到水杯桌上并倒水,"{""On(Coffee,WaterTable)""}"
|
||||
测试VLN：前往2号桌,测试VLN：前往2号桌,"{""At(Robot,Table2)""}"
|
||||
做一杯咖啡,OK，我这就去做一杯咖啡,"{""On(Coffee,CoffeeTable)""}"
|
||||
做一杯咖啡放到吧台上,OK，我这就去做一杯咖啡放到吧台上,"{""On(Coffee,Bar)""}"
|
||||
做一杯咖啡放到水杯桌上，再倒一杯水,OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水,"{""On(Coffee,WaterTable)"",""On(Water,WaterTable)""}"
|
||||
前往2号桌,OK，我这前往2号桌,"{""At(Robot,Table2)""}"
|
||||
测试AEM,测试AEM,"{""EnvExplored()""}"
|
||||
测试VLM：倒一杯水,测试VLM：倒一杯水,"{""On(Water,WaterTable)""}"
|
||||
测试VLM：开空调,测试VLM：开空调,"{""Is(AC,On)""}"
|
||||
测试VLM：关空调,测试VLM：关空调,"{""Is(AC,Off)""}"
|
||||
测试VLM：关大厅灯,测试VLM：关大厅灯,"{""Is(HallLight,Off)""}"
|
||||
测试VLM：开大厅灯,测试VLM：开大厅灯,"{""Is(HallLight,On)""}"
|
||||
测试VLM：关筒灯,测试VLM：关筒灯,"{""Is(TubeLight,Off)""}"
|
||||
测试VLM：开筒灯,测试VLM：开筒灯,"{""Is(TubeLight,On)""}"
|
||||
测试VLM：关窗帘,测试VLM：关窗帘,"{""Is(Curtain,Off)""}"
|
||||
测试VLM：开窗帘,测试VLM：开窗帘,"{""Is(Curtain,On)""}"
|
||||
测试VLM：拖地,测试VLM：拖地,"{""Is(Floor,Clean)""}"
|
||||
测试VLM：擦桌子,测试VLM：擦桌子,"{""Is(Table1,Clean)""}"
|
||||
测试VLM：整理椅子,测试VLM：整理椅子,"{""Is(Chairs,Clean)""}"
|
||||
测试VLM：把冰红茶放到Table2,测试VLM：把冰红茶放到Table2,"{""On(BottledDrink,Table2)""}"
|
||||
倒一杯水,OK，我这就去倒一杯水,"{""On(Water,WaterTable)""}"
|
||||
开空调,OK，我这就去开空调,"{""Is(AC,On)""}"
|
||||
关空调,OK，我这就去关空调,"{""Is(AC,Off)""}"
|
||||
关大厅灯,OK，我这就去关大厅灯,"{""Is(HallLight,Off)""}"
|
||||
开大厅灯,OK，我这就去开大厅灯,"{""Is(HallLight,On)""}"
|
||||
关筒灯,OK，我这就去关筒灯,"{""Is(TubeLight,Off)""}"
|
||||
开筒灯,OK，我这就去开筒灯,"{""Is(TubeLight,On)""}"
|
||||
关窗帘,OK，我这就去关窗帘,"{""Is(Curtain,Off)""}"
|
||||
开窗帘,OK，我这就去开窗帘,"{""Is(Curtain,On)""}"
|
||||
拖地,OK，我这就去拖地,"{""Is(Floor,Clean)""}"
|
||||
擦桌子,OK，我这就去擦桌子,"{""Is(Table1,Clean)""}"
|
||||
整理椅子,OK，我这就去整理椅子,"{""Is(Chairs,Clean)""}"
|
||||
把冰红茶放到Table2,OK，我这就去把冰红茶放到Table2,"{""On(BottledDrink,Table2)""}"
|
||||
我有点热，能开个空调吗？,当然可以，我现在就开！,"{""Is(AC,On)""}"
|
||||
可以带我去吗,当然可以，前往一号桌,"{""At(Robot,Table1)""}"
|
||||
把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。,明白，我这就去办！,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
|
||||
下班啦！打扫卫生，关灯关空调关窗帘。,太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
|
||||
|
|
|
|||
|
|
|
@ -24,4 +24,3 @@ with open(csv_file_path, mode='r', encoding='gbk') as csv_file, \
|
|||
json_str = json.dumps(output_dict, ensure_ascii=False)
|
||||
# 将JSON字符串写入JSONL文件，并添加换行符
|
||||
jsonl_file.write(json_str + '\n')
|
||||
s
|
||||
Binary file not shown.
|
|
@ -0,0 +1,40 @@
|
|||
你能到另一个吧台这边来吗？ create_sub_task {At(Robot,Bar2)} 当然可以，我现在过去，请您稍等
|
||||
哈喽！服务员，在吧台那里借用下充电宝行不行? create_sub_task {At(Robot,Bar)} 没问题!我马上就过去.
|
||||
麻烦拿个空杯子过来。 create_sub_task {At(Robot,WaterTable)} 没问题，马上给您拿一个空杯子过来。
|
||||
你好，我想点一杯咖啡。 create_sub_task {At(Robot,Table3)} 当然，请稍等片刻，我马上为您准备。
|
||||
你好，请问有没有空位？ create_sub_task {At(Robot,Table1)} 是的，我们还有空座位。请随意选择一个位置坐下即可。
|
||||
Hi! there! Could you bring me a milkdrink? Thankyou! create_sub_task {On(MilkDrink,CoffeeTable)} Certainly! I will bring it to you right away.
|
||||
请将甜点和牛奶饮料放到2号桌上。 create_sub_task {On(Dessert,Table2),On(MilkDrink,Table2)} 好的，甜点和牛奶饮料将马上送到2号桌上。
|
||||
请把AD钙奶放到咖啡台上。 create_sub_task {On(ADMilk,CoffeeTable)} 好的，马上为您放置AD钙奶。
|
||||
请给我一瓶瓶装冰红茶，我在旁边的吧台这里。 create_sub_task {On(BottledDrink,Bar2)} 好的，我会马上为您准备一瓶瓶装冰红茶。
|
||||
请给我来杯酸奶，在吧台处。 create_sub_task {On(Yogurt,Bar)} 当然可以，请稍等片刻，我们将为您准备杯酸奶并放置于吧台处。
|
||||
可以打开大厅灯并且关掉筒灯吗? create_sub_task {Is(HallLight,On),Is(TubeLight,Off)} 当然,我会打开大厅的灯并关闭筒灯.
|
||||
请打开空调并调低温度。 create_sub_task {Is(AC,On),Is(ACTemperature,Down)} 好的，空调已经打开，并且温度已经调低了。
|
||||
你好，能不能给我整理一下这边的椅子？ create_sub_task {Is(Chairs,Clean)} 当然，请稍等片刻。
|
||||
可以帮我打开空调吗？这里太热了。 create_sub_task {Is(AC,On)} 当然可以，我会为您打开空调并将温度降低到适宜的程度。
|
||||
这里太暗了，你能帮我打开筒灯吗？ create_sub_task {Is(TubeLight,On)} 当然可以，请稍等片刻。马上为您打开筒灯。
|
||||
请给我一个酸奶。 create_sub_task {Holding(Yogurt)} 好的，请稍等片刻，马上就会有酸奶送到您手中。
|
||||
我想要酸奶和牛奶饮料/真果粒 create_sub_task {Holding(Yogurt),Holding(MilkDrink)} 当然可以。这是您的酸奶和牛奶饮料/真果粒。享用吧！
|
||||
抱歉打扰了，请问有什么推荐的甜点吗？ create_sub_task {Holding(Nothing)} 当然！我们有很多美味的甜点供您选择。让我向您介绍一下...
|
||||
可以帮忙递给我那瓶瓶装冰红茶吗？ create_sub_task {Holding(BottledDrink)} 没问题，这是您要的瓶装冰红茶。
|
||||
你能给我一个保温杯吗？ create_sub_task {Holding(VacuumCup)} 当然，这是您的保温杯。
|
||||
只需要打扫地板就行了，在这里喝我的咖啡。 create_sub_task {On(Coffee,Bar2),Is(Floor,Dirty)} 当然没问题！让我们先打扫地板再享受你的美味咖啡吧！
|
||||
在这个桌子上放置我的咖啡之前能帮忙把它清理下吗？ create_sub_task {On(Coffee,Bar2),Is(Table1,Dirty)} 当然没问题！让我们先将桌子清洁干净再放置您的咖啡。
|
||||
你好!可以给我一个甜点吗? create_sub_task {On(Dessert,CoffeeTable)} 当然!我会为您准备一个甜点.
|
||||
你好，请给我来杯咖啡。可以从吧台那里拿过来吗？ create_sub_task {On(Coffee,Bar)} 当然，请稍等片刻。让我为您准备一杯咖啡并放在吧台上供您享用。
|
||||
嘿，在吧台那里有甜点。麻烦拿一个过来。 create_sub_task {On(Dessert,Bar)} 当然，请稍等片刻，我会为您取一个甜点过来。
|
||||
请问这里有没有牙膏? get_env_info 牙膏，None 不好意思，我们是咖啡厅，暂时没有提供牙膏。如果您需要的话，可以去附近的超市购买。
|
||||
你们这里卫生纸用完了吗?我需要拿几张。 get_env_info 卫生纸，柜台 非常抱歉，目前我们暂时缺货。但是您可以在柜台处领取湿巾来代替使用。给您带来不便深感抱歉。
|
||||
我想要一杯Bernachon牛奶热巧克力,这里能做得到吧? get_env_info Bernachon牛奶热巧克力，咖啡机 非常抱歉，我们的咖啡机暂时不支持制作Bernachon牛奶热巧克力。但是您可以尝试其他口味的饮品或者选择其他喜欢的热巧克力。
|
||||
请问我可以用洗手间吗? get_env_info 洗手间（卫生间） 卫生间门旁边的走廊上方指示牌上标注了“卫生间”字样。当然可以，请沿着卫生间门旁边的走廊前行即可看到指示牌。
|
||||
请问这里有面包卖吗? get_env_info 面包，橱柜 是的，我们这里有面包出售。您可以去橱柜那边看看，选择自己喜欢口感和口味的面包哦！
|
||||
我要买蛋糕,能告诉我蛋糕柜在哪里吗? get_env_info 蛋糕柜，吧台 当然可以！如果您想要购买蛋糕,我们的蛋糕柜就位于咖啡厅的吧台旁边.您只需要沿着吧台走一小段路,就能找到我们精心准备的美味甜点了.希望您享受这个愉快而甜蜜的时刻!
|
||||
请问冰箱在哪里?我需要存放我的食物。 get_env_info 冰箱，橱柜 非常抱歉，我们咖啡厅并没有提供冰箱来为客人存放食物。不过您可以将食物交给服务员，并告知他们需要保存的时间和要求，他们会为您做好安排。
|
||||
我想找一个舒适的沙发坐一下，能告诉我在哪里吗? get_env_info 沙发，休闲区 当然可以！我们咖啡厅内设有专门供客人使用的舒适沙发，您可以前往咖啡厅中央附近的休闲区寻找。
|
||||
你们这里提供免费Wi-Fi服务吗?我需要上网工作。 get_env_info 无线网络，服务台 当然！我们提供免费Wi-Fi服务给所有顾客使用。最接近无线网络信号强度最高处是在服务台周围,如果您需要连接互联网进行工作或其他事情，请尽管前去。
|
||||
请问这里有没有香柜? get_env_info 香柜 非常抱歉，我们这里暂时没有提供香柜服务。不过您可以在附近的商店中寻找哦。
|
||||
能告诉我今天特别推荐哪种咖啡吗？ — — 今天特别推荐我们自家调配的手冲单品咖啡。每日精选不同产区豆子，在专业师傅细心烘焙后现场为您沏制出最佳风味。
|
||||
请问你们在什么时间段营业？ — — 我们从早上8点开始营业到晚上10点结束。欢迎随时来店享用美味的咖啡和舒适环境。
|
||||
今天下午来得人很多,我需要等很久吗？ — — 很抱歉给您带来不便。由于人流较多，可能会有一些等待时间。我们会尽快为您提供服务，请耐心稍候。
|
||||
请问你们还接受支付宝或微信支付吗？ — — 是的，我们支持支付宝和微信支付两种方式。方便快捷地完成付款流程，并确保了安全性。
|
||||
请问附近是否有停车场？如果没有，那么最近哪个停车场离得比较近？ — — 很抱歉，在咖啡馆周围暂时没有专门停车场。但距离不远处就有一个大型商业中心停车场可供选择。
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import os
|
||||
import sys
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.path.append(os.path.join('../../llm_client'))
|
||||
|
||||
from tool_api import run_conversation
|
||||
from tool_register import get_tools
|
||||
|
||||
functions = get_tools()
|
||||
|
||||
with open(os.path.join('./validate_data.txt'), 'r', encoding='utf-8') as file:
|
||||
lines = file.readlines()
|
||||
lines = [line[:-1].split('\t') for line in lines]
|
||||
|
||||
with open(os.path.join('./validate_llm_1.txt'), 'w', encoding='utf-8') as file:
|
||||
for line in tqdm(lines):
|
||||
query = line[0]
|
||||
file.write(str(run_conversation(query=query, stream=False)) + '\n')
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
{'Answer': '\n 好的，请问您需要我帮忙做什么呢？', 'Goal': None}
|
||||
{'Answer': '好的', 'Goal': 'At(ChargingPower,Bar)'}
|
||||
{'Answer': '\n 好的，请告诉我您需要把杯子放在哪里？', 'Goal': None}
|
||||
{'Answer': '\n 好的，我可以帮你点一杯咖啡。请告诉我你想要什么口味的咖啡？', 'Goal': None}
|
||||
{'Answer': '\n 好的，请问您想坐在哪一张桌子呢？', 'Goal': None}
|
||||
{'Answer': "\n Sure! I'll get you a milk drink. What size would you like?<|assistant|> \n How about a large one?", 'Goal': None}
|
||||
{'Answer': '好的', 'Goal': 'At(Dessert, Table2)'}
|
||||
{'Answer': '好的', 'Goal': 'At(Coffee, Stage)'}
|
||||
{'Answer': '\n 根据您的要求，我已经在旁边找到了一瓶瓶装冰红茶，放在了对应的桌子上。请您享用！', 'Goal': None}
|
||||
{'Answer': '好的', 'Goal': 'At(Bar,Yogurt)'}
|
||||
{'Answer': '\n 当前场景中没有找到名称为“HallLight”的物体。', 'Goal': None}
|
||||
{'Answer': '好的', 'Goal': 'Is(AC,On)'}
|
||||
{'Answer': '\n 当然可以，请问你需要整理多少把椅子呢？', 'Goal': None}
|
||||
{'Answer': '\n 好的，我可以帮你打开空调。请问你在哪个房间？', 'Goal': None}
|
||||
{'Answer': '\n 好的，我会为您打开筒灯。请告诉我筒灯在哪里。', 'Goal': None}
|
||||
{'Answer': '好的', 'Goal': 'Have(yogurt,1)'}
|
||||
{'Answer': '好的', 'Goal': 'Have Yogurt and Milk Beverage with True Fruit Grape'}
|
||||
{'Answer': '\n 你好，我可以帮你推荐一些甜点。请问你喜欢的口味是什么？', 'Goal': None}
|
||||
{'Answer': '\n 很抱歉，我无法找到您要求的瓶装冰红茶。', 'Goal': None}
|
||||
{'Answer': '\n 好的，请告诉我您的位置，我将为您送过去。', 'Goal': None}
|
||||
{'Answer': '好的', 'Goal': 'Is(Floor,Clean)'}
|
||||
{'Answer': '好的', 'Goal': 'At(Coordinates,OneTable)'}
|
||||
{'Answer': '\n 当然可以，请问您想要什么类型的甜点呢？', 'Goal': None}
|
||||
{'Answer': '\n 当然可以，您想要什么口味的咖啡呢？', 'Goal': None}
|
||||
{'Answer': '\n 好的，请告诉我您想要哪个口味的甜点呢？', 'Goal': None}
|
||||
{'Answer': '\n 这里的牙膏放在了桌子的抽屉里。', 'Goal': None}
|
||||
{'Answer': '\n 好的，请问你需要多少张卫生纸呢？', 'Goal': None}
|
||||
{'Answer': '好的', 'Goal': 'At(Bar,Bernachon牛奶热巧克力)'}
|
||||
{'Answer': '\n 您不能使用洗手间。', 'Goal': None}
|
||||
{'Answer': '\n 很抱歉，根据您的查询，我们无法找到任何面包。', 'Goal': None}
|
||||
{'Answer': '\n 很抱歉，我无法找到蛋糕柜的位置。', 'Goal': None}
|
||||
{'Answer': '\n 您可以到厨房里找一下冰箱，一般来说它都会放在 cabinets 和 countertop 之间。', 'Goal': None}
|
||||
{'Answer': '\n 很抱歉，我无法找到舒适的沙发。', 'Goal': None}
|
||||
{'Answer': '\n 你好，是的，我们这里提供免费的Wi-Fi服务。请问你需要连接吗？', 'Goal': None}
|
||||
{'Answer': '\n 根据您的询问，我查询了当前场景中的物品信息，但是没有找到香柜的存在。', 'Goal': None}
|
||||
{'Answer': '\n 你好，我可以帮你推荐一些咖啡。请问你想要什么口味的咖啡呢？', 'Goal': None}
|
||||
{'Answer': '\n 根据我所了解到的信息，这家商店的营业时间没有具体的规定。不过，通常来说，商店的营业时间一般是在早上8点至晚上8点之间。如果您需要了解更多详细的信息，建议您直接联系商店的管理人员。', 'Goal': None}
|
||||
{'Answer': '\n 根据当前情况，您可能需要等待一段时间。', 'Goal': None}
|
||||
{'Answer': '\n 我目前没有收微信或支付宝的功能。不过我们这里支持的是现金支付和银行卡支付。', 'Goal': None}
|
||||
{'Answer': '\n 很抱歉，我无法找到附近的停车场和最近的一个停车场。建议您使用地图应用或询问周围的居民来获取相关信息。', 'Goal': None}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import json
|
||||
import re
|
||||
|
||||
import openai
|
||||
from colorama import init, Fore
|
||||
from loguru import logger
|
||||
import json
|
||||
|
|
@ -9,6 +9,7 @@ import requests
|
|||
import json
|
||||
|
||||
import urllib3
|
||||
|
||||
init(autoreset=True)
|
||||
|
||||
########################################
|
||||
|
|
@ -18,7 +19,8 @@ init(autoreset=True)
|
|||
# 忽略https的安全性警告
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址
|
||||
base_url = "https://45.125.46.134:25344" # 本地部署的地址,或者使用你访问模型的API地址
|
||||
|
||||
|
||||
def get_response(**kwargs):
|
||||
data = kwargs
|
||||
|
|
@ -27,12 +29,14 @@ def get_response(**kwargs):
|
|||
decoded_line = response.json()
|
||||
return decoded_line
|
||||
|
||||
|
||||
functions = get_tools()
|
||||
|
||||
def run_conversation(query: str, stream=False, max_retry=5):
|
||||
|
||||
def run_conversation(query: str, stream=False, max_retry=5):
|
||||
params = dict(model="chatglm3", messages=[{"role": "user", "content": query}], stream=stream)
|
||||
params["functions"] = functions
|
||||
print(params)
|
||||
# print(params)
|
||||
response = get_response(**params)
|
||||
|
||||
for _ in range(max_retry):
|
||||
|
|
@ -69,7 +73,42 @@ def run_conversation(query: str, stream=False, max_retry=5):
|
|||
response = get_response(**params)
|
||||
|
||||
|
||||
def run_conversation_for_test_only(query: str, stream=False, max_retry=5):
|
||||
params = dict(model="chatglm3", messages=[{"role": "user", "content": query}], stream=stream)
|
||||
params["functions"] = functions
|
||||
response = get_response(**params)
|
||||
|
||||
response_string = ''
|
||||
|
||||
for _ in range(max_retry):
|
||||
if response["choices"][0]["message"].get("function_call"):
|
||||
function_call = response["choices"][0]["message"]["function_call"]
|
||||
response_string += f"Function Call: {function_call} \t"
|
||||
|
||||
function_args = json.loads(function_call["arguments"])
|
||||
if function_call["name"]:
|
||||
tool_response = dispatch_tool(function_call["name"], function_args)
|
||||
response_string += f"Tool Call: %s \t" % (re.sub(r'\n', '', tool_response))
|
||||
else:
|
||||
response_string += f"LLM Cannot find the function call."
|
||||
|
||||
params["messages"].append(response["choices"][0]["message"])
|
||||
params["messages"].append(
|
||||
{
|
||||
"role": "function",
|
||||
"name": function_call["name"],
|
||||
"content": tool_response, # 调用函数返回结果
|
||||
}
|
||||
)
|
||||
response = get_response(**params)['choices'][0]
|
||||
return response_string + "\tResponse: " + str(response)
|
||||
else:
|
||||
reply = response["choices"][0]["message"]["content"]
|
||||
response_string += f"Final Reply: %s" % (re.sub(r'\n', '', reply))
|
||||
response = get_response(**params)['choices'][0]
|
||||
return response_string + "\tResponse: " + str(response)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
query = "可以带我去吗"
|
||||
print(run_conversation(query, stream=False))
|
||||
print(run_conversation_for_test_only(query, stream=False))
|
||||
|
|
|
|||
|
|
@ -1,13 +1,27 @@
|
|||
selector
|
||||
{
|
||||
sequence
|
||||
{
|
||||
{
|
||||
cond Chatting()
|
||||
act DealChat()
|
||||
|
||||
} sequence
|
||||
{
|
||||
}
|
||||
sequence
|
||||
{
|
||||
cond HasSubTask()
|
||||
sequence
|
||||
{
|
||||
act SubTaskPlaceHolder()}}}
|
||||
{
|
||||
act SubTaskPlaceHolder()
|
||||
}
|
||||
}
|
||||
sequence
|
||||
{
|
||||
selector
|
||||
{
|
||||
cond At(Robot,Bar)
|
||||
act MoveTo(Bar)
|
||||
}
|
||||
cond DetectCustomer()
|
||||
act GreatCustomer
|
||||
}
|
||||
}
|
||||
|
|
@ -61,7 +61,10 @@ class Robot(object):
|
|||
# print(a.name)
|
||||
print("--------------------\n")
|
||||
|
||||
|
||||
# 如果目标是下班，规划的时候就直接快捷导入？
|
||||
# end_goal = {"Is(Floor,Clean)","Is(Table1,Clean)","Is(Chairs,Clean)","Is(AC,Off)","Is(HallLight,Off)","Is(TubeLight,Off)","Is(Curtain,Off)"}
|
||||
# if goal & end_goal == goal
|
||||
# else:
|
||||
algo = BTOptExpInterface(self.action_list,self.scene)
|
||||
|
||||
ptml_string = algo.process(goal)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
"""
|
||||
人提出请求，机器人完成任务
|
||||
1. 做咖啡（固定动画）：接收到做咖啡指令、走到咖啡机、拿杯子、操作咖啡机、取杯子、送到客人桌子上
|
||||
2. 倒水
|
||||
3. 夹点心
|
||||
|
||||
具体描述：设计一套点单规则（如菜单包含咖啡、水、点心等），按照规则拟造随机的订单。在收到订单后，通过大模型让机器人输出合理的备餐计划，并尝试在模拟环境中按照这个规划实现任务。
|
||||
|
||||
"""
|
||||
|
||||
# todo: 接收点单信息，大模型生成任务规划
|
||||
|
||||
from robowaiter.scene.scene import Scene
|
||||
|
||||
class SubScene(Scene):
|
||||
|
||||
def __init__(self, robot):
|
||||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件
|
||||
|
||||
def _reset(self):
|
||||
pass
|
||||
|
||||
|
||||
def _step(self):
|
||||
if len(self.sub_task_seq.children) == 0:
|
||||
question = input("请输入指令：")
|
||||
if question[-1] == ")":
|
||||
print(f"设置目标:{question}")
|
||||
self.set_goal(question)()
|
||||
else:
|
||||
self.state['chat_list'].append(question)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
from robowaiter.robot.robot import Robot
|
||||
|
||||
robot = Robot()
|
||||
|
||||
# create task
|
||||
task = SubScene(robot)
|
||||
task.reset()
|
||||
task.run()
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
import pickle
|
||||
import sys
|
||||
import time
|
||||
import grpc
|
||||
|
|
@ -10,6 +9,9 @@ import math
|
|||
from robowaiter.proto import GrabSim_pb2
|
||||
from robowaiter.proto import GrabSim_pb2_grpc
|
||||
|
||||
|
||||
|
||||
|
||||
channel = grpc.insecure_channel(
|
||||
"localhost:30001",
|
||||
options=[
|
||||
|
|
@ -30,13 +32,13 @@ def init_world(scene_num=1, mapID=11):
|
|||
|
||||
def show_image(camera_data):
|
||||
print('------------------show_image----------------------')
|
||||
# 获取第0张照片
|
||||
#获取第0张照片
|
||||
im = camera_data.images[0]
|
||||
# 使用numpy(np) 数值类型矩阵的frombuffer，将im.data以流的形式（向量的形式）读入，在变型reshape成三位矩阵的形式(长度，宽度，深度)即三阶张量
|
||||
#使用numpy(np) 数值类型矩阵的frombuffer，将im.data以流的形式（向量的形式）读入，在变型reshape成三位矩阵的形式(长度，宽度，深度)即三阶张量
|
||||
d = np.frombuffer(im.data, dtype=im.dtype).reshape((im.height, im.width, im.channels))
|
||||
# matplotlib中的plt方法 对矩阵d 进行图形绘制，如果 深度相机拍摄的带深度的图片（图片名字中有depth信息），则转换成黑白图即灰度图
|
||||
#matplotlib中的plt方法 对矩阵d 进行图形绘制，如果 深度相机拍摄的带深度的图片（图片名字中有depth信息），则转换成黑白图即灰度图
|
||||
plt.imshow(d, cmap="gray" if "depth" in im.name.lower() else None)
|
||||
# 图像展示在屏幕上
|
||||
#图像展示在屏幕上
|
||||
plt.show()
|
||||
|
||||
return d
|
||||
|
|
@ -56,10 +58,10 @@ class Scene:
|
|||
"sub_goal_list": [], # 子目标列表
|
||||
"status": None, # 仿真器中的观测信息，见下方详细解释
|
||||
"condition_set": {'At(Robot,Bar)', 'Is(AC,Off)',
|
||||
'Holding(Nothing)',
|
||||
# 'Holding(Yogurt)'
|
||||
'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
|
||||
'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
|
||||
'Holding(Nothing)',
|
||||
# 'Holding(Yogurt)'
|
||||
'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
|
||||
'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
|
||||
}
|
||||
"""
|
||||
status:
|
||||
|
|
@ -74,7 +76,7 @@ class Scene:
|
|||
collision: str, info: str
|
||||
"""
|
||||
|
||||
def __init__(self, robot=None, sceneID=0):
|
||||
def __init__(self,robot=None, sceneID=0):
|
||||
self.sceneID = sceneID
|
||||
self.use_offset = False
|
||||
self.start_time = time.time()
|
||||
|
|
@ -89,29 +91,26 @@ class Scene:
|
|||
|
||||
# myx op
|
||||
# 1-7 正常执行, 8-10 控灯操作移动到6, 11-12窗帘操作不需要移动,
|
||||
self.op_dialog = ["", "制作咖啡", "倒水", "夹点心", "拖地", "擦桌子", "开筒灯", "搬椅子", # 1-7
|
||||
"关筒灯", "开大厅灯", "关大厅灯", "关闭窗帘", "打开窗帘", # 8-12
|
||||
"调整空调开关", "调高空调温度", "调低空调温度", # 13-15
|
||||
"抓握物体", "放置物体"] # 16-17
|
||||
self.op_act_num = [0, 3, 4, 6, 3, 2, 0, 1,
|
||||
0, 0, 0, 0, 0,
|
||||
0, 0, 0,
|
||||
0, 0]
|
||||
self.op_v_list = [[0.0, 0.0], [250.0, 310.0], [-70.0, 480.0], [250.0, 630.0], [-70.0, 740.0], [260.0, 1120.0],
|
||||
[300.0, -220.0],
|
||||
self.op_dialog = ["","制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7
|
||||
"关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12
|
||||
"调整空调开关","调高空调温度","调低空调温度", # 13-15
|
||||
"抓握物体","放置物体"] # 16-17
|
||||
self.op_act_num = [0,3,4,6,3,2,0,1,
|
||||
0,0,0,0,0,
|
||||
0,0,0,
|
||||
0,0]
|
||||
self.op_v_list = [[0.0,0.0],[250.0, 310.0],[-70.0, 480.0],[250.0, 630.0],[-70.0, 740.0],[260.0, 1120.0],[300.0, -220.0],
|
||||
[0.0, -70.0]]
|
||||
self.op_typeToAct = {8: [6, 2], 9: [6, 3], 10: [6, 4], 11: [8, 1], 12: [8, 2]}
|
||||
self.op_typeToAct = {8:[6,2],9:[6,3],10:[6,4],11:[8,1],12:[8,2]}
|
||||
# 空调面板位置
|
||||
self.obj_loc = [300.5, -140.0, 114]
|
||||
self.obj_loc = [300.5, -140.0,114]
|
||||
|
||||
# AEM
|
||||
self.visited = set()
|
||||
self.all_frontier_list = set()
|
||||
self.semantic_map = semantic_map
|
||||
self.auto_map = np.ones((800, 1550))
|
||||
self.filename = "../proto/map_1.pkl"
|
||||
with open(self.filename, 'rb') as file:
|
||||
self.map_file = pickle.load(file)
|
||||
|
||||
|
||||
def reset(self):
|
||||
# 基类reset，默认执行仿真器初始化操作
|
||||
|
|
@ -119,6 +118,9 @@ class Scene:
|
|||
|
||||
# reset state
|
||||
self.state = self.default_state
|
||||
|
||||
|
||||
|
||||
print("场景初始化完成")
|
||||
self._reset()
|
||||
|
||||
|
|
@ -142,15 +144,15 @@ class Scene:
|
|||
self.robot.step()
|
||||
|
||||
def deal_event(self):
|
||||
if len(self.event_list) > 0:
|
||||
if len(self.event_list)>0:
|
||||
next_event = self.event_list[0]
|
||||
t, func = next_event
|
||||
t,func = next_event
|
||||
if self.time >= t:
|
||||
print(f'event: {t}, {func.__name__}')
|
||||
self.event_list.pop(0)
|
||||
func()
|
||||
|
||||
def create_chat_event(self, sentence):
|
||||
def create_chat_event(self,sentence):
|
||||
def customer_say():
|
||||
print(f'顾客说：{sentence}')
|
||||
if self.show_bubble:
|
||||
|
|
@ -159,6 +161,14 @@ class Scene:
|
|||
|
||||
return customer_say
|
||||
|
||||
def set_goal(self,goal):
|
||||
g = eval("{'" + goal + "'}")
|
||||
def set_sub_task():
|
||||
self.state['chat_list'].append(g)
|
||||
|
||||
return set_sub_task
|
||||
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
return stub.Observe(GrabSim_pb2.SceneID(value=self.sceneID))
|
||||
|
|
@ -166,7 +176,7 @@ class Scene:
|
|||
def reset_sim(self):
|
||||
# reset world
|
||||
init_world()
|
||||
|
||||
|
||||
stub.Reset(GrabSim_pb2.ResetParams(scene=self.sceneID))
|
||||
|
||||
def _reset(self):
|
||||
|
|
@ -181,6 +191,10 @@ class Scene:
|
|||
# 场景自定义的step
|
||||
pass
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def walker_control_generator(self, walkerID, autowalk, speed, X, Y, Yaw):
|
||||
if self.use_offset:
|
||||
X, Y = X + loc_offset[0], Y + loc_offset[1]
|
||||
|
|
@ -192,7 +206,7 @@ class Scene:
|
|||
)
|
||||
|
||||
def walk_to(self, X, Y, Yaw=100, velocity=200, dis_limit=0):
|
||||
walk_v = [X, Y, Yaw, velocity, dis_limit]
|
||||
walk_v = [X,Y,Yaw,velocity,dis_limit]
|
||||
action = GrabSim_pb2.Action(
|
||||
scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v
|
||||
)
|
||||
|
|
@ -200,6 +214,7 @@ class Scene:
|
|||
|
||||
return scene
|
||||
|
||||
|
||||
def reachable_check(self, X, Y, Yaw):
|
||||
if self.use_offset:
|
||||
X, Y = X + loc_offset[0], Y + loc_offset[1]
|
||||
|
|
@ -215,11 +230,11 @@ class Scene:
|
|||
else:
|
||||
return True
|
||||
|
||||
def add_walkers(self, walker_loc=[[0, 880], [250, 1200], [-55, 750], [70, -200]]):
|
||||
def add_walkers(self,walker_loc=[[0, 880], [250, 1200], [-55, 750], [70, -200]]):
|
||||
print('------------------add_walkers----------------------')
|
||||
walker_list = []
|
||||
for i in range(len(walker_loc)):
|
||||
loc = walker_loc[i] + [0, 0, 100]
|
||||
loc = walker_loc[i] + [0,0, 100]
|
||||
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=loc)
|
||||
scene = stub.Do(action)
|
||||
print(scene.info)
|
||||
|
|
@ -251,6 +266,19 @@ class Scene:
|
|||
GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
|
||||
)
|
||||
|
||||
def control_walkers(self,walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True):
|
||||
"""pose:表示行人的终止位置姿态"""
|
||||
scene = self.status
|
||||
walker_loc = walker_loc
|
||||
controls = []
|
||||
for i in range(len(scene.walkers)):
|
||||
loc = walker_loc[i]
|
||||
is_autowalk = is_autowalk
|
||||
pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180)
|
||||
controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=80, pose=pose))
|
||||
scene = stub.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=self.sceneID))
|
||||
|
||||
|
||||
def control_joints(self, angles):
|
||||
stub.Do(
|
||||
GrabSim_pb2.Action(
|
||||
|
|
@ -379,14 +407,13 @@ class Scene:
|
|||
def ik_control_joints(self, handNum=2, x=30, y=40, z=80):
|
||||
# print('------------------ik_control_joints----------------------')
|
||||
# IK控制,双手, 1左手, 2右手; 暂时只动右手
|
||||
HandPostureObject = [
|
||||
GrabSim_pb2.HandPostureInfos.HandPostureObject(handNum=handNum, x=x, y=y, z=z, roll=0, pitch=0, yaw=0),
|
||||
# GrabSim_pb2.HandPostureInfos.HandPostureObject(handNum=1, x=0, y=0, z=0, roll=0, pitch=0, yaw=0),
|
||||
]
|
||||
temp = stub.GetIKControlInfos(
|
||||
GrabSim_pb2.HandPostureInfos(scene=self.sceneID, handPostureObjects=HandPostureObject))
|
||||
HandPostureObject = [GrabSim_pb2.HandPostureInfos.HandPostureObject(handNum=handNum, x=x, y=y, z=z, roll=0, pitch=0, yaw=0),
|
||||
# GrabSim_pb2.HandPostureInfos.HandPostureObject(handNum=1, x=0, y=0, z=0, roll=0, pitch=0, yaw=0),
|
||||
]
|
||||
temp = stub.GetIKControlInfos(GrabSim_pb2.HandPostureInfos(scene=self.sceneID, handPostureObjects=HandPostureObject))
|
||||
|
||||
def move_to_obj(self, obj_id):
|
||||
|
||||
def move_to_obj(self,obj_id):
|
||||
scene = self.status
|
||||
obj_info = scene.objects[obj_id]
|
||||
# Robot
|
||||
|
|
@ -396,27 +423,28 @@ class Scene:
|
|||
walk_v = [obj_x + 40, obj_y - 35, 130, 180, 0]
|
||||
obj_x += 3
|
||||
obj_y += 2.5
|
||||
walk_v[0] += 1
|
||||
print("walk:", walk_v)
|
||||
walk_v[0]+=1
|
||||
print("walk:",walk_v)
|
||||
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v)
|
||||
scene = stub.Do(action)
|
||||
print("After Walk Position:", [scene.location.X, scene.location.Y, scene.rotation.Yaw])
|
||||
|
||||
|
||||
# 移动到进行操作任务的指定地点
|
||||
def move_task_area(self, op_type, obj_id=0, release_pos=[247.0, 520.0, 100.0]):
|
||||
def move_task_area(self,op_type,obj_id=0, release_pos=[247.0, 520.0, 100.0]):
|
||||
scene = self.status
|
||||
cur_pos = [scene.location.X, scene.location.Y, scene.rotation.Yaw]
|
||||
print("Current Position:", cur_pos, "开始任务:", self.op_dialog[op_type])
|
||||
|
||||
if op_type == 11 or op_type == 12: # 开关窗帘不需要移动
|
||||
if op_type==11 or op_type==12: # 开关窗帘不需要移动
|
||||
return
|
||||
print('------------------moveTo_Area----------------------')
|
||||
if op_type < 8:
|
||||
walk_v = self.op_v_list[op_type] + [scene.rotation.Yaw, 180, 0] # 动画控制
|
||||
print("walk_v:", walk_v)
|
||||
if op_type >= 8 and op_type <= 10: walk_v = self.op_v_list[6] + [scene.rotation.Yaw, 180, 0] # 控灯
|
||||
if op_type in [13, 14, 15]: walk_v = [240, -140.0] + [0, 180, 0] # 空调
|
||||
if op_type == 16: # 抓握物体，移动到物体周围的可达区域
|
||||
walk_v = self.op_v_list[op_type] + [scene.rotation.Yaw, 180, 0] # 动画控制
|
||||
print("walk_v:",walk_v)
|
||||
if op_type>=8 and op_type<=10: walk_v = self.op_v_list[6] + [scene.rotation.Yaw, 180, 0] # 控灯
|
||||
if op_type in [13,14,15]: walk_v = [240, -140.0] + [0, 180, 0] # 空调
|
||||
if op_type==16: # 抓握物体，移动到物体周围的可达区域
|
||||
scene = self.status
|
||||
obj_info = scene.objects[obj_id]
|
||||
# Robot
|
||||
|
|
@ -426,11 +454,11 @@ class Scene:
|
|||
walk_v = [obj_x + 40, obj_y - 35, 130, 180, 0]
|
||||
obj_x += 3
|
||||
obj_y += 2.5
|
||||
if op_type == 17: # 放置物体，移动到物体周围的可达区域
|
||||
if op_type==17: # 放置物体，移动到物体周围的可达区域
|
||||
walk_v = release_pos[:-1] + [180, 180, 0]
|
||||
if release_pos == [340.0, 900.0, 99.0]:
|
||||
walk_v[2] = 130
|
||||
print("walk_v:", walk_v)
|
||||
print("walk_v:",walk_v)
|
||||
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v)
|
||||
scene = stub.Do(action)
|
||||
print("After Walk Position:", [scene.location.X, scene.location.Y, scene.rotation.Yaw])
|
||||
|
|
@ -449,12 +477,12 @@ class Scene:
|
|||
print(scene.info)
|
||||
return False
|
||||
|
||||
def adjust_kongtiao(self, op_type):
|
||||
print("self.obj_loc:", self.obj_loc)
|
||||
def adjust_kongtiao(self,op_type):
|
||||
print("self.obj_loc:",self.obj_loc)
|
||||
obj_loc = self.obj_loc[:]
|
||||
print("obj_loc:", obj_loc, "self.obj_loc:", self.obj_loc)
|
||||
print("obj_loc:",obj_loc,"self.obj_loc:", self.obj_loc)
|
||||
obj_loc[2] -= 5
|
||||
print("obj_loc:", obj_loc)
|
||||
print("obj_loc:",obj_loc)
|
||||
if op_type == 13: obj_loc[1] -= 2
|
||||
if op_type == 14: obj_loc[1] -= 0
|
||||
if op_type == 15: obj_loc[1] += 2
|
||||
|
|
@ -463,44 +491,42 @@ class Scene:
|
|||
self.robo_recover()
|
||||
return True
|
||||
|
||||
def gen_obj(self, h=100):
|
||||
def gen_obj(self,h=100):
|
||||
# 4;冰红(盒) 5;酸奶 7:保温杯 9;冰红(瓶) 13:代语词典 14:cake 61:甜牛奶
|
||||
scene = self.status
|
||||
ginger_loc = [scene.location.X, scene.location.Y, scene.location.Z]
|
||||
obj_list = [
|
||||
GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z=95, roll=0, pitch=0, yaw=0,
|
||||
type=5),
|
||||
# GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9),
|
||||
GrabSim_pb2.ObjectList.Object(x=340, y=960, z=88, roll=0, pitch=0, yaw=0, type=9),
|
||||
]
|
||||
obj_list = [GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 55, y=ginger_loc[1] - 40, z = 95, roll=0, pitch=0, yaw=0, type=5),
|
||||
# GrabSim_pb2.ObjectList.Object(x=ginger_loc[0] - 50, y=ginger_loc[1] - 40, z=h, roll=0, pitch=0, yaw=0, type=9),
|
||||
GrabSim_pb2.ObjectList.Object(x=340, y=960, z = 88, roll=0, pitch=0, yaw=0, type=9),
|
||||
]
|
||||
scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID))
|
||||
time.sleep(1.0)
|
||||
|
||||
def grasp_obj(self, obj_id, hand_id=1):
|
||||
def grasp_obj(self,obj_id,hand_id=1):
|
||||
scene = self.status
|
||||
obj_info = scene.objects[obj_id]
|
||||
obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z
|
||||
if obj_info.name == "CoffeeCup":
|
||||
if obj_info.name=="CoffeeCup":
|
||||
pass
|
||||
if obj_info.name == "Glass":
|
||||
if obj_info.name=="Glass":
|
||||
pass
|
||||
# Finger
|
||||
self.ik_control_joints(2, obj_x - 9, obj_y, obj_z) # -10, 0, 0
|
||||
self.ik_control_joints(2, obj_x-9, obj_y, obj_z) # -10, 0, 0
|
||||
time.sleep(3.0)
|
||||
# Grasp Obj
|
||||
print('------------------grasp_obj----------------------')
|
||||
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Grasp,
|
||||
values=[hand_id, obj_id])
|
||||
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Grasp, values=[hand_id, obj_id])
|
||||
scene = stub.Do(action)
|
||||
time.sleep(3.0)
|
||||
return True
|
||||
|
||||
# robot的肢体恢复原位
|
||||
def robo_recover(self):
|
||||
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.RotateJoints, # 恢复原位
|
||||
action = GrabSim_pb2.Action(scene=self.sceneID,action=GrabSim_pb2.Action.ActionType.RotateJoints, # 恢复原位
|
||||
values=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
|
||||
scene = stub.Do(action)
|
||||
|
||||
|
||||
def robo_stoop_parallel(self):
|
||||
# 0-3是躯干，4-6是脖子和头，7-13是左胳膊，14-20是右胳膊
|
||||
scene = self.status
|
||||
|
|
@ -508,14 +534,14 @@ class Scene:
|
|||
angle[0] = 15
|
||||
angle[19] = -15
|
||||
angle[20] = -30
|
||||
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.RotateJoints, # 弯腰
|
||||
action = GrabSim_pb2.Action(scene=self.sceneID,action=GrabSim_pb2.Action.ActionType.RotateJoints, # 弯腰
|
||||
values=angle)
|
||||
scene = stub.Do(action)
|
||||
time.sleep(1.0)
|
||||
|
||||
def release_obj(self, release_pos):
|
||||
def release_obj(self,release_pos):
|
||||
print("------------------release_obj----------------------")
|
||||
if release_pos == [340.0, 900.0, 99.0]:
|
||||
if release_pos==[340.0, 900.0, 99.0]:
|
||||
self.ik_control_joints(2, 300.0, 935, release_pos[2])
|
||||
time.sleep(2.0)
|
||||
else:
|
||||
|
|
@ -531,31 +557,30 @@ class Scene:
|
|||
return True
|
||||
|
||||
# 执行过程：输出"开始(任务名)" -> 按步骤数执行任务 -> Robot输出成功或失败的对话
|
||||
def op_task_execute(self, op_type, obj_id=0, release_pos=[247.0, 520.0, 100.0]):
|
||||
self.control_robot_action(0, 1, "开始" + self.op_dialog[op_type]) # 开始制作咖啡
|
||||
if op_type < 8: result = self.control_robot_action(op_type, 1)
|
||||
if op_type >= 8 and op_type <= 12: result = self.control_robot_action(self.op_typeToAct[op_type][0],
|
||||
self.op_typeToAct[op_type][1])
|
||||
if op_type in [13, 14, 15]: # 调整空调:13代表按开关,14升温,15降温
|
||||
def op_task_execute(self,op_type,obj_id=0,release_pos=[247.0, 520.0, 100.0]):
|
||||
self.control_robot_action(0, 1, "开始"+self.op_dialog[op_type]) # 开始制作咖啡
|
||||
if op_type<8: result = self.control_robot_action(op_type, 1)
|
||||
if op_type>=8 and op_type<=12: result = self.control_robot_action(self.op_typeToAct[op_type][0], self.op_typeToAct[op_type][1])
|
||||
if op_type in [13,14,15]: # 调整空调:13代表按开关,14升温,15降温
|
||||
result = self.adjust_kongtiao(op_type)
|
||||
if op_type == 16: # 抓握物体
|
||||
if op_type ==16: # 抓握物体
|
||||
result = self.grasp_obj(obj_id)
|
||||
if op_type == 17: # 放置物体
|
||||
if op_type ==17: # 放置物体
|
||||
result = self.release_obj(release_pos)
|
||||
self.control_robot_action(0, 2)
|
||||
if result:
|
||||
if self.op_act_num[op_type] > 0:
|
||||
for i in range(2, 2 + self.op_act_num[op_type]):
|
||||
self.control_robot_action(op_type, i)
|
||||
if self.op_act_num[op_type]>0:
|
||||
for i in range(2,2+self.op_act_num[op_type]):
|
||||
self.control_robot_action(op_type,i)
|
||||
self.control_robot_action(0, 2)
|
||||
# self.control_robot_action(0, 1, "成功"+self.op_dialog[op_type])
|
||||
# else:
|
||||
# self.control_robot_action(0, 1, self.op_dialog[op_type]+"失败")
|
||||
|
||||
def find_obj(self, name):
|
||||
def find_obj(self,name):
|
||||
for id, item in enumerate(self.status.objects):
|
||||
if item.name == name:
|
||||
print("name:", name, "id:", id, "X:", item.location.X, "Y:", item.location.Y, "Z:", item.location.Z, )
|
||||
print("name:",name,"id:",id,"X:",item.location.X,"Y:",item.location.Y,"Z:",item.location.Z,)
|
||||
|
||||
def test_move(self):
|
||||
v_list = [[0, 880], [250, 1200], [-55, 750], [70, -200]]
|
||||
|
|
@ -583,8 +608,7 @@ class Scene:
|
|||
print("walk_v", walk_v)
|
||||
action = GrabSim_pb2.Action(scene=scene_id, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v)
|
||||
scene = stub.Do(action)
|
||||
cur_objs, objs_name_set = camera.get_semantic_map(GrabSim_pb2.CameraName.Head_Segment, cur_objs,
|
||||
objs_name_set)
|
||||
cur_objs, objs_name_set = camera.get_semantic_map(GrabSim_pb2.CameraName.Head_Segment, cur_objs, objs_name_set)
|
||||
# if scene.info == "Unreachable":
|
||||
print(scene.info)
|
||||
return cur_objs, objs_name_set
|
||||
|
|
@ -669,24 +693,10 @@ class Scene:
|
|||
frontier_best = frontier
|
||||
return frontier_best
|
||||
|
||||
def cal_distance_to_robot(self, objx, objy, objz):
|
||||
|
||||
|
||||
def cal_distance_to_robot(self,objx,objy,objz):
|
||||
scene = self.status
|
||||
ginger_x, ginger_y, ginger_z = [int(scene.location.X), int(scene.location.Y), 100]
|
||||
ginger_x, ginger_y, ginger_z = [int(scene.location.X), int(scene.location.Y),100]
|
||||
return math.sqrt((ginger_x - objx) ** 2 + (ginger_y - objy) ** 2 + (ginger_z - objz) ** 2)
|
||||
|
||||
# def test_yaw(self):
|
||||
# walk_v = [247.0, 480.0, 180.0, 180, 0]
|
||||
# action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v)
|
||||
# scene = stub.Do(action)
|
||||
# time.sleep(4)
|
||||
# walk_v = [247.0, 500.0, 0.0, 180, 0]
|
||||
# action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v)
|
||||
# scene = stub.Do(action)
|
||||
|
||||
# 读取map文件判断是否可达
|
||||
def reachable(self, pos):
|
||||
x, y = self.real2map(pos[0], pos[1])
|
||||
if self.map_file[x, y] == 1:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class SceneOT(Scene):
|
|||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件
|
||||
self.event_list = [
|
||||
(5, self.create_chat_event("给我一杯咖啡"))
|
||||
(5, self.set_goal("At(Robot,Table2)"))
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
|
|
|
|||
|
|
@ -11,21 +11,49 @@ class SceneVLM(Scene):
|
|||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
|
||||
self.event_list = [
|
||||
(5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
|
||||
# (10, self.create_chat_event("开空调")),
|
||||
|
||||
# (15, self.create_chat_event("下班啦！打扫卫生，关灯关空调关窗帘。")),
|
||||
|
||||
|
||||
# (9, self.create_chat_event("关窗帘")),
|
||||
# (9, self.create_chat_event("开大厅灯")),
|
||||
# (5, self.create_chat_event("测试VLM：做一杯咖啡")),
|
||||
# (5, self.create_chat_event("测试VLM：倒一杯水")),
|
||||
# (5, self.create_chat_event("测试VLM：开空调")),
|
||||
# (5, self.create_chat_event("测试VLM：关空调")),
|
||||
# (5, self.create_chat_event("测试VLM：开大厅灯")),
|
||||
# (5, self.create_chat_event("测试VLM：拖地")),
|
||||
# (5, self.create_chat_event("测试VLM：擦桌子")),
|
||||
# (5, self.create_chat_event("测试VLM：整理椅子")),
|
||||
# (5, self.create_chat_event("测试VLM：把冰红茶放到Table2")),
|
||||
# (5, self.create_chat_event("测试VLM：关大厅灯"))
|
||||
# (5, self.create_chat_event("测试VLM：做一杯咖啡放到吧台上")),
|
||||
(5, self.create_chat_event("测试VLM：做一杯咖啡放到水杯桌上并倒水")),
|
||||
# (5, self.create_chat_event("测试VLM：做一杯咖啡放到水杯桌上，再倒一杯水")),
|
||||
# (10, self.create_chat_event("测试VLM：关窗帘")),
|
||||
# (5, self.create_chat_event("测试VLN：前往2号桌")),
|
||||
# (11, self.create_chat_event("测试VLM：拖地")),
|
||||
# (12, self.create_chat_event("测试VLM：擦桌子")),
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
self.gen_obj()
|
||||
self.state["condition_set"] = {'At(Robot,Bar)', 'Is(AC,Off)',
|
||||
'Holding(Nothing)','Exist(Yogurt)','Exist(Softdrink)','On(Yogurt,Bar)','On(Softdrink,Table1)',
|
||||
'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
|
||||
'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'}
|
||||
self.add_walkers([[0, 880], [250, 1200], [-55, 750], [70, -200]])
|
||||
# self.add_walkers([[-500, 500]])
|
||||
self.control_walkers(walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True)
|
||||
# self.control_walkers(walker_loc=[[-55, 750]],is_autowalk = False)
|
||||
# 在场景中随机增加一堆行人。
|
||||
# walker_loc = [[-55, 750], [70, -200], [250, 1200], [0, 880]]
|
||||
# controls = []
|
||||
# for i in range(len(s.walkers)):
|
||||
# loc = walker_loc[i]
|
||||
# is_autowalk = False
|
||||
# pose = GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=180)
|
||||
# controls.append(GrabSim_pb2.WalkerControls.WControl(id=i, autowalk=is_autowalk, speed=200, pose=pose))
|
||||
# scene = sim_client.ControlWalkers(GrabSim_pb2.WalkerControls(controls=controls, scene=scene_id))
|
||||
|
||||
# self.gen_obj(type=5)
|
||||
# self.gen_obj(type=9)
|
||||
|
|
@ -34,8 +62,6 @@ class SceneVLM(Scene):
|
|||
pass
|
||||
|
||||
def _run(self, op_type=10):
|
||||
|
||||
|
||||
# 共17个操作
|
||||
# "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7
|
||||
# "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12
|
||||
|
|
@ -58,7 +84,7 @@ class SceneVLM(Scene):
|
|||
|
||||
# 流程测试
|
||||
# 抓握放置:抓吧台前生成的酸奶，放到抹布桌上
|
||||
self.gen_obj()
|
||||
# self.gen_obj()
|
||||
# self.move_task_area(16, obj_id=0)
|
||||
# self.op_task_execute(16, obj_id=0)
|
||||
# pos = [340.0, 900.0, 99.0]
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
"""
|
||||
视觉语言操作
|
||||
机器人根据指令人的指令调节空调，自主探索环境导航到目标点，通过手臂的运动规划能力操作空调，比如开关按钮、调温按钮、显示面板
|
||||
"""
|
||||
|
||||
import time
|
||||
from robowaiter.scene.scene import Scene
|
||||
|
||||
class SceneVLM(Scene):
|
||||
def __init__(self, robot):
|
||||
super().__init__(robot)
|
||||
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
|
||||
self.event_list = [
|
||||
|
||||
]
|
||||
|
||||
def _reset(self):
|
||||
self.gen_obj()
|
||||
pass
|
||||
|
||||
def _run(self, op_type=10):
|
||||
# 一个行人从门口走到 吧台
|
||||
# 打招呼需要什么
|
||||
# 行人说 哪里有位置，想晒个太阳
|
||||
# 带领行人去有太阳的地方
|
||||
# 行人说 有点热
|
||||
# 好的，这就去开空调
|
||||
scene = self.add_walkers([[47, 920]])
|
||||
self.control_walker(
|
||||
[self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=0)])
|
||||
|
||||
cont = scene.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳"
|
||||
self.control_robot_action(0,3,cont)
|
||||
|
||||
# 共17个操作
|
||||
# "制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7
|
||||
# "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12
|
||||
# "调整空调开关","调高空调温度","调低空调温度", # 13-15
|
||||
# "抓握物体","放置物体" # 16-17
|
||||
|
||||
# self.gen_obj()
|
||||
# if op_type <=15:
|
||||
# self.move_task_area(op_type)
|
||||
# self.op_task_execute(op_type)
|
||||
# if op_type == 16: # 16: 抓操作需要传入物品id
|
||||
# self.move_task_area(op_type, obj_id=0)
|
||||
# self.op_task_execute(op_type, obj_id=0)
|
||||
# # 原始吧台处:[247.0, 520.0, 100.0], 空调开关旁吧台:[240.0, 40.0, 100.0], 水杯桌:[-70.0, 500.0, 107]
|
||||
# # 桌子1:[-55.0, 0.0, 107],抹布桌:[340.0, 900.0, 99.0] # 桌子2:[-55.0, 150.0, 107],
|
||||
# if op_type == 17: # 17: 放操作需要传入放置位置周围的可达区域
|
||||
# pos = [240.0, 40.0, 100.0]
|
||||
# self.move_task_area(op_type, release_pos=pos)
|
||||
# self.op_task_execute(op_type, release_pos=pos) # [325.0, 860.0, 100]
|
||||
|
||||
|
||||
pass
|
||||
|
||||
def _step(self):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import os
|
||||
from robowaiter.robot.robot import Robot
|
||||
|
||||
robot = Robot()
|
||||
|
||||
# create task
|
||||
task = SceneVLM(robot)
|
||||
task.reset()
|
||||
task.run()
|
||||
|
|
@ -1,5 +1,10 @@
|
|||
selector{
|
||||
cond On(Coffee,CoffeeTable)
|
||||
|
||||
sequence{
|
||||
cond On(Coffee,CoffeeTable)
|
||||
cond On(Coffee,CoffeeTable)
|
||||
}
|
||||
|
||||
sequence{
|
||||
cond Holding(Nothing)
|
||||
act Make(Coffee)
|
||||
|
|
|
|||
Loading…
Reference in New Issue