EmbodiedAI
/
RoboWaiter
mirror of https://github.com/HPCL-EI/RoboWaiter.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/HPCL-EI/RoboWaiter

This commit is contained in:
Caiyishuai 2023-11-18 14:26:10 +08:00
parent aed3cd9aae d7b5fc9f6c
commit bad1b8fb3d
20 changed files with 5300 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
robowaiter/behavior_lib/_base/Behavior.py
View File

@ -83,6 +83,7 @@ class Bahavior(ptree.behaviour.Behaviour):
# let behavior node interact with the scene # let behavior node interact with the scene
def set_scene(self, scene): def set_scene(self, scene):
self.scene = scene self.scene = scene
self.robot = scene.robot
def setup(self, **kwargs: Any) -> None: def setup(self, **kwargs: Any) -> None:
return super().setup(**kwargs) return super().setup(**kwargs)

3
robowaiter/behavior_lib/act/DealChat.py
View File

@ -10,6 +10,9 @@ class DealChat(Act):
def _update(self) -> ptree.common.Status: def _update(self) -> ptree.common.Status:
# if self.scene.status? # if self.scene.status?
name,sentence = self.scene.state['chat_list'][0]
chat = self.scene.state['chat_list'].pop() chat = self.scene.state['chat_list'].pop()
if isinstance(chat,set): if isinstance(chat,set):
self.create_sub_task(chat) self.create_sub_task(chat)

31
robowaiter/behavior_lib/act/GreetCustomer.py Normal file
View File

@ -0,0 +1,31 @@
import py_trees as ptree
from robowaiter.behavior_lib._base.Act import Act
from robowaiter.algos.navigator.navigate import Navigator
class GreetCustomer(Act):
can_be_expanded = True
num_args = 0
valid_args = ()
def __init__(self, *args):
super().__init__(*args)
@classmethod
def get_info(cls):
info = {}
info['pre'] = set()
info["add"] = set()
info["del_set"] = set()
info['cost']=0
return info
def _update(self) -> ptree.common.Status:
goal = Act.place_xyz_dic['Bar']
self.scene.walk_to(goal[0]-5,goal[1], 180, 180, 0)
self.scene.chat_bubble("欢迎光临！请问有什么可以帮您？")
customer_name = self.scene.state['attention']['customer']
self.scene.state['greeted_customers'].add(customer_name)
return ptree.common.Status.RUNNING

4
robowaiter/behavior_lib/act/GreatCustomer.py → robowaiter/behavior_lib/act/ServeCustomer.py
View File

@ -2,7 +2,7 @@ import py_trees as ptree
from robowaiter.behavior_lib._base.Act import Act from robowaiter.behavior_lib._base.Act import Act
from robowaiter.algos.navigator.navigate import Navigator from robowaiter.algos.navigator.navigate import Navigator
class GreatCustomer(Act): class ServeCustomer(Act):
can_be_expanded = False can_be_expanded = False
num_args = 0 num_args = 0
valid_args = () valid_args = ()
@ -14,7 +14,7 @@ class GreatCustomer(Act):
def get_info(cls): def get_info(cls):
info = {} info = {}
info['pre'] = set() info['pre'] = set()
info["add"] = set() info["add"] = {"CustomerServed()"}
info["del_set"] = set() info["del_set"] = set()
info['cost']=0 info['cost']=0
return info return info

24
robowaiter/behavior_lib/cond/CustomerServed.py Normal file
View File

@ -0,0 +1,24 @@
import py_trees as ptree
from typing import Any
from robowaiter.behavior_lib._base.Cond import Cond
import itertools
class CustomerServed(Cond):
can_be_expanded = True
def __init__(self,*args):
super().__init__(*args)
def _update(self) -> ptree.common.Status:
# if self.scene.status?
if self.name in self.scene.state["condition_set"]:
return ptree.common.Status.SUCCESS
else:
return ptree.common.Status.FAILURE
# if self.scene.state['chat_list'] == []:
# return ptree.common.Status.FAILURE
# else:
# return ptree.common.Status.SUCCESS

13
robowaiter/behavior_lib/cond/DetectCustomer.py
View File

@ -18,14 +18,21 @@ class DetectCustomer(Cond):
# bar (247.0, 520.0, 100.0) # bar (247.0, 520.0, 100.0)
close_to_bar = False close_to_bar = False
scene = self.scene.status scene = self.scene.status
queue_list = []
for walker in scene.walkers: for walker in scene.walkers:
x, y, yaw = walker.pose.X, walker.pose.Y, walker.pose.Yaw x, y, yaw = walker.pose.X, walker.pose.Y, walker.pose.Yaw
# 到达一定区域就打招呼 # 到达一定区域就打招呼
if y >= 450 and y <= 620 and x >= 40 and x <= 100 and yaw>=-10 and yaw <=10: if y >= 450 and y <= 620 and x >= 40 and x <= 100 and yaw>=-10 and yaw <=10:
close_to_bar = True # close_to_bar = True
break queue_list.append((x,y,walker.name))
if close_to_bar: if queue_list == []:
return ptree.common.Status.FAILURE
queue_list.sort()
x,y,name = queue_list[0]
if name not in self.scene.state["greeted_customers"]:
self.scene.state['attention']["customer"] = name
return ptree.common.Status.SUCCESS return ptree.common.Status.SUCCESS
else: else:
return ptree.common.Status.FAILURE return ptree.common.Status.FAILURE

2
robowaiter/behavior_tree/dataset/goal_states_generation.py
View File

@ -297,7 +297,7 @@ def mutex(path: str):
new_line = re.sub('#', '', new_line) new_line = re.sub('#', '', new_line)
lines = re.sub('#', '', lines) lines = re.sub('#', '', lines)
with open(os.path.join(path), 'a', encoding='utf-8') as file: with open(os.path.join(path), 'w', encoding='utf-8') as file:
file.write(new_line*13 + lines * 13) file.write(new_line*13 + lines * 13)

5018
robowaiter/behavior_tree/dataset/goal_states_with_description.jsonl Normal file
View File

File diff suppressed because it is too large Load Diff

193
robowaiter/behavior_tree/dataset/goal_states_with_description.txt
View File

@ -1,196 +1,3 @@
At(Robot,Bar) 请你来一下#Bar。
At(Robot,Bar) 请你去一下#Bar。
At(Robot,Bar) 你能去#Bar那个位置吗？
At(Robot,WaterTable) 请你来一下#WaterTable。
At(Robot,WaterTable) 请你去一下#WaterTable。
At(Robot,WaterTable) 你能去#WaterTable那个位置吗？
At(Robot,CoffeeTable) 请你来一下#CoffeeTable。
At(Robot,CoffeeTable) 请你去一下#CoffeeTable。
At(Robot,CoffeeTable) 你能去#CoffeeTable那个位置吗？
At(Robot,Bar2) 请你来一下#Bar2。
At(Robot,Bar2) 请你去一下#Bar2。
At(Robot,Bar2) 你能去#Bar2那个位置吗？
At(Robot,Table1) 请你来一下#Table1。
At(Robot,Table1) 请你去一下#Table1。
At(Robot,Table1) 你能去#Table1那个位置吗？
At(Robot,Table2) 请你来一下#Table2。
At(Robot,Table2) 请你去一下#Table2。
At(Robot,Table2) 你能去#Table2那个位置吗？
At(Robot,Table3) 请你来一下#Table3。
At(Robot,Table3) 请你去一下#Table3。
At(Robot,Table3) 你能去#Table3那个位置吗？
On(Softdrink,Bar) 请你把#Softdrink放到#Bar那个位置。
On(Softdrink,Bar) 请你拿一下#Softdrink到#Bar位置。
On(Softdrink,WaterTable) 请你把#Softdrink放到#WaterTable那个位置。
On(Softdrink,WaterTable) 请你拿一下#Softdrink到#WaterTable位置。
On(Softdrink,CoffeeTable) 请你把#Softdrink放到#CoffeeTable那个位置。
On(Softdrink,CoffeeTable) 请你拿一下#Softdrink到#CoffeeTable位置。
On(Softdrink,Bar2) 请你把#Softdrink放到#Bar2那个位置。
On(Softdrink,Bar2) 请你拿一下#Softdrink到#Bar2位置。
On(Softdrink,Table1) 请你把#Softdrink放到#Table1那个位置。
On(Softdrink,Table1) 请你拿一下#Softdrink到#Table1位置。
On(Softdrink,Table2) 请你把#Softdrink放到#Table2那个位置。
On(Softdrink,Table2) 请你拿一下#Softdrink到#Table2位置。
On(Softdrink,Table3) 请你把#Softdrink放到#Table3那个位置。
On(Softdrink,Table3) 请你拿一下#Softdrink到#Table3位置。
On(BottledDrink,Bar) 请你把#BottledDrink放到#Bar那个位置。
On(BottledDrink,Bar) 请你拿一下#BottledDrink到#Bar位置。
On(BottledDrink,WaterTable) 请你把#BottledDrink放到#WaterTable那个位置。
On(BottledDrink,WaterTable) 请你拿一下#BottledDrink到#WaterTable位置。
On(BottledDrink,CoffeeTable) 请你把#BottledDrink放到#CoffeeTable那个位置。
On(BottledDrink,CoffeeTable) 请你拿一下#BottledDrink到#CoffeeTable位置。
On(BottledDrink,Bar2) 请你把#BottledDrink放到#Bar2那个位置。
On(BottledDrink,Bar2) 请你拿一下#BottledDrink到#Bar2位置。
On(BottledDrink,Table1) 请你把#BottledDrink放到#Table1那个位置。
On(BottledDrink,Table1) 请你拿一下#BottledDrink到#Table1位置。
On(BottledDrink,Table2) 请你把#BottledDrink放到#Table2那个位置。
On(BottledDrink,Table2) 请你拿一下#BottledDrink到#Table2位置。
On(BottledDrink,Table3) 请你把#BottledDrink放到#Table3那个位置。
On(BottledDrink,Table3) 请你拿一下#BottledDrink到#Table3位置。
On(Yogurt,Bar) 请你把#Yogurt放到#Bar那个位置。
On(Yogurt,Bar) 请你拿一下#Yogurt到#Bar位置。
On(Yogurt,WaterTable) 请你把#Yogurt放到#WaterTable那个位置。
On(Yogurt,WaterTable) 请你拿一下#Yogurt到#WaterTable位置。
On(Yogurt,CoffeeTable) 请你把#Yogurt放到#CoffeeTable那个位置。
On(Yogurt,CoffeeTable) 请你拿一下#Yogurt到#CoffeeTable位置。
On(Yogurt,Bar2) 请你把#Yogurt放到#Bar2那个位置。
On(Yogurt,Bar2) 请你拿一下#Yogurt到#Bar2位置。
On(Yogurt,Table1) 请你把#Yogurt放到#Table1那个位置。
On(Yogurt,Table1) 请你拿一下#Yogurt到#Table1位置。
On(Yogurt,Table2) 请你把#Yogurt放到#Table2那个位置。
On(Yogurt,Table2) 请你拿一下#Yogurt到#Table2位置。
On(Yogurt,Table3) 请你把#Yogurt放到#Table3那个位置。
On(Yogurt,Table3) 请你拿一下#Yogurt到#Table3位置。
On(ADMilk,Bar) 请你把#ADMilk放到#Bar那个位置。
On(ADMilk,Bar) 请你拿一下#ADMilk到#Bar位置。
On(ADMilk,WaterTable) 请你把#ADMilk放到#WaterTable那个位置。
On(ADMilk,WaterTable) 请你拿一下#ADMilk到#WaterTable位置。
On(ADMilk,CoffeeTable) 请你把#ADMilk放到#CoffeeTable那个位置。
On(ADMilk,CoffeeTable) 请你拿一下#ADMilk到#CoffeeTable位置。
On(ADMilk,Bar2) 请你把#ADMilk放到#Bar2那个位置。
On(ADMilk,Bar2) 请你拿一下#ADMilk到#Bar2位置。
On(ADMilk,Table1) 请你把#ADMilk放到#Table1那个位置。
On(ADMilk,Table1) 请你拿一下#ADMilk到#Table1位置。
On(ADMilk,Table2) 请你把#ADMilk放到#Table2那个位置。
On(ADMilk,Table2) 请你拿一下#ADMilk到#Table2位置。
On(ADMilk,Table3) 请你把#ADMilk放到#Table3那个位置。
On(ADMilk,Table3) 请你拿一下#ADMilk到#Table3位置。
On(MilkDrink,Bar) 请你把#MilkDrink放到#Bar那个位置。
On(MilkDrink,Bar) 请你拿一下#MilkDrink到#Bar位置。
On(MilkDrink,WaterTable) 请你把#MilkDrink放到#WaterTable那个位置。
On(MilkDrink,WaterTable) 请你拿一下#MilkDrink到#WaterTable位置。
On(MilkDrink,CoffeeTable) 请你把#MilkDrink放到#CoffeeTable那个位置。
On(MilkDrink,CoffeeTable) 请你拿一下#MilkDrink到#CoffeeTable位置。
On(MilkDrink,Bar2) 请你把#MilkDrink放到#Bar2那个位置。
On(MilkDrink,Bar2) 请你拿一下#MilkDrink到#Bar2位置。
On(MilkDrink,Table1) 请你把#MilkDrink放到#Table1那个位置。
On(MilkDrink,Table1) 请你拿一下#MilkDrink到#Table1位置。
On(MilkDrink,Table2) 请你把#MilkDrink放到#Table2那个位置。
On(MilkDrink,Table2) 请你拿一下#MilkDrink到#Table2位置。
On(MilkDrink,Table3) 请你把#MilkDrink放到#Table3那个位置。
On(MilkDrink,Table3) 请你拿一下#MilkDrink到#Table3位置。
On(Milk,Bar) 请你把#Milk放到#Bar那个位置。
On(Milk,Bar) 请你拿一下#Milk到#Bar位置。
On(Milk,WaterTable) 请你把#Milk放到#WaterTable那个位置。
On(Milk,WaterTable) 请你拿一下#Milk到#WaterTable位置。
On(Milk,CoffeeTable) 请你把#Milk放到#CoffeeTable那个位置。
On(Milk,CoffeeTable) 请你拿一下#Milk到#CoffeeTable位置。
On(Milk,Bar2) 请你把#Milk放到#Bar2那个位置。
On(Milk,Bar2) 请你拿一下#Milk到#Bar2位置。
On(Milk,Table1) 请你把#Milk放到#Table1那个位置。
On(Milk,Table1) 请你拿一下#Milk到#Table1位置。
On(Milk,Table2) 请你把#Milk放到#Table2那个位置。
On(Milk,Table2) 请你拿一下#Milk到#Table2位置。
On(Milk,Table3) 请你把#Milk放到#Table3那个位置。
On(Milk,Table3) 请你拿一下#Milk到#Table3位置。
On(VacuumCup,Bar) 请你把#VacuumCup放到#Bar那个位置。
On(VacuumCup,Bar) 请你拿一下#VacuumCup到#Bar位置。
On(VacuumCup,WaterTable) 请你把#VacuumCup放到#WaterTable那个位置。
On(VacuumCup,WaterTable) 请你拿一下#VacuumCup到#WaterTable位置。
On(VacuumCup,CoffeeTable) 请你把#VacuumCup放到#CoffeeTable那个位置。
On(VacuumCup,CoffeeTable) 请你拿一下#VacuumCup到#CoffeeTable位置。
On(VacuumCup,Bar2) 请你把#VacuumCup放到#Bar2那个位置。
On(VacuumCup,Bar2) 请你拿一下#VacuumCup到#Bar2位置。
On(VacuumCup,Table1) 请你把#VacuumCup放到#Table1那个位置。
On(VacuumCup,Table1) 请你拿一下#VacuumCup到#Table1位置。
On(VacuumCup,Table2) 请你把#VacuumCup放到#Table2那个位置。
On(VacuumCup,Table2) 请你拿一下#VacuumCup到#Table2位置。
On(VacuumCup,Table3) 请你把#VacuumCup放到#Table3那个位置。
On(VacuumCup,Table3) 请你拿一下#VacuumCup到#Table3位置。
Is(AC,0) 你能把#AC#关闭一下吗？
Is(AC,1) 你能把#AC#打开一下吗？
Is(ACTemperature,0) 你能把#ACTemperature#调高一下吗？
Is(ACTemperature,1) 你能把#ACTemperature#调低一下吗？
Is(HallLight,0) 你能把#HallLight#关闭一下吗？
Is(HallLight,1) 你能把#HallLight#打开一下吗？
Is(TubeLight,0) 你能把#TubeLight#关闭一下吗？
Is(TubeLight,1) 你能把#TubeLight#打开一下吗？
Is(Curtain,0) 你能把#Curtain#关闭一下吗？
Is(Curtain,1) 你能把#Curtain#打开一下吗？
Is(Chairs,0) 你能把#Chairs#脏一下吗？
Is(Chairs,1) 你能把#Chairs#打扫干净一下吗？
Is(Floor,0) 你能把#Floor#脏一下吗？
Is(Floor,1) 你能把#Floor#打扫干净一下吗？
Is(Table1,0) 你能把#Table1#脏一下吗？
Is(Table1,1) 你能把#Table1#打扫干净一下吗？
Holding(Softdrink) 你能把#Softdrink抓在手里吗？
Holding(Softdrink) 你能一直拿着#Softdrink吗？
Holding(BottledDrink) 你能把#BottledDrink抓在手里吗？
Holding(BottledDrink) 你能一直拿着#BottledDrink吗？
Holding(Yogurt) 你能把#Yogurt抓在手里吗？
Holding(Yogurt) 你能一直拿着#Yogurt吗？
Holding(ADMilk) 你能把#ADMilk抓在手里吗？
Holding(ADMilk) 你能一直拿着#ADMilk吗？
Holding(MilkDrink) 你能把#MilkDrink抓在手里吗？
Holding(MilkDrink) 你能一直拿着#MilkDrink吗？
Holding(Milk) 你能把#Milk抓在手里吗？
Holding(Milk) 你能一直拿着#Milk吗？
Holding(VacuumCup) 你能把#VacuumCup抓在手里吗？
Holding(VacuumCup) 你能一直拿着#VacuumCup吗？
Holding(Nothing) 你能把#Nothing抓在手里吗？
Holding(Nothing) 你能一直拿着#Nothing吗？
On(Coffee,Bar) 你能制作#Coffee并把它端到#Bar这里来吗？
On(Coffee,Bar) 给我来点#Coffee，并把它端到#Bar这里来。
On(Coffee,WaterTable) 你能制作#Coffee并把它端到#WaterTable这里来吗？
On(Coffee,WaterTable) 给我来点#Coffee，并把它端到#WaterTable这里来。
On(Coffee,CoffeeTable) 你能制作#Coffee并把它端到#CoffeeTable这里来吗？
On(Coffee,CoffeeTable) 给我来点#Coffee，并把它端到#CoffeeTable这里来。
On(Coffee,Bar2) 你能制作#Coffee并把它端到#Bar2这里来吗？
On(Coffee,Bar2) 给我来点#Coffee，并把它端到#Bar2这里来。
On(Coffee,Table1) 你能制作#Coffee并把它端到#Table1这里来吗？
On(Coffee,Table1) 给我来点#Coffee，并把它端到#Table1这里来。
On(Coffee,Table2) 你能制作#Coffee并把它端到#Table2这里来吗？
On(Coffee,Table2) 给我来点#Coffee，并把它端到#Table2这里来。
On(Coffee,Table3) 你能制作#Coffee并把它端到#Table3这里来吗？
On(Coffee,Table3) 给我来点#Coffee，并把它端到#Table3这里来。
On(Water,Bar) 你能制作#Water并把它端到#Bar这里来吗？
On(Water,Bar) 给我来点#Water，并把它端到#Bar这里来。
On(Water,WaterTable) 你能制作#Water并把它端到#WaterTable这里来吗？
On(Water,WaterTable) 给我来点#Water，并把它端到#WaterTable这里来。
On(Water,CoffeeTable) 你能制作#Water并把它端到#CoffeeTable这里来吗？
On(Water,CoffeeTable) 给我来点#Water，并把它端到#CoffeeTable这里来。
On(Water,Bar2) 你能制作#Water并把它端到#Bar2这里来吗？
On(Water,Bar2) 给我来点#Water，并把它端到#Bar2这里来。
On(Water,Table1) 你能制作#Water并把它端到#Table1这里来吗？
On(Water,Table1) 给我来点#Water，并把它端到#Table1这里来。
On(Water,Table2) 你能制作#Water并把它端到#Table2这里来吗？
On(Water,Table2) 给我来点#Water，并把它端到#Table2这里来。
On(Water,Table3) 你能制作#Water并把它端到#Table3这里来吗？
On(Water,Table3) 给我来点#Water，并把它端到#Table3这里来。
On(Dessert,Bar) 你能制作#Dessert并把它端到#Bar这里来吗？
On(Dessert,Bar) 给我来点#Dessert，并把它端到#Bar这里来。
On(Dessert,WaterTable) 你能制作#Dessert并把它端到#WaterTable这里来吗？
On(Dessert,WaterTable) 给我来点#Dessert，并把它端到#WaterTable这里来。
On(Dessert,CoffeeTable) 你能制作#Dessert并把它端到#CoffeeTable这里来吗？
On(Dessert,CoffeeTable) 给我来点#Dessert，并把它端到#CoffeeTable这里来。
On(Dessert,Bar2) 你能制作#Dessert并把它端到#Bar2这里来吗？
On(Dessert,Bar2) 给我来点#Dessert，并把它端到#Bar2这里来。
On(Dessert,Table1) 你能制作#Dessert并把它端到#Table1这里来吗？
On(Dessert,Table1) 给我来点#Dessert，并把它端到#Table1这里来。
On(Dessert,Table2) 你能制作#Dessert并把它端到#Table2这里来吗？
On(Dessert,Table2) 给我来点#Dessert，并把它端到#Table2这里来。
On(Dessert,Table3) 你能制作#Dessert并把它端到#Table3这里来吗？
On(Dessert,Table3) 给我来点#Dessert，并把它端到#Table3这里来。
At(Robot,Bar) 请你来一下吧台。 At(Robot,Bar) 请你来一下吧台。
At(Robot,Bar) 请你去一下吧台。 At(Robot,Bar) 请你去一下吧台。
At(Robot,Bar) 你能去吧台那个位置吗？ At(Robot,Bar) 你能去吧台那个位置吗？

10
robowaiter/behavior_tree/dataset/goal_states_with_description_to_json.py Normal file
View File

@ -0,0 +1,10 @@
import os
with open(os.path.join('./goal_states_with_description.txt'), 'r', encoding='utf-8') as file:
lines = file.readlines()
with open(os.path.join('./goal_states_with_description.jsonl'), 'w', encoding='utf-8') as file:
for line in lines:
tmp = line[:-1].split('\t')
file.write("""{"title":"%s","text":"%s"}\n""" % (tmp[1], tmp[0]))

2
robowaiter/llm_client/find_obj_utils.py
View File

@ -35,7 +35,7 @@ all_loc_en = ['bar', 'Table', 'sofa', 'stove', 'Gate', 'light switch', 'aircondi
'cake display', 'ChargingStations', 'refrigerator', 'bookshelf'] 'cake display', 'ChargingStations', 'refrigerator', 'bookshelf']
loc_map_en = {'bar': {'工作台', '服务台', '收银台', '蛋糕柜'}, loc_map_en = {'bar': {'工作台', '服务台', '收银台', '蛋糕柜'},
'Table': {'沙发', '大门', '窗户', '休闲区', '墙角', '椅子', '书架'}, 'Table': {'大门', '休闲区', '墙角'},
'sofa': {'餐桌', '窗户', '音响', '休闲区', '墙角', '书架'}, 'sofa': {'餐桌', '窗户', '音响', '休闲区', '墙角', '书架'},
'stove': {'吧台', '橱柜', '工作台', '服务台', '收银台', '蛋糕柜', '冰箱'}, 'stove': {'吧台', '橱柜', '工作台', '服务台', '收银台', '蛋糕柜', '冰箱'},
'Gate': {'吧台', '灯开关', '空调开关', '卫生间', '墙角'}, 'Gate': {'吧台', '灯开关', '空调开关', '卫生间', '墙角'},

28
robowaiter/llm_client/tool_api.py
View File

@ -60,7 +60,7 @@ def run_conversation(query: str, stream=False, max_retry=5):
"content": tool_response, # 调用函数返回结果 "content": tool_response, # 调用函数返回结果
} }
) )
# del params["functions"] del params["functions"]
else: else:
reply = response["choices"][0]["message"]["content"] reply = response["choices"][0]["message"]["content"]
return { return {
@ -116,12 +116,24 @@ if __name__ == "__main__":
query = "卫生间在哪里" # query = "卫生间在哪里" #
print(run_conversation(query, stream=False)) print(run_conversation(query, stream=False))
query = "我想看看冰箱,请问哪里可以找到冰箱"
print(run_conversation(query, stream=False))
query = "我想找个充电的地方，你能告诉我在哪儿吗"
print(run_conversation(query, stream=False))
query = "我想找张桌子" # query = "我想找张桌子" #
print(run_conversation(query, stream=False)) print(run_conversation(query, stream=False))
# for query in
query = "我想看看休闲区,请问哪里可以找到休闲区"
print(run_conversation(query, stream=False))
#
# query = "我想找个充电的地方，你能告诉我在哪儿吗"
# print(run_conversation(query, stream=False))
query = "我想来一个面包"
print(run_conversation(query, stream=False))
query = "哪里有卫生纸"
print(run_conversation(query, stream=False))
query = "插座在哪里你知道吗"
print(run_conversation(query, stream=False))
query = "你们的咖啡厅里有香蕉吗"
print(run_conversation(query, stream=False))

2
robowaiter/llm_client/tool_api_multi_round.py
View File

@ -36,7 +36,7 @@ if __name__ == "__main__":
question = input("\n顾客：") question = input("\n顾客：")
data_memory = [{ data_memory = [{
"role": "system", "role": "system",
"content": "你是RoboWaiter,一个由HPCL团队开发的机器人服务员，你在咖啡厅工作。接受顾客的指令并调用工具函数来完成各种服务任务。", "content": "你是RoboWaiter,一个由HPCL团队开发的机器人服务员，你在咖啡厅工作。接受顾客的指令并调用工具函数来完成各种服务任务。如果顾客问你们这里有什么，或者想要点单，你说我们咖啡厅提供咖啡，水，点心，酸奶等食物。如果顾客不需要你了，你就回到吧台招待。",
},] },]
n = 1 n = 1
max_retry = 5 max_retry = 5

2
robowaiter/robot/Default.ptml
View File

@ -22,6 +22,6 @@ selector
cond At(Robot,Bar) cond At(Robot,Bar)
act MoveTo(Bar) act MoveTo(Bar)
} }
act GreatCustomer() act GreetCustomer()
} }
} }

155
robowaiter/scene/scene.py
View File

@ -6,6 +6,7 @@ import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from robowaiter.proto import camera from robowaiter.proto import camera
from robowaiter.proto import semantic_map from robowaiter.proto import semantic_map
from robowaiter.algos.navigator.navigate import Navigator
import math import math
from robowaiter.proto import GrabSim_pb2 from robowaiter.proto import GrabSim_pb2
from robowaiter.proto import GrabSim_pb2_grpc from robowaiter.proto import GrabSim_pb2_grpc
@ -49,6 +50,7 @@ def show_image(camera_data):
class Scene: class Scene:
robot = None robot = None
event_list = [] event_list = []
new_event_list = []
show_bubble = False show_bubble = False
default_state = { default_state = {
@ -62,7 +64,12 @@ class Scene:
"condition_set": {'At(Robot,Bar)', 'Is(AC,Off)', "condition_set": {'At(Robot,Bar)', 'Is(AC,Off)',
'Holding(Nothing)','Exist(Yogurt)','Exist(BottledDrink)','On(Yogurt,Bar)','On(BottledDrink,Table1)', 'Holding(Nothing)','Exist(Yogurt)','Exist(BottledDrink)','On(Yogurt,Bar)','On(BottledDrink,Table1)',
'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)', 'Is(HallLight,Off)', 'Is(TubeLight,On)', 'Is(Curtain,On)',
'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'} 'Is(Table1,Dirty)', 'Is(Floor,Dirty)', 'Is(Chairs,Dirty)'},
"obj_mem":{},
"customer_mem":{},
"served_mem":{},
"greeted_customers":set(),
"attention":{}
} }
""" """
status: status:
@ -90,21 +97,21 @@ class Scene:
robot.load_BT() robot.load_BT()
self.robot = robot self.robot = robot
# myx op
# 1-7 正常执行, 8-10 控灯操作移动到6, 11-12窗帘操作不需要移动, # 1-7 正常执行, 8-10 控灯操作移动到6, 11-12窗帘操作不需要移动,
self.op_dialog = ["","制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7 self.op_dialog = ["","制作咖啡","倒水","夹点心","拖地","擦桌子","开筒灯","搬椅子", # 1-7
"关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12 "关筒灯","开大厅灯","关大厅灯","关闭窗帘","打开窗帘", # 8-12
"调整空调开关","调高空调温度","调低空调温度", # 13-15 "调整空调开关","调高空调温度","调低空调温度", # 13-15
"抓握物体","放置物体"] # 16-17 "抓握物体","放置物体"] # 16-17
# 动画控制的执行步骤数
self.op_act_num = [0,3,4,6,3,2,0,1, self.op_act_num = [0,3,4,6,3,2,0,1,
0,0,0,0,0, 0,0,0,0,0,
0,0,0, 0,0,0,
0,0] 0,0]
# 动画控制的执行区域坐标
self.op_v_list = [[0.0,0.0],[250.0, 310.0],[-70.0, 480.0],[250.0, 630.0],[-70.0, 740.0],[260.0, 1120.0],[300.0, -220.0], self.op_v_list = [[0.0,0.0],[250.0, 310.0],[-70.0, 480.0],[250.0, 630.0],[-70.0, 740.0],[260.0, 1120.0],[300.0, -220.0],
[0.0, -70.0]] [0.0, -70.0]]
self.op_typeToAct = {8:[6,2],9:[6,3],10:[6,4],11:[8,1],12:[8,2]} self.op_typeToAct = {8:[6,2],9:[6,3],10:[6,4],11:[8,1],12:[8,2]} # 任务类型到行动的映射
# 空调面板位置 self.obj_loc = [300.5, -140.0,114] # 空调面板位置
self.obj_loc = [300.5, -140.0,114]
# AEM # AEM
self.visited = set() self.visited = set()
@ -143,9 +150,20 @@ class Scene:
self.time = time.time() - self.start_time self.time = time.time() - self.start_time
self.deal_event() self.deal_event()
self.deal_new_event()
self._step() self._step()
self.robot.step() self.robot.step()
def deal_new_event(self):
if len(self.new_event_list)>0:
next_event = self.new_event_list[0]
t,func,args = next_event
if self.time >= t:
print(f'event: {t}, {func.__name__}')
self.new_event_list.pop(0)
func(*args)
def deal_event(self): def deal_event(self):
if len(self.event_list)>0: if len(self.event_list)>0:
next_event = self.event_list[0] next_event = self.event_list[0]
@ -157,9 +175,9 @@ class Scene:
def create_chat_event(self,sentence): def create_chat_event(self,sentence):
def customer_say(): def customer_say():
print(f'顾客说：{sentence}') print(f'{sentence}')
if self.show_bubble: if self.show_bubble:
self.chat_bubble(f'顾客说：{sentence}') self.chat_bubble(f'{sentence}')
self.state['chat_list'].append(f'{sentence}') self.state['chat_list'].append(f'{sentence}')
return customer_say return customer_say
@ -173,6 +191,10 @@ class Scene:
return set_sub_task return set_sub_task
def new_set_goal(self,goal):
g = eval("{'" + goal + "'}")
self.state['chat_list'].append(g)
@property @property
def status(self): def status(self):
@ -198,7 +220,6 @@ class Scene:
pass pass
def walker_control_generator(self, walkerID, autowalk, speed, X, Y, Yaw): def walker_control_generator(self, walkerID, autowalk, speed, X, Y, Yaw):
if self.use_offset: if self.use_offset:
X, Y = X + loc_offset[0], Y + loc_offset[1] X, Y = X + loc_offset[0], Y + loc_offset[1]
@ -218,6 +239,10 @@ class Scene:
return scene return scene
def walker_walk_to(self,walkerID,X,Y,speed=50,Yaw=0):
self.control_walker(
[self.walker_control_generator(walkerID=walkerID, autowalk=False, speed=speed, X=X, Y=Y, Yaw=Yaw)])
def reachable_check(self, X, Y, Yaw): def reachable_check(self, X, Y, Yaw):
if self.use_offset: if self.use_offset:
@ -245,9 +270,19 @@ class Scene:
print('当前位置不可达,无法初始化NPC') print('当前位置不可达,无法初始化NPC')
else: else:
walker_list.append( walker_list.append(
GrabSim_pb2.WalkerList.Walker(id=id+5, pose=GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=loc[2]))) GrabSim_pb2.WalkerList.Walker(id=id, pose=GrabSim_pb2.Pose(X=loc[0], Y=loc[1], Yaw=loc[2])))
stub.AddWalker(GrabSim_pb2.WalkerList(walkers=walker_list, scene=self.sceneID)) stub.AddWalker(GrabSim_pb2.WalkerList(walkers=walker_list, scene=self.sceneID))
w = self.status.walkers
num_customer = len(w)
self.state["customer_mem"][w[-1].name] = num_customer-1
def walker_index2mem(self,index):
for mem,i in self.state["customer_mem"].items():
if index == i:
return mem
def add_walkers(self,walker_loc=[[0, 880], [250, 1200], [-55, 750], [70, -200]]): def add_walkers(self,walker_loc=[[0, 880], [250, 1200], [-55, 750], [70, -200]]):
print('------------------add_walkers----------------------') print('------------------add_walkers----------------------')
for id,walker in enumerate(walker_loc): for id,walker in enumerate(walker_loc):
@ -269,8 +304,16 @@ class Scene:
# walkerID is the index of the walker in status.walkers. # walkerID is the index of the walker in status.walkers.
# Since status.walkers is a list, some walkerIDs would change after removing a walker. # Since status.walkers is a list, some walkerIDs would change after removing a walker.
remove_list.append(walkerID) remove_list.append(walkerID)
index_shift_list = [ 0 for _ in range(len(self.state["customer_mem"])) ]
stub.RemoveWalkers(GrabSim_pb2.RemoveList(IDs=remove_list, scene=self.sceneID)) stub.RemoveWalkers(GrabSim_pb2.RemoveList(IDs=remove_list, scene=self.sceneID))
w = self.status.walkers
for i in range(len(w)):
self.state["customer_mem"][w[i].name] = i
def clean_walker(self): def clean_walker(self):
stub.CleanWalkers(GrabSim_pb2.SceneID(value=self.sceneID)) stub.CleanWalkers(GrabSim_pb2.SceneID(value=self.sceneID))
@ -399,11 +442,15 @@ class Scene:
) )
) )
# def walker_bubble(self, message): def walker_bubble(self, name, message):
# status = self.status talk_content = name + ":" + message
# walker_name = status.walkers[0].name self.control_robot_action(0, 3, talk_content)
# talk_content = walker_name + ":" + message
# self.control_robot_action(0, 0, 3, talk_content) def customer_say(self,name,sentence,show_bubble=True):
print(f'{name} say: {sentence}')
if show_bubble:
self.walker_bubble(name,sentence)
self.state['chat_list'].append((name,sentence))
# def control_robot_action(self, scene_id=0, type=0, action=0, message="你好"): # def control_robot_action(self, scene_id=0, type=0, action=0, message="你好"):
# print('------------------control_robot_action----------------------') # print('------------------control_robot_action----------------------')
@ -464,22 +511,21 @@ class Scene:
scene = self.status scene = self.status
cur_pos = [scene.location.X, scene.location.Y, scene.rotation.Yaw] cur_pos = [scene.location.X, scene.location.Y, scene.rotation.Yaw]
print("Current Position:", cur_pos, "开始任务:", self.op_dialog[op_type]) print("Current Position:", cur_pos, "开始任务:", self.op_dialog[op_type])
if op_type == 11 or op_type == 12: # 开关窗帘不需要移动 if op_type == 11 or op_type == 12: # 开关窗帘不需要移动
return return
print('------------------moveTo_Area----------------------') print('------------------moveTo_Area----------------------')
if op_type < 8: if op_type < 8: # 动画控制相关任务的移动目标
walk_v = self.op_v_list[op_type] + [scene.rotation.Yaw, 180, 0] # 动画控制 walk_v = self.op_v_list[op_type] + [scene.rotation.Yaw, 180, 0]
print("walk_v:",walk_v) if 8 <= op_type <= 10: # 控灯相关任务的移动目标
if op_type>=8 and op_type<=10: walk_v = self.op_v_list[6] + [scene.rotation.Yaw, 180, 0] # 控灯 walk_v = self.op_v_list[6] + [scene.rotation.Yaw, 180, 0]
if op_type in [13,14,15]: walk_v = [240, -140.0] + [0, 180, 0] # 空调 if op_type in [13,14,15]: # 空调相关任务的移动目标
walk_v = [240, -140.0] + [0, 180, 0]
if op_type == 16: # 抓握物体，移动到物体周围的可达区域 if op_type == 16: # 抓握物体，移动到物体周围的可达区域
scene = self.status scene = self.status
obj_info = scene.objects[obj_id] obj_info = scene.objects[obj_id]
# Robot
obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z
walk_v = [obj_x + 50, obj_y] + [180, 180, 0] walk_v = [obj_x + 50, obj_y] + [180, 180, 0]
if obj_y >= 820 and obj_y <= 1200 and obj_x >= 240 and obj_x <= 500: # 物品位于斜的抹布桌上 ([240,500],[820,1200]) if 820 <= obj_y <= 1200 and 240 <= obj_x <= 500: # 物品位于斜的抹布桌上 ([240,500],[820,1200])
walk_v = [obj_x + 40, obj_y - 35, 130, 180, 0] walk_v = [obj_x + 40, obj_y - 35, 130, 180, 0]
obj_x += 3 obj_x += 3
obj_y += 2.5 obj_y += 2.5
@ -487,10 +533,13 @@ class Scene:
walk_v = release_pos[:-1] + [180, 180, 0] walk_v = release_pos[:-1] + [180, 180, 0]
if release_pos == [340.0, 900.0, 99.0]: if release_pos == [340.0, 900.0, 99.0]:
walk_v[2] = 130 walk_v[2] = 130
# 移动到目标位置
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v) action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v)
scene = stub.Do(action) scene = stub.Do(action)
print("After Walk Position:", [scene.location.X, scene.location.Y, scene.rotation.Yaw]) print("After Walk Position:", [scene.location.X, scene.location.Y, scene.rotation.Yaw])
# 相应的行动，由主办方封装 # 相应的行动，由主办方封装
def control_robot_action(self, type=0, action=0, message="你好"): def control_robot_action(self, type=0, action=0, message="你好"):
scene = stub.ControlRobot( scene = stub.ControlRobot(
@ -505,18 +554,16 @@ class Scene:
print(scene.info) print(scene.info)
return False return False
# 调整空调开关、温度
def adjust_kongtiao(self,op_type): def adjust_kongtiao(self,op_type):
print("self.obj_loc:",self.obj_loc)
obj_loc = self.obj_loc[:] obj_loc = self.obj_loc[:]
print("obj_loc:",obj_loc,"self.obj_loc:", self.obj_loc)
obj_loc[2] -= 5 obj_loc[2] -= 5
print("obj_loc:",obj_loc)
if op_type == 13: obj_loc[1] -= 2 if op_type == 13: obj_loc[1] -= 2
if op_type == 14: obj_loc[1] -= 0 if op_type == 14: obj_loc[1] -= 0
if op_type == 15: obj_loc[1] += 2 if op_type == 15: obj_loc[1] += 2
self.ik_control_joints(2, obj_loc[0], obj_loc[1], obj_loc[2]) self.ik_control_joints(2, obj_loc[0], obj_loc[1], obj_loc[2])
time.sleep(3.0) time.sleep(3.0)
self.robo_recover() self.robo_recover() # 恢复肢体关节
return True return True
def gen_obj(self,h=100): def gen_obj(self,h=100):
@ -530,7 +577,9 @@ class Scene:
scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID)) scene = stub.AddObjects(GrabSim_pb2.ObjectList(objects=obj_list, scene=self.sceneID))
time.sleep(1.0) time.sleep(1.0)
# 实现抓握操作
def grasp_obj(self,obj_id,hand_id=1): def grasp_obj(self,obj_id,hand_id=1):
print('------------------adjust_joints----------------------')
scene = self.status scene = self.status
obj_info = scene.objects[obj_id] obj_info = scene.objects[obj_id]
obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z obj_x, obj_y, obj_z = obj_info.location.X, obj_info.location.Y, obj_info.location.Z
@ -539,7 +588,7 @@ class Scene:
# obj_y -= 1 # obj_y -= 1
# values = [0,0,0,0,0, 10,-25,-45,-45,-45] # values = [0,0,0,0,0, 10,-25,-45,-45,-45]
# values= [-6, 0, 0, 0, 0, -6, 0, 45, 45, 45] # values= [-6, 0, 0, 0, 0, -6, 0, 45, 45, 45]
stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values)) # stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values))
pass pass
if obj_info.name=="Glass": if obj_info.name=="Glass":
pass pass
@ -559,12 +608,13 @@ class Scene:
values=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) values=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
scene = stub.Do(action) scene = stub.Do(action)
# 恢复手指关节
def standard_finger(self): def standard_finger(self):
values = [0,0,0,0,0, 0,0,0,0,0] values = [0,0,0,0,0, 0,0,0,0,0]
stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values)) stub.Do(GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Finger, values=values))
time.sleep(1.0) time.sleep(1.0)
# 弯腰以及手掌与放置面平齐
def robo_stoop_parallel(self): def robo_stoop_parallel(self):
# 0-3是躯干，4-6是脖子和头，7-13是左胳膊，14-20是右胳膊 # 0-3是躯干，4-6是脖子和头，7-13是左胳膊，14-20是右胳膊
scene = self.status scene = self.status
@ -577,8 +627,9 @@ class Scene:
scene = stub.Do(action) scene = stub.Do(action)
time.sleep(1.0) time.sleep(1.0)
# 实现放置操作
def release_obj(self,release_pos): def release_obj(self,release_pos):
print("------------------release_obj----------------------") print("------------------adjust_joints----------------------")
if release_pos==[340.0, 900.0, 99.0]: if release_pos==[340.0, 900.0, 99.0]:
self.ik_control_joints(2, release_pos[0]-40, release_pos[1]+35, release_pos[2]) self.ik_control_joints(2, release_pos[0]-40, release_pos[1]+35, release_pos[2])
time.sleep(2.0) time.sleep(2.0)
@ -586,25 +637,26 @@ class Scene:
self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2]) self.ik_control_joints(2, release_pos[0] - 80, release_pos[1], release_pos[2])
time.sleep(2.0) time.sleep(2.0)
self.robo_stoop_parallel() self.robo_stoop_parallel()
print("------------------release_obj----------------------")
action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Release, values=[1]) action = GrabSim_pb2.Action(scene=self.sceneID, action=GrabSim_pb2.Action.ActionType.Release, values=[1])
scene = stub.Do(action) scene = stub.Do(action)
time.sleep(2.0) time.sleep(2.0)
self.robo_recover() self.robo_recover() # 恢复肢体关节
self.standard_finger() self.standard_finger() # 恢复手指关节
return True return True
# 执行过程：输出"开始(任务名)" -> 按步骤数执行任务 -> Robot输出成功或失败的对话 # 执行过程: Robot输出"开始(任务名)" -> 按步骤数执行任务 -> Robot输出成功或失败的对话
def op_task_execute(self,op_type,obj_id=0,release_pos=[247.0, 520.0, 100.0]): def op_task_execute(self,op_type,obj_id=0,release_pos=[247.0, 520.0, 100.0]):
self.control_robot_action(0, 1, "开始"+self.op_dialog[op_type]) # 开始制作咖啡 self.control_robot_action(0, 1, "开始"+self.op_dialog[op_type]) # 输出正在执行的任务
if op_type<8: result = self.control_robot_action(op_type, 1) if op_type < 8:
if op_type>=8 and op_type<=12: result = self.control_robot_action(self.op_typeToAct[op_type][0], self.op_typeToAct[op_type][1]) result = self.control_robot_action(op_type, 1)
if 8 <= op_type <= 12:
result = self.control_robot_action(self.op_typeToAct[op_type][0], self.op_typeToAct[op_type][1])
if op_type in [13, 14, 15]: # 调整空调:13代表按开关,14升温,15降温 if op_type in [13, 14, 15]: # 调整空调:13代表按开关,14升温,15降温
result = self.adjust_kongtiao(op_type) result = self.adjust_kongtiao(op_type)
if op_type ==16: # 抓握物体 if op_type == 16: # 抓握物体, 需要传入物品id
result = self.grasp_obj(obj_id) result = self.grasp_obj(obj_id)
if op_type ==17: # 放置物体 if op_type == 17: # 放置物体, 放置物品, 需要传入放置地点
result = self.release_obj(release_pos) result = self.release_obj(release_pos)
self.control_robot_action(0, 2) self.control_robot_action(0, 2)
if result: if result:
@ -634,20 +686,35 @@ class Scene:
def navigation_move(self, cur_objs, objs_name_set, v_list, scene_id=0, map_id=11): def navigation_move(self, cur_objs, objs_name_set, v_list, scene_id=0, map_id=11):
print('------------------navigation_move----------------------') print('------------------navigation_move----------------------')
scene = stub.Observe(GrabSim_pb2.SceneID(value=scene_id)) scene = stub.Observe(GrabSim_pb2.SceneID(value=scene_id))
walk_value = [scene.location.X, scene.location.Y, scene.rotation.Yaw] walk_value = [scene.location.X, scene.location.Y]
print("position:", walk_value) print("position:", walk_value)
if not cur_objs:
walk_v = [scene.location.X, scene.location.Y + 1]
yaw = Navigator.get_yaw(walk_value, walk_v)
walk_v = walk_value + [yaw, 250, 10]
print("walk_v", walk_v)
action = GrabSim_pb2.Action(scene=scene_id, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v)
scene = stub.Do(action)
cur_objs, objs_name_set = camera.get_semantic_map(GrabSim_pb2.CameraName.Head_Segment, cur_objs,
objs_name_set)
# if scene.info == "Unreachable":
print(scene.info)
# if map_id == 11: # coffee # if map_id == 11: # coffee
# v_list = [[0, 880], [250, 1200], [-55, 750], [70, -200]] # v_list = [[0, 880], [250, 1200], [-55, 750], [70, -200]]
# else: # else:
# v_list = [[0.0, 0.0]] # v_list = [[0.0, 0.0]]
else:
for walk_v in v_list: for walk_v in v_list:
walk_v = walk_v + [scene.rotation.Yaw - 90, 250, 10] yaw = Navigator.get_yaw(walk_value, walk_v)
walk_v = walk_v + [yaw, 250, 10]
print("walk_v", walk_v) print("walk_v", walk_v)
action = GrabSim_pb2.Action(scene=scene_id, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v) action = GrabSim_pb2.Action(scene=scene_id, action=GrabSim_pb2.Action.ActionType.WalkTo, values=walk_v)
scene = stub.Do(action) scene = stub.Do(action)
cur_objs, objs_name_set = camera.get_semantic_map(GrabSim_pb2.CameraName.Head_Segment, cur_objs, objs_name_set) cur_objs, objs_name_set = camera.get_semantic_map(GrabSim_pb2.CameraName.Head_Segment, cur_objs,
objs_name_set)
# if scene.info == "Unreachable": # if scene.info == "Unreachable":
print(scene.info) print(scene.info)
return cur_objs, objs_name_set return cur_objs, objs_name_set
@ -746,7 +813,3 @@ class Scene:
return True return True
else: else:
return False return False

1
robowaiter/scene/tasks/Auto_tasks.py
View File

@ -32,7 +32,6 @@ class SceneAT(Scene):
pass pass
if __name__ == '__main__': if __name__ == '__main__':
import os
from robowaiter.robot.robot import Robot from robowaiter.robot.robot import Robot
robot = Robot() robot = Robot()

0
robowaiter/scene/tasks/Open_tasks.py → robowaiter/scene/tasks/OT/Open_tasks.py
View File

0
robowaiter/scene/tasks/Open_tasks_test.py → robowaiter/scene/tasks/OT/Open_tasks_bright_table.py
View File

57
robowaiter/scene/tasks/OT/Open_tasks_test.py Normal file
View File

@ -0,0 +1,57 @@
"""
人提出请求，机器人完成任务
1. 做咖啡（固定动画）：接收到做咖啡指令、走到咖啡机、拿杯子、操作咖啡机、取杯子、送到客人桌子上
2. 倒水
3. 夹点心
具体描述：设计一套点单规则（如菜单包含咖啡、水、点心等），按照规则拟造随机的订单。在收到订单后，通过大模型让机器人输出合理的备餐计划，并尝试在模拟环境中按照这个规划实现任务。
"""
# todo: 接收点单信息，大模型生成任务规划
from robowaiter.scene.scene import Scene
class SceneOT(Scene):
def __init__(self, robot):
super().__init__(robot)
# 在这里加入场景中发生的事件
self.new_event_list = [
# (9,self.add_walkers,([[0, 880]],)),
# (10,self.walker_walk_to,(2,50,500))
# (5, self.set_goal("On(Yogurt,Table4)"))
# (5, self.set_goal("At(Robot,BrightTable4)"))
]
def _reset(self):
# self.add_walkers([[0, 880], [250, 1200]])
# 展示顾客，前8个id是小孩，后面都是大人
for i in range(4):
self.add_walker(i,50,300 + i * 50)
name1 = self.walker_index2mem(1)
name2 = self.walker_index2mem(3)
self.remove_walker(0,2)
index1 = self.state["customer_mem"][name1]
index2 = self.state["customer_mem"][name2]
self.walker_bubble(name1,f"我是第{index1}个")
self.walker_bubble(name2,f"我是第{index2}个")
def _run(self):
pass
if __name__ == '__main__':
import os
from robowaiter.robot.robot import Robot
robot = Robot()
# create task
task = SceneOT(robot)
task.reset()
task.run()

0
robowaiter/scene/tasks/OT/__init__.py Normal file
View File