EmbodiedAI
/
RoboWaiter
mirror of https://github.com/HPCL-EI/RoboWaiter.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

更新 VLM2 机器人带小女孩到窗边的位置

This commit is contained in:
Caiyishuai 2023-11-17 20:04:39 +08:00
parent 68fcf72d62
commit aed3cd9aae
7 changed files with 60 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
robowaiter/behavior_lib/_base/Behavior.py
View File

@ -30,7 +30,7 @@ class Bahavior(ptree.behaviour.Behaviour):
# all_place=set()
place_xyz_dic={
'Bar': (247.0, 520.0, 100.0),
'Bar': (247.0, 520.0, 180.0), #(247.0, 520.0, 100.0)
'Bar2': (240.0, 40.0, 70.0),
'WaterTable':(-70.0, 500.0, 107),
'CoffeeTable':(250.0, 310.0, 100.0),

2
robowaiter/llm_client/data/test_questions.txt
View File

@ -1 +1 @@
{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "请问可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}
{"做一杯咖啡": {"Answer": "OK，我这就去做一杯咖啡", "Goal": "{\"On(Coffee,CoffeeTable)\"}"}, "做一杯咖啡放到吧台上": {"Answer": "OK，我这就去做一杯咖啡放到吧台上", "Goal": "{\"On(Coffee,Bar)\"}"}, "做一杯咖啡放到水杯桌上，再倒一杯水": {"Answer": "OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Water,WaterTable)\"}"}, "前往2号桌": {"Answer": "OK，我这前往2号桌", "Goal": "{\"At(Robot,Table2)\"}"}, "测试AEM": {"Answer": "测试AEM", "Goal": "{\"EnvExplored()\"}"}, "倒一杯水": {"Answer": "OK，我这就去倒一杯水", "Goal": "{\"On(Water,WaterTable)\"}"}, "开空调": {"Answer": "OK，我这就去开空调", "Goal": "{\"Is(AC,On)\"}"}, "关空调": {"Answer": "OK，我这就去关空调", "Goal": "{\"Is(AC,Off)\"}"}, "关大厅灯": {"Answer": "OK，我这就去关大厅灯", "Goal": "{\"Is(HallLight,Off)\"}"}, "开大厅灯": {"Answer": "OK，我这就去开大厅灯", "Goal": "{\"Is(HallLight,On)\"}"}, "关筒灯": {"Answer": "OK，我这就去关筒灯", "Goal": "{\"Is(TubeLight,Off)\"}"}, "开筒灯": {"Answer": "OK，我这就去开筒灯", "Goal": "{\"Is(TubeLight,On)\"}"}, "关窗帘": {"Answer": "OK，我这就去关窗帘", "Goal": "{\"Is(Curtain,Off)\"}"}, "开窗帘": {"Answer": "OK，我这就去开窗帘", "Goal": "{\"Is(Curtain,On)\"}"}, "拖地": {"Answer": "OK，我这就去拖地", "Goal": "{\"Is(Floor,Clean)\"}"}, "擦桌子": {"Answer": "OK，我这就去擦桌子", "Goal": "{\"Is(Table1,Clean)\"}"}, "整理椅子": {"Answer": "OK，我这就去整理椅子", "Goal": "{\"Is(Chairs,Clean)\"}"}, "把冰红茶放到Table2": {"Answer": "OK，我这就去把冰红茶放到Table2", "Goal": "{\"On(BottledDrink,Table2)\"}"}, "我有点热，能开个空调吗？": {"Answer": "当然可以，我现在就开！", "Goal": "{\"Is(AC,On)\"}"}, "可以带我去吗": {"Answer": "当然可以，前往一号桌", "Goal": "{\"At(Robot,Table1)\"}"}, "把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。": {"Answer": "明白，我这就去办！", "Goal": "{\"On(Coffee,WaterTable)\",\"On(Yogurt,Table1)\",\"On(Water,WaterTable)\"}"}, "下班啦！打扫卫生，关灯关空调关窗帘。": {"Answer": "太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。", "Goal": "{\"Is(Floor,Clean)\",\"Is(Table1,Clean)\",\"Is(Chairs,Clean)\",\"Is(AC,Off)\",\"Is(HallLight,Off)\",\"Is(TubeLight,Off)\",\"Is(Curtain,Off)\"}"}, "可以带我去空位上嘛？我想晒太阳。": {"Answer": "没问题！请跟我来。", "Goal": "{\"At(Robot,BrightTable1)\"}"}}

2
robowaiter/llm_client/data_raw/test_questions.csv
View File

@ -21,4 +21,4 @@ Question,Answer,Goal
可以带我去吗,当然可以，前往一号桌,"{""At(Robot,Table1)""}"
把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。,明白，我这就去办！,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
下班啦！打扫卫生，关灯关空调关窗帘。,太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
请问可以带我去空位上嘛？我想晒太阳。,没问题！请跟我来。,"{""At(Robot,BrightTable1)""}"
可以带我去空位上嘛？我想晒太阳。,没问题！请跟我来。,"{""At(Robot,BrightTable1)""}"

1 Question Answer Goal
21 可以带我去吗 当然可以，前往一号桌 {"At(Robot,Table1)"}
22 把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。 明白，我这就去办！ {"On(Coffee,WaterTable)","On(Yogurt,Table1)","On(Water,WaterTable)"}
23 下班啦！打扫卫生，关灯关空调关窗帘。 太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。 {"Is(Floor,Clean)","Is(Table1,Clean)","Is(Chairs,Clean)","Is(AC,Off)","Is(HallLight,Off)","Is(TubeLight,Off)","Is(Curtain,Off)"}
24 请问可以带我去空位上嘛？我想晒太阳。 可以带我去空位上嘛？我想晒太阳。 没问题！请跟我来。 {"At(Robot,BrightTable1)"}

24
robowaiter/llm_client/data_raw/test_questions.csv~ Normal file
View File

@ -0,0 +1,24 @@
Question,Answer,Goal
做一杯咖啡,OK，我这就去做一杯咖啡,"{""On(Coffee,CoffeeTable)""}"
做一杯咖啡放到吧台上,OK，我这就去做一杯咖啡放到吧台上,"{""On(Coffee,Bar)""}"
做一杯咖啡放到水杯桌上，再倒一杯水,OK，我这就去做一杯咖啡放到水杯桌上，再倒一杯水,"{""On(Coffee,WaterTable)"",""On(Water,WaterTable)""}"
前往2号桌,OK，我这前往2号桌,"{""At(Robot,Table2)""}"
测试AEM,测试AEM,"{""EnvExplored()""}"
倒一杯水,OK，我这就去倒一杯水,"{""On(Water,WaterTable)""}"
开空调,OK，我这就去开空调,"{""Is(AC,On)""}"
关空调,OK，我这就去关空调,"{""Is(AC,Off)""}"
关大厅灯,OK，我这就去关大厅灯,"{""Is(HallLight,Off)""}"
开大厅灯,OK，我这就去开大厅灯,"{""Is(HallLight,On)""}"
关筒灯,OK，我这就去关筒灯,"{""Is(TubeLight,Off)""}"
开筒灯,OK，我这就去开筒灯,"{""Is(TubeLight,On)""}"
关窗帘,OK，我这就去关窗帘,"{""Is(Curtain,Off)""}"
开窗帘,OK，我这就去开窗帘,"{""Is(Curtain,On)""}"
拖地,OK，我这就去拖地,"{""Is(Floor,Clean)""}"
擦桌子,OK，我这就去擦桌子,"{""Is(Table1,Clean)""}"
整理椅子,OK，我这就去整理椅子,"{""Is(Chairs,Clean)""}"
把冰红茶放到Table2,OK，我这就去把冰红茶放到Table2,"{""On(BottledDrink,Table2)""}"
我有点热，能开个空调吗？,当然可以，我现在就开！,"{""Is(AC,On)""}"
可以带我去吗,当然可以，前往一号桌,"{""At(Robot,Table1)""}"
把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。,明白，我这就去办！,"{""On(Coffee,WaterTable)"",""On(Yogurt,Table1)"",""On(Water,WaterTable)""}"
下班啦！打扫卫生，关灯关空调关窗帘。,太棒啦，下班啦！我这就去打扫卫生，关灯关空调关窗帘。,"{""Is(Floor,Clean)"",""Is(Table1,Clean)"",""Is(Chairs,Clean)"",""Is(AC,Off)"",""Is(HallLight,Off)"",""Is(TubeLight,Off)"",""Is(Curtain,Off)""}"
请问可以带我去空位上嘛？我想晒太阳。,没问题！请跟我来。,"{""At(Robot,BrightTable1)""}"

17
robowaiter/scene/scene.py
View File

@ -164,6 +164,8 @@ class Scene:
return customer_say
def set_goal(self,goal):
g = eval("{'" + goal + "'}")
def set_sub_task():
@ -277,6 +279,21 @@ class Scene:
GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
)
def control_walkers_and_say(self, control_list_ls):
""" 同时处理行人的行走和对话
control_list_ls =[walkerID,autowalk,speed,X,Y,Yaw,cont]
"""
control_list= []
for control in control_list_ls:
if control[-1]!= None:
walkerID = control[0]
cont = self.status.walkers[walkerID].name + ":"+control[-1]
self.control_robot_action(control[walkerID], 3, cont)
control_list.append(self.walker_control_generator(walkerID=control[0], autowalk=control[1], speed=control[2], X=control[3], Y=control[4], Yaw=control[5]))
# 收集没有对话的统一控制
stub.ControlWalkers(
GrabSim_pb2.WalkerControls(controls=control_list, scene=self.sceneID)
)
def control_walkers(self,walker_loc=[[-55, 750], [70, -200], [250, 1200], [0, 880]],is_autowalk = True):
"""pose:表示行人的终止位置姿态"""

34
robowaiter/scene/tasks/VLM/VLM2.py
View File

@ -11,7 +11,7 @@ class SceneVLM(Scene):
super().__init__(robot)
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
self.event_list = [
(5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
(5, self.create_chat_event("可以带我去空位上嘛？我想晒太阳。"))
]
def _reset(self):
@ -26,28 +26,24 @@ class SceneVLM(Scene):
# 带领行人去有太阳的地方
# 行人说 有点热
# 好的，这就去开空调
self.control_walker(
[self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=0)])
time.sleep(3)
self.chat_bubble("欢迎光临！请问有什么可以帮您？")
time.sleep(3)
cont = self.status.walkers[0].name+":可以带我去空位上嘛？我想晒太阳。"
self.control_robot_action(0,3,cont)
time.sleep(4)
# self.event_list.append((5, self.create_chat_event("可以带我去空位上嘛？我想晒太阳。")))
pass
def _step(self):
self.control_walker(
[self.walker_control_generator(walkerID=0, autowalk=False, speed=200, X=60, Y=520, Yaw=180)])
time.sleep(3)
cont = self.status.walkers[0].name+":请问可以带我去空位上嘛？我想晒太阳。"
self.control_robot_action(0,3,cont)
# 如果机器人不在 吧台
# if "At(Robot,Bar)" not in self.state['condition_set']:
end = [self.status.location.X, self.status.location.Y]
# print("end:",end)
if end[1]>=600 or end[1]<=450 or end[0]>=250:
# if int(self.status.location.X)!=247 or int(self.status.location.X)!=520:
self.control_walker(
[self.walker_control_generator(walkerID=0, autowalk=False, speed=100, X=end[0], Y=end[1], Yaw=-90)])
cont = self.status.walkers[0].name+"谢谢！"
self.control_robot_action(0,3,cont)
if "At(Robot,Bar)" not in self.state['condition_set']:
end = [self.status.location.X, self.status.location.Y]
self.control_walkers_and_say([[0, False, 100, end[0], end[1], -90, "谢谢！"]]) # walkerID,autowalk,speed,X,Y,Yaw,cont
pass

2
robowaiter/scene/tasks/VLM/VLM3.py
View File

@ -11,7 +11,7 @@ class SceneVLM(Scene):
super().__init__(robot)
# 在这里加入场景中发生的事件， (事件发生的时间，事件函数)
self.event_list = [
(5, self.create_chat_event("请问可以带我去空位上嘛？我想晒太阳。")),
(5, self.create_chat_event("把酸奶放到1号桌，再做一杯咖啡送到水杯桌上，再倒一杯水。")),
]
def _reset(self):