wangzixiang
/
lerobot
mirror of https://github.com/huggingface/lerobot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Link cameras with their corresponding microphones for joint data handling

This commit is contained in:
CarolinePascal 2025-04-01 20:36:29 +02:00
parent e6ea8e75c3
commit 8ddfb299fd
No known key found for this signature in database
3 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerobot/common/robot_devices/cameras/configs.py
View File

@ -48,6 +48,8 @@ class OpenCVCameraConfig(CameraConfig):
rotation: int | None = None rotation: int | None = None
mock: bool = False mock: bool = False
microphone: str | None = None
def __post_init__(self): def __post_init__(self):
if self.color_mode not in ["rgb", "bgr"]: if self.color_mode not in ["rgb", "bgr"]:
raise ValueError( raise ValueError(

2
lerobot/common/robot_devices/cameras/opencv.py
View File

@ -281,6 +281,8 @@ class OpenCVCamera:
elif config.rotation == 180: elif config.rotation == 180:
self.rotation = cv2.ROTATE_180 self.rotation = cv2.ROTATE_180
self.microphone = config.microphone
def connect(self): def connect(self):
if self.is_connected: if self.is_connected:
raise RobotDeviceAlreadyConnectedError(f"OpenCVCamera({self.camera_index}) is already connected.") raise RobotDeviceAlreadyConnectedError(f"OpenCVCamera({self.camera_index}) is already connected.")

7
lerobot/common/robot_devices/robots/manipulator.py
View File

@ -181,6 +181,7 @@ class ManipulatorRobot:
"shape": (cam.height, cam.width, cam.channels), "shape": (cam.height, cam.width, cam.channels),
"names": ["height", "width", "channels"], "names": ["height", "width", "channels"],
"info": None, "info": None,
"audio": "observation.audio." + cam.microphone if cam.microphone is not None else None,
} }
return cam_ft return cam_ft
@ -207,9 +208,9 @@ class ManipulatorRobot:
for mic_key, mic in self.microphones.items(): for mic_key, mic in self.microphones.items():
key = f"observation.audio.{mic_key}" key = f"observation.audio.{mic_key}"
mic_ft[key] = { mic_ft[key] = {
"dtype": mic.data_type, "shape": (len(mic.channels),),
"shape": (mic.channels,), "names": "channels",
"info": None, "info" : None,
} }
return mic_ft return mic_ft