Link cameras with their corresponding microphones for joint data handling

lerobot/common/robot_devices/cameras/configs.py

@@ -48,6 +48,8 @@ class OpenCVCameraConfig(CameraConfig):
     rotation: int | None = None
     mock: bool = False
 
+    microphone: str | None = None
+
     def __post_init__(self):
         if self.color_mode not in ["rgb", "bgr"]:
             raise ValueError(

lerobot/common/robot_devices/cameras/opencv.py

@@ -281,6 +281,8 @@ class OpenCVCamera:
         elif config.rotation == 180:
             self.rotation = cv2.ROTATE_180
 
+        self.microphone = config.microphone
+
     def connect(self):
         if self.is_connected:
             raise RobotDeviceAlreadyConnectedError(f"OpenCVCamera({self.camera_index}) is already connected.")

lerobot/common/robot_devices/robots/manipulator.py

@@ -181,6 +181,7 @@ class ManipulatorRobot:
                 "shape": (cam.height, cam.width, cam.channels),
                 "names": ["height", "width", "channels"],
                 "info": None,
+                "audio": "observation.audio." + cam.microphone if cam.microphone is not None else None,
             }
         return cam_ft
 
@@ -207,9 +208,9 @@ class ManipulatorRobot:
         for mic_key, mic in self.microphones.items():
             key = f"observation.audio.{mic_key}"
             mic_ft[key] = {
-                "dtype": mic.data_type,
-                "shape": (mic.channels,),
-                "info": None,
+                "shape": (len(mic.channels),),
+                "names": "channels",
+                "info" : None,
             }
         return mic_ft