From 8ddfb299fdd0a8255dd9cb838e1e8533e264be33 Mon Sep 17 00:00:00 2001
From: CarolinePascal <caroline8.pascal@gmail.com>
Date: Tue, 1 Apr 2025 20:36:29 +0200
Subject: [PATCH] Link cameras with their corresponding microphones for joint
 data handling

---
 lerobot/common/robot_devices/cameras/configs.py    | 2 ++
 lerobot/common/robot_devices/cameras/opencv.py     | 2 ++
 lerobot/common/robot_devices/robots/manipulator.py | 7 ++++---
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/lerobot/common/robot_devices/cameras/configs.py b/lerobot/common/robot_devices/cameras/configs.py
index 013419a9..b1bb588c 100644
--- a/lerobot/common/robot_devices/cameras/configs.py
+++ b/lerobot/common/robot_devices/cameras/configs.py
@@ -48,6 +48,8 @@ class OpenCVCameraConfig(CameraConfig):
     rotation: int | None = None
     mock: bool = False
 
+    microphone: str | None = None
+
     def __post_init__(self):
         if self.color_mode not in ["rgb", "bgr"]:
             raise ValueError(
diff --git a/lerobot/common/robot_devices/cameras/opencv.py b/lerobot/common/robot_devices/cameras/opencv.py
index f279f315..757b3d9f 100644
--- a/lerobot/common/robot_devices/cameras/opencv.py
+++ b/lerobot/common/robot_devices/cameras/opencv.py
@@ -281,6 +281,8 @@ class OpenCVCamera:
         elif config.rotation == 180:
             self.rotation = cv2.ROTATE_180
 
+        self.microphone = config.microphone
+
     def connect(self):
         if self.is_connected:
             raise RobotDeviceAlreadyConnectedError(f"OpenCVCamera({self.camera_index}) is already connected.")
diff --git a/lerobot/common/robot_devices/robots/manipulator.py b/lerobot/common/robot_devices/robots/manipulator.py
index 9212bb9c..443466ed 100644
--- a/lerobot/common/robot_devices/robots/manipulator.py
+++ b/lerobot/common/robot_devices/robots/manipulator.py
@@ -181,6 +181,7 @@ class ManipulatorRobot:
                 "shape": (cam.height, cam.width, cam.channels),
                 "names": ["height", "width", "channels"],
                 "info": None,
+                "audio": "observation.audio." + cam.microphone if cam.microphone is not None else None,
             }
         return cam_ft
 
@@ -207,9 +208,9 @@ class ManipulatorRobot:
         for mic_key, mic in self.microphones.items():
             key = f"observation.audio.{mic_key}"
             mic_ft[key] = {
-                "dtype": mic.data_type,
-                "shape": (mic.channels,),
-                "info": None,
+                "shape": (len(mic.channels),),
+                "names": "channels",
+                "info" : None,
             }
         return mic_ft