From c586de3e026b8ef34e3ac63502d89b0263fa867d Mon Sep 17 00:00:00 2001 From: Steven Palma Date: Wed, 9 Apr 2025 13:44:07 +0200 Subject: [PATCH] chore: address PR feedback --- README.md | 5 +++++ benchmarks/video/capture_camera_feed.py | 12 +++++++----- examples/10_use_so100.md | 6 +----- examples/11_use_lekiwi.md | 6 +----- examples/11_use_moss.md | 6 +----- examples/7_get_started_with_real_robot.md | 10 +++++----- examples/8_use_stretch.md | 7 +------ examples/9_use_aloha.md | 6 +----- lerobot/common/robot_devices/control_configs.py | 3 +++ lerobot/common/robot_devices/control_utils.py | 4 ---- lerobot/scripts/control_robot.py | 16 +++++++--------- pyproject.toml | 2 +- 12 files changed, 33 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 20ebeee8..3850f2d4 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,11 @@ When using `miniconda`, install `ffmpeg` in your environment: conda install ffmpeg -c conda-forge ``` +> **NOTE:** This usually installs `ffmpeg 7.X` for your platform (check the version installed with `ffmpeg -encoders | grep libsvtav1`). If it isn't `ffmpeg 7.X` or lacks `libsvtav1` support, you can explicitly install `ffmpeg 7.X` using: +``` +conda install ffmpeg=7.1.1 -c conda-forge +``` + Install 🤗 LeRobot: ```bash pip install -e . diff --git a/benchmarks/video/capture_camera_feed.py b/benchmarks/video/capture_camera_feed.py index 5abba244..ce248f20 100644 --- a/benchmarks/video/capture_camera_feed.py +++ b/benchmarks/video/capture_camera_feed.py @@ -24,11 +24,13 @@ from pathlib import Path import cv2 import rerun as rr +# see https://rerun.io/docs/howto/visualization/limit-ram +RERUN_MEMORY_LIMIT = os.getenv("LEROBOT_RERUN_MEMORY_LIMIT", "5%") + def display_and_save_video_stream(output_dir: Path, fps: int, width: int, height: int, duration: int): rr.init("lerobot_capture_camera_feed") - memory_limit = os.getenv("LEROBOT_RERUN_MEMORY_LIMIT", "5%") - rr.spawn(memory_limit=memory_limit) + rr.spawn(memory_limit=RERUN_MEMORY_LIMIT) now = dt.datetime.now() capture_dir = output_dir / f"{now:%Y-%m-%d}" / f"{now:%H-%M-%S}" @@ -57,10 +59,10 @@ def display_and_save_video_stream(output_dir: Path, fps: int, width: int, height cv2.imwrite(str(capture_dir / f"frame_{frame_index:06d}.png"), frame) frame_index += 1 - # Release the capture and destroy all windows + # Release the capture cap.release() - # TODO(Steven): Find a way to close visualizer: https://github.com/rerun-io/rerun/pull/9400 - # cv2.destroyAllWindows() + + # TODO(Steven): Add a graceful shutdown via a close() method for the Viewer context, though not currently supported in the Rerun API. if __name__ == "__main__": diff --git a/examples/10_use_so100.md b/examples/10_use_so100.md index 38f90452..9dbe974c 100644 --- a/examples/10_use_so100.md +++ b/examples/10_use_so100.md @@ -498,11 +498,7 @@ python lerobot/scripts/control_robot.py \ #### a. Teleop with displaying cameras Follow [this guide to setup your cameras](https://github.com/huggingface/lerobot/blob/main/examples/7_get_started_with_real_robot.md#c-add-your-cameras-with-opencvcamera). Then you will be able to display the cameras on your computer while you are teleoperating by running the following code. This is useful to prepare your setup before recording your first dataset. -> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. You can adjust the viewer's behavior with these environment variables (refer to `rerun` documentation for more information): -> - [`RERUN_FLUSH_NUM_BYTES`](https://rerun.io/docs/reference/sdk/micro-batching) [default: 8000] -> - [`LEROBOT_RERUN_MEMORY_LIMIT`](https://rerun.io/docs/howto/visualization/limit-ram) [default: 5%] -> - [`LEROBOT_VIEWER_IP`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: None] -> - [`LEROBOT_VIEWER_PORT`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: 9876] +> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. ```bash python lerobot/scripts/control_robot.py \ diff --git a/examples/11_use_lekiwi.md b/examples/11_use_lekiwi.md index 1a6517d4..1be7cbc4 100644 --- a/examples/11_use_lekiwi.md +++ b/examples/11_use_lekiwi.md @@ -424,11 +424,7 @@ python lerobot/scripts/control_robot.py \ --control.fps=30 ``` -> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. You can adjust the viewer's behavior with these environment variables (refer to `rerun` documentation for more information): -> - [`RERUN_FLUSH_NUM_BYTES`](https://rerun.io/docs/reference/sdk/micro-batching) [default: 8000] -> - [`LEROBOT_RERUN_MEMORY_LIMIT`](https://rerun.io/docs/howto/visualization/limit-ram) [default: 5%] -> - [`LEROBOT_VIEWER_IP`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: None] -> - [`LEROBOT_VIEWER_PORT`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: 9876] +> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. For the `--control.type=remote_robot` you will also need to set `--control.viewer_ip` and `--control.viewer_port` You should see on your laptop something like this: ```[INFO] Connected to remote robot at tcp://172.17.133.91:5555 and video stream at tcp://172.17.133.91:5556.``` Now you can move the leader arm and use the keyboard (w,a,s,d) to drive forward, left, backwards, right. And use (z,x) to turn left or turn right. You can use (r,f) to increase and decrease the speed of the mobile robot. There are three speed modes, see the table below: | Speed Mode | Linear Speed (m/s) | Rotation Speed (deg/s) | diff --git a/examples/11_use_moss.md b/examples/11_use_moss.md index 074c2197..1b6f23b9 100644 --- a/examples/11_use_moss.md +++ b/examples/11_use_moss.md @@ -219,11 +219,7 @@ python lerobot/scripts/control_robot.py \ **Teleop with displaying cameras** Follow [this guide to setup your cameras](https://github.com/huggingface/lerobot/blob/main/examples/7_get_started_with_real_robot.md#c-add-your-cameras-with-opencvcamera). Then you will be able to display the cameras on your computer while you are teleoperating by running the following code. This is useful to prepare your setup before recording your first dataset. -> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. You can adjust the viewer's behavior with these environment variables (refer to `rerun` documentation for more information): -> - [`RERUN_FLUSH_NUM_BYTES`](https://rerun.io/docs/reference/sdk/micro-batching) [default: 8000] -> - [`LEROBOT_RERUN_MEMORY_LIMIT`](https://rerun.io/docs/howto/visualization/limit-ram) [default: 5%] -> - [`LEROBOT_VIEWER_IP`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: None] -> - [`LEROBOT_VIEWER_PORT`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: 9876] +> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. ```bash python lerobot/scripts/control_robot.py \ diff --git a/examples/7_get_started_with_real_robot.md b/examples/7_get_started_with_real_robot.md index dd29e960..97d03a2c 100644 --- a/examples/7_get_started_with_real_robot.md +++ b/examples/7_get_started_with_real_robot.md @@ -56,11 +56,7 @@ Now you are ready to configure your motors for the first time, as detailed in th If you have already configured your motors the first time, you can streamline the process by directly running the teleoperate script (which is detailed further in the tutorial): -> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. You can adjust the viewer's behavior with these environment variables (refer to `rerun` documentation for more information): -> - [`RERUN_FLUSH_NUM_BYTES`](https://rerun.io/docs/reference/sdk/micro-batching) [default: 8000] -> - [`LEROBOT_RERUN_MEMORY_LIMIT`](https://rerun.io/docs/howto/visualization/limit-ram) [default: 5%] -> - [`LEROBOT_VIEWER_IP`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: None] -> - [`LEROBOT_VIEWER_PORT`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: 9876] +> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. ```bash python lerobot/scripts/control_robot.py \ @@ -834,6 +830,10 @@ It contains: - `dtRphone:33.84 (29.5hz)` which is the delta time of capturing an image from the phone camera in the thread running asynchronously. Troubleshooting: +- On Linux, if you encounter any issue during video encoding with `ffmpeg: unknown encoder libsvtav1`, you can: + - install with conda-forge by running `conda install -c conda-forge ffmpeg` (it should be compiled with `libsvtav1`), + - or, install [ffmpeg build dependencies](https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu#GettheDependencies) and [compile ffmpeg from source with libsvtav1](https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu#libsvtav1), + - and, make sure you use the corresponding ffmpeg binary to your install with `which ffmpeg`. - On Linux, if the left and right arrow keys and escape key don't have any effect during data recording, make sure you've set the `$DISPLAY` environment variable. See [pynput limitations](https://pynput.readthedocs.io/en/latest/limitations.html#linux). At the end of data recording, your dataset will be uploaded on your Hugging Face page (e.g. https://huggingface.co/datasets/cadene/koch_test) that you can obtain by running: diff --git a/examples/8_use_stretch.md b/examples/8_use_stretch.md index 56ca8006..a7a7dde1 100644 --- a/examples/8_use_stretch.md +++ b/examples/8_use_stretch.md @@ -103,12 +103,7 @@ Before trying teleoperation, you need activate the gamepad controller by pressin Now try out teleoperation (see above documentation to learn about the gamepad controls): -> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. You can adjust the viewer's behavior with these environment variables (refer to `rerun` documentation for more information): -> - [`RERUN_FLUSH_NUM_BYTES`](https://rerun.io/docs/reference/sdk/micro-batching) [default: 8000] -> - [`LEROBOT_RERUN_MEMORY_LIMIT`](https://rerun.io/docs/howto/visualization/limit-ram) [default: 5%] -> - [`LEROBOT_VIEWER_IP`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: None] -> - [`LEROBOT_VIEWER_PORT`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: 9876] - +> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. ```bash python lerobot/scripts/control_robot.py \ --robot.type=stretch \ diff --git a/examples/9_use_aloha.md b/examples/9_use_aloha.md index cbe8d797..77cff161 100644 --- a/examples/9_use_aloha.md +++ b/examples/9_use_aloha.md @@ -49,11 +49,7 @@ Teleoperation consists in manually operating the leader arms to move the followe By running the following code, you can start your first **SAFE** teleoperation: -> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. You can adjust the viewer's behavior with these environment variables (refer to `rerun` documentation for more information): -> - [`RERUN_FLUSH_NUM_BYTES`](https://rerun.io/docs/reference/sdk/micro-batching) [default: 8000] -> - [`LEROBOT_RERUN_MEMORY_LIMIT`](https://rerun.io/docs/howto/visualization/limit-ram) [default: 5%] -> - [`LEROBOT_VIEWER_IP`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: None] -> - [`LEROBOT_VIEWER_PORT`](https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) (only for remote robots) [default: 9876] +> **NOTE:** To visualize the data, enable `--control.display_data=true`. This streams the data using `rerun`. ```bash python lerobot/scripts/control_robot.py \ diff --git a/lerobot/common/robot_devices/control_configs.py b/lerobot/common/robot_devices/control_configs.py index f61907bd..cb558c71 100644 --- a/lerobot/common/robot_devices/control_configs.py +++ b/lerobot/common/robot_devices/control_configs.py @@ -118,6 +118,9 @@ class RemoteRobotConfig(ControlConfig): log_interval: int = 100 # Display all cameras on screen display_data: bool = False + # Rerun configuration for remote robot (https://ref.rerun.io/docs/python/0.22.1/common/initialization_functions/#rerun.connect_tcp) + viewer_ip: str | None = None + viewer_port: str | None = None @dataclass diff --git a/lerobot/common/robot_devices/control_utils.py b/lerobot/common/robot_devices/control_utils.py index c24e8778..4e42a989 100644 --- a/lerobot/common/robot_devices/control_utils.py +++ b/lerobot/common/robot_devices/control_utils.py @@ -307,10 +307,6 @@ def stop_recording(robot, listener, display_data): if not is_headless() and listener is not None: listener.stop() - # TODO(Steven): Find a way to close visualizer: https://github.com/rerun-io/rerun/pull/9400 - # if display_data: - # cv2.destroyAllWindows() - def sanity_check_dataset_name(repo_id, policy_cfg): _, dataset_name = repo_id.split("/") diff --git a/lerobot/scripts/control_robot.py b/lerobot/scripts/control_robot.py index 337d052b..3daea98d 100644 --- a/lerobot/scripts/control_robot.py +++ b/lerobot/scripts/control_robot.py @@ -381,26 +381,24 @@ def _init_rerun(control_config: ControlConfig, session_name: str = "lerobot_cont if (control_config.display_data and not is_headless()) or ( control_config.display_data and isinstance(control_config, RemoteRobotConfig) ): - # Configure Rerun flush batch size + # Configure Rerun flush batch size default to 8KB if not set batch_size = os.getenv("RERUN_FLUSH_NUM_BYTES", "8000") os.environ["RERUN_FLUSH_NUM_BYTES"] = batch_size - # Get memory limit and viewer connection parameters - memory_limit = os.getenv("LEROBOT_RERUN_MEMORY_LIMIT", "5%") - viewer_ip = os.getenv("LEROBOT_VIEWER_IP") - viewer_port = os.getenv("LEROBOT_VIEWER_PORT", "9876") - # Initialize Rerun based on configuration rr.init(session_name) if isinstance(control_config, RemoteRobotConfig): - if not viewer_ip: + viewer_ip = control_config.viewer_ip + viewer_port = control_config.viewer_port + if not viewer_ip or not viewer_port: raise ValueError( - "Viewer IP required for remote config. Set LEROBOT_VIEWER_IP " - "or disable control_config.display_data." + "Viewer IP & Port are required for remote config. Set via config file/CLI or disable control_config.display_data." ) logging.info(f"Connecting to viewer at {viewer_ip}:{viewer_port}") rr.connect_tcp(f"{viewer_ip}:{viewer_port}") else: + # Get memory limit for rerun viewer parameters + memory_limit = os.getenv("LEROBOT_RERUN_MEMORY_LIMIT", "10%") rr.spawn(memory_limit=memory_limit) diff --git a/pyproject.toml b/pyproject.toml index 4b917d6c..4b858634 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ dependencies = [ "omegaconf>=2.3.0", "opencv-python-headless>=4.9.0", "packaging>=24.2", - "av>=14.0.0", + "av>=12.0.5", "pymunk>=6.6.0", "pynput>=1.7.7", "pyzmq>=26.2.1",