diff --git a/lerobot/common/datasets/lerobot_dataset.py b/lerobot/common/datasets/lerobot_dataset.py index 1d79edc1..a5ebc475 100644 --- a/lerobot/common/datasets/lerobot_dataset.py +++ b/lerobot/common/datasets/lerobot_dataset.py @@ -516,6 +516,7 @@ class LeRobotDataset(torch.utils.data.Dataset): push_videos: bool = True, private: bool = False, allow_patterns: list[str] | str | None = None, + upload_large_folder: bool = False, **card_kwargs, ) -> None: ignore_patterns = ["images/"] @@ -538,14 +539,19 @@ class LeRobotDataset(torch.utils.data.Dataset): exist_ok=True, ) - hub_api.upload_folder( - repo_id=self.repo_id, - folder_path=self.root, - repo_type="dataset", - revision=branch, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - ) + upload_kwargs = { + "repo_id": self.repo_id, + "folder_path": self.root, + "repo_type": "dataset", + "revision": branch, + "allow_patterns": allow_patterns, + "ignore_patterns": ignore_patterns, + } + if upload_large_folder: + hub_api.upload_large_folder(**upload_kwargs) + else: + hub_api.upload_folder(**upload_kwargs) + if not hub_api.file_exists(self.repo_id, REPOCARD_NAME, repo_type="dataset", revision=branch): card = create_lerobot_dataset_card( tags=tags, dataset_info=self.meta.info, license=license, **card_kwargs