Add upload_large_folder

lerobot/common/datasets/lerobot_dataset.py

@@ -516,6 +516,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
         push_videos: bool = True,
         private: bool = False,
         allow_patterns: list[str] | str | None = None,
+        upload_large_folder: bool = False,
         **card_kwargs,
     ) -> None:
         ignore_patterns = ["images/"]
@@ -538,14 +539,19 @@ class LeRobotDataset(torch.utils.data.Dataset):
                 exist_ok=True,
             )
 
-        hub_api.upload_folder(
-            repo_id=self.repo_id,
-            folder_path=self.root,
-            repo_type="dataset",
-            revision=branch,
-            allow_patterns=allow_patterns,
-            ignore_patterns=ignore_patterns,
-        )
+        upload_kwargs = {
+            "repo_id": self.repo_id,
+            "folder_path": self.root,
+            "repo_type": "dataset",
+            "revision": branch,
+            "allow_patterns": allow_patterns,
+            "ignore_patterns": ignore_patterns,
+        }
+        if upload_large_folder:
+            hub_api.upload_large_folder(**upload_kwargs)
+        else:
+            hub_api.upload_folder(**upload_kwargs)
+
         if not hub_api.file_exists(self.repo_id, REPOCARD_NAME, repo_type="dataset", revision=branch):
             card = create_lerobot_dataset_card(
                 tags=tags, dataset_info=self.meta.info, license=license, **card_kwargs