unidroid-cli 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unidroid_cli/__init__.py +1 -0
- unidroid_cli/__main__.py +3 -0
- unidroid_cli/_common.py +74 -0
- unidroid_cli/agent.py +392 -0
- unidroid_cli/bench.py +143 -0
- unidroid_cli/checks.py +42 -0
- unidroid_cli/cli.py +105 -0
- unidroid_cli/commands/__init__.py +1 -0
- unidroid_cli/commands/account.py +118 -0
- unidroid_cli/commands/auth.py +155 -0
- unidroid_cli/commands/bench.py +132 -0
- unidroid_cli/commands/challenge.py +97 -0
- unidroid_cli/commands/dataset.py +546 -0
- unidroid_cli/commands/submission.py +478 -0
- unidroid_cli/commands/validator.py +124 -0
- unidroid_cli/compare.py +64 -0
- unidroid_cli/config.py +85 -0
- unidroid_cli/dataset.py +414 -0
- unidroid_cli/dataset_sample.py +594 -0
- unidroid_cli/hfrepo.py +202 -0
- unidroid_cli/http.py +66 -0
- unidroid_cli/protocol.py +80 -0
- unidroid_cli/safe_extract.py +58 -0
- unidroid_cli/uploads.py +76 -0
- unidroid_cli/wallet.py +62 -0
- unidroid_cli-0.3.0.dist-info/METADATA +196 -0
- unidroid_cli-0.3.0.dist-info/RECORD +29 -0
- unidroid_cli-0.3.0.dist-info/WHEEL +4 -0
- unidroid_cli-0.3.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,594 @@
|
|
|
1
|
+
"""Dataset folder inspection, N-episode sample extraction for Hugging Face, and preview media extraction."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from .dataset import DatasetError, human_size, iter_dataset_files
|
|
14
|
+
|
|
15
|
+
# Matches episode_000000.parquet, videos/cam/episode_000000.mp4, ep_0.hdf5, demo-001/rgb.png, traj_12.npz, etc.
|
|
16
|
+
_EPISODE_RE = re.compile(r"(?:^|/)(?:episode|ep|demo|traj|trajectory)[_-]?(\d+)(?=\.|/|$)", re.IGNORECASE)
|
|
17
|
+
_SHARD_RE = re.compile(r"[-_](\d{4,6})(?:-of-\d+)?\.(?:parquet|tfrecord|tar|jsonl|bin|hdf5|h5|npz)$", re.IGNORECASE)
|
|
18
|
+
_EPISODE_CONTAINERS = {"episodes", "demos", "trajectories", "data"}
|
|
19
|
+
_META_EXTS = {".json", ".yaml", ".yml", ".txt", ".md", ".csv"}
|
|
20
|
+
_META_MAX_BYTES = 10 * 1024 * 1024
|
|
21
|
+
_VIDEO_EXTS = {".mp4", ".webm"}
|
|
22
|
+
_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp"}
|
|
23
|
+
|
|
24
|
+
# Standard SPDX-ish identifiers Hugging Face accepts in dataset card YAML frontmatter; others map to "other".
|
|
25
|
+
_HF_KNOWN_LICENSES = {
|
|
26
|
+
"apache-2.0", "bsd-2-clause", "bsd-3-clause", "cc-by-4.0", "cc-by-nc-4.0",
|
|
27
|
+
"cc-by-nc-sa-4.0", "cc-by-sa-4.0", "cc0-1.0", "gpl-3.0", "isc", "mit", "odc-by", "unlicense",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class FolderInspection:
|
|
33
|
+
files: list[Path]
|
|
34
|
+
episodes: dict[int, list[Path]]
|
|
35
|
+
metadata_files: list[Path]
|
|
36
|
+
info_json: dict[str, Any] = field(default_factory=dict)
|
|
37
|
+
auto_info: dict[str, Any] = field(default_factory=dict)
|
|
38
|
+
fps: float = 30.0
|
|
39
|
+
cameras: list[str] = field(default_factory=list)
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def total_episodes(self) -> int:
|
|
43
|
+
declared = self.info_json.get("total_episodes")
|
|
44
|
+
if isinstance(declared, int) and declared > 0 and not self.episodes:
|
|
45
|
+
return declared
|
|
46
|
+
return len(self.episodes)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class SampleSummary:
|
|
51
|
+
files: dict[str, Path] # posix path in HF repo -> local file path
|
|
52
|
+
sampled_episodes: int
|
|
53
|
+
total_episodes: int
|
|
54
|
+
total_bytes: int
|
|
55
|
+
sampled_frames: int | None = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _read_json(path: Path) -> dict[str, Any]:
|
|
59
|
+
try:
|
|
60
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
61
|
+
return data if isinstance(data, dict) else {}
|
|
62
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
63
|
+
return {}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _is_meta_file(rel_posix: str, path: Path) -> bool:
|
|
67
|
+
parts = rel_posix.split("/")
|
|
68
|
+
if parts[0] == "meta":
|
|
69
|
+
return path.stat().st_size <= _META_MAX_BYTES
|
|
70
|
+
if len(parts) == 1:
|
|
71
|
+
upper = parts[0].upper()
|
|
72
|
+
if upper.startswith(("README", "LICENSE", "NOTICE", "CITATION")) or path.suffix.lower() in _META_EXTS:
|
|
73
|
+
return path.stat().st_size <= _META_MAX_BYTES
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def inspect_folder(folder: Path, exclude: set[Path] | None = None) -> FolderInspection:
|
|
78
|
+
"""Scan a dataset directory for episodes, metadata, camera names and catalog facts."""
|
|
79
|
+
if not folder.is_dir():
|
|
80
|
+
raise DatasetError(f"{folder} is not a directory")
|
|
81
|
+
files = iter_dataset_files(folder, exclude=exclude)
|
|
82
|
+
if not files:
|
|
83
|
+
raise DatasetError(f"{folder} contains no dataset files")
|
|
84
|
+
|
|
85
|
+
info_path = folder / "meta" / "info.json"
|
|
86
|
+
info_json = _read_json(info_path) if info_path.is_file() else {}
|
|
87
|
+
|
|
88
|
+
episodes: dict[int, list[Path]] = {}
|
|
89
|
+
meta_files: list[Path] = []
|
|
90
|
+
other_files: list[Path] = []
|
|
91
|
+
|
|
92
|
+
# Pass 1: explicit episode/demo/trajectory numbering anywhere in relative path.
|
|
93
|
+
for path in files:
|
|
94
|
+
rel = path.relative_to(folder).as_posix()
|
|
95
|
+
if _is_meta_file(rel, path):
|
|
96
|
+
meta_files.append(path)
|
|
97
|
+
continue
|
|
98
|
+
match = _EPISODE_RE.search(rel)
|
|
99
|
+
if match:
|
|
100
|
+
episodes.setdefault(int(match.group(1)), []).append(path)
|
|
101
|
+
else:
|
|
102
|
+
other_files.append(path)
|
|
103
|
+
|
|
104
|
+
# Pass 2: shard numbering (e.g. shard-00000-of-00100.parquet) if no explicit episode names existed.
|
|
105
|
+
if not episodes and other_files:
|
|
106
|
+
still_other: list[Path] = []
|
|
107
|
+
for path in other_files:
|
|
108
|
+
rel = path.relative_to(folder).as_posix()
|
|
109
|
+
match = _SHARD_RE.search(rel)
|
|
110
|
+
if match:
|
|
111
|
+
episodes.setdefault(int(match.group(1)), []).append(path)
|
|
112
|
+
else:
|
|
113
|
+
still_other.append(path)
|
|
114
|
+
other_files = still_other
|
|
115
|
+
|
|
116
|
+
# Pass 3: subdirectories inside episodes/demos/data, or each data file as one unit.
|
|
117
|
+
if not episodes and other_files:
|
|
118
|
+
by_group: dict[str, list[Path]] = {}
|
|
119
|
+
for path in other_files:
|
|
120
|
+
parts = path.relative_to(folder).parts
|
|
121
|
+
if len(parts) >= 3 and parts[0].lower() in _EPISODE_CONTAINERS:
|
|
122
|
+
key = f"{parts[0]}/{parts[1]}"
|
|
123
|
+
elif len(parts) >= 2 and parts[0].lower() not in ("meta", "videos", "images"):
|
|
124
|
+
key = parts[0]
|
|
125
|
+
else:
|
|
126
|
+
key = path.relative_to(folder).as_posix()
|
|
127
|
+
by_group.setdefault(key, []).append(path)
|
|
128
|
+
for idx, (_, group_paths) in enumerate(sorted(by_group.items())):
|
|
129
|
+
episodes[idx] = group_paths
|
|
130
|
+
elif episodes and other_files:
|
|
131
|
+
# Small shared assets (calibration, urdf, stats) stay with metadata if reasonably small.
|
|
132
|
+
for path in other_files:
|
|
133
|
+
if path.stat().st_size <= _META_MAX_BYTES:
|
|
134
|
+
meta_files.append(path)
|
|
135
|
+
|
|
136
|
+
fps_raw = info_json.get("fps")
|
|
137
|
+
fps = float(fps_raw) if isinstance(fps_raw, (int, float)) and fps_raw > 0 else 30.0
|
|
138
|
+
|
|
139
|
+
cameras: list[str] = []
|
|
140
|
+
features = info_json.get("features")
|
|
141
|
+
if isinstance(features, dict):
|
|
142
|
+
for feat_name, spec in features.items():
|
|
143
|
+
if isinstance(spec, dict) and spec.get("dtype") in ("video", "image"):
|
|
144
|
+
cameras.append(feat_name)
|
|
145
|
+
if not cameras:
|
|
146
|
+
cam_set: set[str] = set()
|
|
147
|
+
for path in files:
|
|
148
|
+
parts = path.relative_to(folder).parts
|
|
149
|
+
if len(parts) >= 3 and parts[0] == "videos" and path.suffix.lower() in _VIDEO_EXTS:
|
|
150
|
+
# videos/chunk-000/<camera>/episode_000000.mp4 or videos/<camera>/episode_000000.mp4
|
|
151
|
+
cam_set.add(parts[2] if parts[1].startswith("chunk-") and len(parts) >= 4 else parts[1])
|
|
152
|
+
cameras = sorted(cam_set)
|
|
153
|
+
|
|
154
|
+
auto_info: dict[str, Any] = {}
|
|
155
|
+
total_eps = info_json.get("total_episodes") if isinstance(info_json.get("total_episodes"), int) else len(episodes)
|
|
156
|
+
if total_eps > 0:
|
|
157
|
+
auto_info["episodes"] = total_eps
|
|
158
|
+
if isinstance(info_json.get("total_frames"), int) and info_json["total_frames"] > 0:
|
|
159
|
+
auto_info["frames"] = info_json["total_frames"]
|
|
160
|
+
if isinstance(fps_raw, (int, float)) and fps_raw > 0:
|
|
161
|
+
auto_info["fps"] = int(fps_raw) if float(fps_raw).is_integer() else round(float(fps_raw), 2)
|
|
162
|
+
if cameras:
|
|
163
|
+
auto_info["cameras"] = len(cameras)
|
|
164
|
+
|
|
165
|
+
return FolderInspection(
|
|
166
|
+
files=files,
|
|
167
|
+
episodes=dict(sorted(episodes.items())),
|
|
168
|
+
metadata_files=meta_files,
|
|
169
|
+
info_json=info_json,
|
|
170
|
+
auto_info=auto_info,
|
|
171
|
+
fps=fps,
|
|
172
|
+
cameras=cameras,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _slice_parquet(src: Path, dst: Path, max_rows: int) -> int:
|
|
177
|
+
try:
|
|
178
|
+
import pyarrow.parquet as pq # type: ignore[import-not-found]
|
|
179
|
+
except ImportError:
|
|
180
|
+
raise DatasetError(
|
|
181
|
+
f"--sample-frames requires pyarrow to slice {src.name}; "
|
|
182
|
+
"install pyarrow (`pip install pyarrow`) or omit --sample-frames to upload full episodes"
|
|
183
|
+
) from None
|
|
184
|
+
table = pq.read_table(src)
|
|
185
|
+
sliced = table.slice(0, max_rows)
|
|
186
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
187
|
+
pq.write_table(sliced, dst)
|
|
188
|
+
return int(sliced.num_rows)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _slice_jsonl(src: Path, dst: Path, max_rows: int) -> int:
|
|
192
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
193
|
+
kept = 0
|
|
194
|
+
with src.open("r", encoding="utf-8") as inp, dst.open("w", encoding="utf-8", newline="\n") as out:
|
|
195
|
+
for line in inp:
|
|
196
|
+
if not line.strip():
|
|
197
|
+
continue
|
|
198
|
+
out.write(line if line.endswith("\n") else line + "\n")
|
|
199
|
+
kept += 1
|
|
200
|
+
if kept >= max_rows:
|
|
201
|
+
break
|
|
202
|
+
return kept
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _trim_video(src: Path, dst: Path, seconds: float) -> None:
|
|
206
|
+
if shutil.which("ffmpeg") is None:
|
|
207
|
+
raise DatasetError(
|
|
208
|
+
f"--sample-frames requires ffmpeg on PATH to trim {src.name}; "
|
|
209
|
+
"install ffmpeg or omit --sample-frames to upload full episodes"
|
|
210
|
+
)
|
|
211
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
212
|
+
duration = f"{max(0.1, seconds):.3f}"
|
|
213
|
+
cmd = [
|
|
214
|
+
"ffmpeg", "-y", "-nostdin", "-hide_banner", "-loglevel", "error",
|
|
215
|
+
"-i", str(src), "-t", duration,
|
|
216
|
+
"-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-pix_fmt", "yuv420p",
|
|
217
|
+
"-an", "-movflags", "+faststart", str(dst),
|
|
218
|
+
]
|
|
219
|
+
try:
|
|
220
|
+
subprocess.run(cmd, check=True, capture_output=True, timeout=180)
|
|
221
|
+
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
|
|
222
|
+
raise DatasetError(f"failed to trim preview video {src.name}: {exc}") from exc
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _filter_episode_jsonl(src: Path, dst: Path, selected: set[int], sample_frames: int | None) -> int | None:
|
|
226
|
+
"""Filter LeRobot meta/episodes.jsonl or episodes_stats.jsonl to selected episode_index values."""
|
|
227
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
228
|
+
total_frames = 0
|
|
229
|
+
saw_length = False
|
|
230
|
+
with src.open("r", encoding="utf-8") as inp, dst.open("w", encoding="utf-8", newline="\n") as out:
|
|
231
|
+
for raw in inp:
|
|
232
|
+
line = raw.strip()
|
|
233
|
+
if not line:
|
|
234
|
+
continue
|
|
235
|
+
try:
|
|
236
|
+
obj = json.loads(line)
|
|
237
|
+
except json.JSONDecodeError:
|
|
238
|
+
out.write(raw if raw.endswith("\n") else raw + "\n")
|
|
239
|
+
continue
|
|
240
|
+
idx = obj.get("episode_index")
|
|
241
|
+
if isinstance(idx, int) and idx not in selected:
|
|
242
|
+
continue
|
|
243
|
+
if sample_frames is not None and isinstance(obj.get("length"), int):
|
|
244
|
+
obj["length"] = min(obj["length"], sample_frames)
|
|
245
|
+
if isinstance(obj.get("length"), int):
|
|
246
|
+
total_frames += int(obj["length"])
|
|
247
|
+
saw_length = True
|
|
248
|
+
out.write(json.dumps(obj, ensure_ascii=False) + "\n")
|
|
249
|
+
return total_frames if saw_length else None
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _hf_license_slug(license_str: str) -> str:
|
|
253
|
+
slug = license_str.strip().lower()
|
|
254
|
+
return slug if slug in _HF_KNOWN_LICENSES else "other"
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def render_sample_readme(
|
|
258
|
+
*,
|
|
259
|
+
dataset_id: str,
|
|
260
|
+
name: str,
|
|
261
|
+
description: str,
|
|
262
|
+
license_name: str,
|
|
263
|
+
robot: str,
|
|
264
|
+
robot_type: str,
|
|
265
|
+
modality: str,
|
|
266
|
+
sampled_episodes: int,
|
|
267
|
+
total_episodes: int,
|
|
268
|
+
sample_frames: int | None,
|
|
269
|
+
full_size_bytes: int | None,
|
|
270
|
+
info: dict[str, Any],
|
|
271
|
+
) -> str:
|
|
272
|
+
"""Build a Hugging Face Dataset Card (YAML frontmatter + markdown overview) for the sample repository."""
|
|
273
|
+
hf_license = _hf_license_slug(license_name)
|
|
274
|
+
tags = ["unidroid", "robotics"]
|
|
275
|
+
for extra in (robot, robot_type, modality):
|
|
276
|
+
cleaned = re.sub(r"[^a-z0-9-]+", "-", extra.lower()).strip("-")
|
|
277
|
+
if cleaned and cleaned not in tags:
|
|
278
|
+
tags.append(cleaned)
|
|
279
|
+
frontmatter = [
|
|
280
|
+
"---",
|
|
281
|
+
f"license: {hf_license}",
|
|
282
|
+
*([] if hf_license != "other" else [f"license_name: {json.dumps(license_name)}"]),
|
|
283
|
+
"task_categories:",
|
|
284
|
+
" - robotics",
|
|
285
|
+
"tags:",
|
|
286
|
+
*[f" - {t}" for t in tags],
|
|
287
|
+
"---",
|
|
288
|
+
"",
|
|
289
|
+
]
|
|
290
|
+
slice_note = f" (first {sample_frames} frames per episode)" if sample_frames else ""
|
|
291
|
+
total_note = f" of **{total_episodes}** total episodes" if total_episodes > sampled_episodes else ""
|
|
292
|
+
size_note = f" ({human_size(full_size_bytes)} full archive)" if full_size_bytes else ""
|
|
293
|
+
|
|
294
|
+
lines = [
|
|
295
|
+
*frontmatter,
|
|
296
|
+
f"# {name} (Preview Sample)",
|
|
297
|
+
"",
|
|
298
|
+
f"> **Free preview sample** containing **{sampled_episodes}** episode{'s' if sampled_episodes != 1 else ''}"
|
|
299
|
+
f"{slice_note}{total_note}{size_note} from the **`{dataset_id}`** dataset on [Unidroid](https://unidroid.ai/datasets/{dataset_id}).",
|
|
300
|
+
"",
|
|
301
|
+
]
|
|
302
|
+
if description.strip():
|
|
303
|
+
lines += [description.strip(), ""]
|
|
304
|
+
|
|
305
|
+
facts: list[tuple[str, str]] = [
|
|
306
|
+
("Dataset ID", f"`{dataset_id}`"),
|
|
307
|
+
("Sample episodes", f"{sampled_episodes}" + (f" / {total_episodes}" if total_episodes else "")),
|
|
308
|
+
("License", license_name),
|
|
309
|
+
]
|
|
310
|
+
robot_desc = " · ".join(x for x in (robot, robot_type) if x)
|
|
311
|
+
if robot_desc:
|
|
312
|
+
facts.append(("Robot", robot_desc))
|
|
313
|
+
if modality:
|
|
314
|
+
facts.append(("Modality", modality))
|
|
315
|
+
for k, v in info.items():
|
|
316
|
+
if k in ("sample_episodes", "sample_frames"):
|
|
317
|
+
continue
|
|
318
|
+
facts.append((k.replace("_", " ").title(), json.dumps(v) if isinstance(v, (dict, list)) else str(v)))
|
|
319
|
+
|
|
320
|
+
lines += [
|
|
321
|
+
"## Dataset Summary",
|
|
322
|
+
"",
|
|
323
|
+
"| Field | Value |",
|
|
324
|
+
"| --- | --- |",
|
|
325
|
+
*[f"| {k} | {v} |" for k, v in facts],
|
|
326
|
+
"",
|
|
327
|
+
"## Full Dataset Access",
|
|
328
|
+
"",
|
|
329
|
+
"Use `udcli` (`pip install unidroid-cli`) to inspect, purchase, and download the complete dataset archive:",
|
|
330
|
+
"",
|
|
331
|
+
"```bash",
|
|
332
|
+
f"udcli dataset info {dataset_id}",
|
|
333
|
+
f"udcli dataset buy {dataset_id}",
|
|
334
|
+
f"udcli dataset download {dataset_id} --extract",
|
|
335
|
+
"```",
|
|
336
|
+
"",
|
|
337
|
+
]
|
|
338
|
+
return "\n".join(lines)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def build_sample(
|
|
342
|
+
folder: Path,
|
|
343
|
+
out_dir: Path,
|
|
344
|
+
*,
|
|
345
|
+
sample_episodes: int = 1,
|
|
346
|
+
sample_frames: int | None = None,
|
|
347
|
+
inspection: FolderInspection | None = None,
|
|
348
|
+
dataset_id: str,
|
|
349
|
+
name: str,
|
|
350
|
+
description: str = "",
|
|
351
|
+
license_name: str = "CC-BY-4.0",
|
|
352
|
+
robot: str = "",
|
|
353
|
+
robot_type: str = "",
|
|
354
|
+
modality: str = "",
|
|
355
|
+
info: dict[str, Any] | None = None,
|
|
356
|
+
full_size_bytes: int | None = None,
|
|
357
|
+
exclude: set[Path] | None = None,
|
|
358
|
+
) -> SampleSummary:
|
|
359
|
+
"""Materialize an N-episode preview sample into ``out_dir`` ready to commit to Hugging Face."""
|
|
360
|
+
if sample_episodes < 1:
|
|
361
|
+
raise DatasetError("--sample-episodes must be at least 1")
|
|
362
|
+
if sample_frames is not None and sample_frames < 1:
|
|
363
|
+
raise DatasetError("--sample-frames must be at least 1")
|
|
364
|
+
|
|
365
|
+
insp = inspection or inspect_folder(folder, exclude=exclude)
|
|
366
|
+
if not insp.episodes:
|
|
367
|
+
raise DatasetError(f"{folder}: could not find any episodes to sample")
|
|
368
|
+
|
|
369
|
+
episode_keys = list(insp.episodes.keys())
|
|
370
|
+
chosen_keys = episode_keys[:sample_episodes]
|
|
371
|
+
chosen_set = set(chosen_keys)
|
|
372
|
+
|
|
373
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
374
|
+
repo_files: dict[str, Path] = {}
|
|
375
|
+
sliced_rows_total = 0
|
|
376
|
+
counted_parquet = False
|
|
377
|
+
sampled_video_count = 0
|
|
378
|
+
|
|
379
|
+
# 1. Copy (or slice) every file belonging to the selected N episodes.
|
|
380
|
+
for ep_idx in chosen_keys:
|
|
381
|
+
for src in insp.episodes[ep_idx]:
|
|
382
|
+
rel = src.relative_to(folder).as_posix()
|
|
383
|
+
dst = out_dir / rel
|
|
384
|
+
suffix = src.suffix.lower()
|
|
385
|
+
if suffix in _VIDEO_EXTS:
|
|
386
|
+
sampled_video_count += 1
|
|
387
|
+
if sample_frames is not None and suffix == ".parquet":
|
|
388
|
+
sliced_rows_total += _slice_parquet(src, dst, sample_frames)
|
|
389
|
+
counted_parquet = True
|
|
390
|
+
repo_files[rel] = dst
|
|
391
|
+
elif sample_frames is not None and suffix == ".jsonl":
|
|
392
|
+
sliced_rows_total += _slice_jsonl(src, dst, sample_frames)
|
|
393
|
+
repo_files[rel] = dst
|
|
394
|
+
elif sample_frames is not None and suffix in _VIDEO_EXTS:
|
|
395
|
+
_trim_video(src, dst, sample_frames / insp.fps)
|
|
396
|
+
repo_files[rel] = dst
|
|
397
|
+
elif sample_frames is None:
|
|
398
|
+
# Zero-copy reference straight to the source file when unmodified!
|
|
399
|
+
repo_files[rel] = src
|
|
400
|
+
else:
|
|
401
|
+
repo_files[rel] = src
|
|
402
|
+
|
|
403
|
+
# 2. Copy/adapt metadata files (rewriting LeRobot episode counts so loaders work out of the box).
|
|
404
|
+
jsonl_frames: int | None = None
|
|
405
|
+
for src in insp.metadata_files:
|
|
406
|
+
rel = src.relative_to(folder).as_posix()
|
|
407
|
+
if rel.upper() == "README.MD":
|
|
408
|
+
continue # replaced by the generated sample Dataset Card below
|
|
409
|
+
if rel in ("meta/episodes.jsonl", "meta/episodes_stats.jsonl"):
|
|
410
|
+
dst = out_dir / rel
|
|
411
|
+
frames = _filter_episode_jsonl(src, dst, chosen_set, sample_frames)
|
|
412
|
+
if rel == "meta/episodes.jsonl" and frames is not None:
|
|
413
|
+
jsonl_frames = frames
|
|
414
|
+
repo_files[rel] = dst
|
|
415
|
+
elif rel == "meta/info.json" and insp.info_json:
|
|
416
|
+
dst = out_dir / rel
|
|
417
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
418
|
+
patched = dict(insp.info_json)
|
|
419
|
+
patched["total_episodes"] = len(chosen_keys)
|
|
420
|
+
known_frames = jsonl_frames if jsonl_frames is not None else (sliced_rows_total if counted_parquet else None)
|
|
421
|
+
if known_frames is not None:
|
|
422
|
+
patched["total_frames"] = known_frames
|
|
423
|
+
if "total_videos" in patched and sampled_video_count:
|
|
424
|
+
patched["total_videos"] = sampled_video_count
|
|
425
|
+
if isinstance(patched.get("splits"), dict):
|
|
426
|
+
patched["splits"] = {"train": f"0:{len(chosen_keys)}"}
|
|
427
|
+
dst.write_text(json.dumps(patched, indent=2) + "\n", encoding="utf-8")
|
|
428
|
+
repo_files[rel] = dst
|
|
429
|
+
else:
|
|
430
|
+
repo_files[rel] = src
|
|
431
|
+
|
|
432
|
+
sampled_frames = jsonl_frames if jsonl_frames is not None else (sliced_rows_total if counted_parquet else None)
|
|
433
|
+
if sampled_frames is not None and (out_dir / "meta" / "info.json").is_file() and insp.info_json:
|
|
434
|
+
# Ensure meta/info.json has jsonl_frames even if meta/info.json was visited before meta/episodes.jsonl.
|
|
435
|
+
patched = _read_json(out_dir / "meta" / "info.json")
|
|
436
|
+
if patched.get("total_frames") != sampled_frames:
|
|
437
|
+
patched["total_frames"] = sampled_frames
|
|
438
|
+
(out_dir / "meta" / "info.json").write_text(json.dumps(patched, indent=2) + "\n", encoding="utf-8")
|
|
439
|
+
|
|
440
|
+
# 3. Always include a clear Hugging Face Dataset Card README.md.
|
|
441
|
+
readme = out_dir / "README.md"
|
|
442
|
+
readme.write_text(
|
|
443
|
+
render_sample_readme(
|
|
444
|
+
dataset_id=dataset_id,
|
|
445
|
+
name=name,
|
|
446
|
+
description=description,
|
|
447
|
+
license_name=license_name,
|
|
448
|
+
robot=robot,
|
|
449
|
+
robot_type=robot_type,
|
|
450
|
+
modality=modality,
|
|
451
|
+
sampled_episodes=len(chosen_keys),
|
|
452
|
+
total_episodes=insp.total_episodes,
|
|
453
|
+
sample_frames=sample_frames,
|
|
454
|
+
full_size_bytes=full_size_bytes,
|
|
455
|
+
info=info or insp.auto_info,
|
|
456
|
+
),
|
|
457
|
+
encoding="utf-8",
|
|
458
|
+
)
|
|
459
|
+
repo_files["README.md"] = readme
|
|
460
|
+
|
|
461
|
+
total_bytes = sum(p.stat().st_size for p in repo_files.values())
|
|
462
|
+
return SampleSummary(
|
|
463
|
+
files=repo_files,
|
|
464
|
+
sampled_episodes=len(chosen_keys),
|
|
465
|
+
total_episodes=insp.total_episodes,
|
|
466
|
+
total_bytes=total_bytes,
|
|
467
|
+
sampled_frames=sampled_frames,
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _probe_duration(video: Path) -> float | None:
|
|
472
|
+
if shutil.which("ffprobe") is None:
|
|
473
|
+
return None
|
|
474
|
+
try:
|
|
475
|
+
proc = subprocess.run(
|
|
476
|
+
["ffprobe", "-v", "error", "-show_entries", "format=duration",
|
|
477
|
+
"-of", "default=noprint_wrappers=1:nokey=1", str(video)],
|
|
478
|
+
check=True, capture_output=True, text=True, timeout=15,
|
|
479
|
+
)
|
|
480
|
+
val = float(proc.stdout.strip())
|
|
481
|
+
return val if val > 0 else None
|
|
482
|
+
except (OSError, ValueError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
|
|
483
|
+
return None
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def auto_extract_media(
|
|
487
|
+
folder: Path,
|
|
488
|
+
out_dir: Path,
|
|
489
|
+
*,
|
|
490
|
+
inspection: FolderInspection | None = None,
|
|
491
|
+
clip_seconds: float = 15.0,
|
|
492
|
+
need_cover: bool = True,
|
|
493
|
+
need_video: bool = True,
|
|
494
|
+
exclude: set[Path] | None = None,
|
|
495
|
+
) -> tuple[Path | None, Path | None]:
|
|
496
|
+
"""Extract a thumbnail image and/or short preview MP4 clip from the first episode of ``folder``."""
|
|
497
|
+
insp = inspection or inspect_folder(folder, exclude=exclude)
|
|
498
|
+
ordered_paths: list[Path] = []
|
|
499
|
+
for ep_paths in insp.episodes.values():
|
|
500
|
+
ordered_paths.extend(ep_paths)
|
|
501
|
+
ordered_paths.extend(p for p in insp.files if p not in set(ordered_paths))
|
|
502
|
+
|
|
503
|
+
videos = [p for p in ordered_paths if p.suffix.lower() in _VIDEO_EXTS]
|
|
504
|
+
images = [p for p in ordered_paths if p.suffix.lower() in _IMAGE_EXTS]
|
|
505
|
+
|
|
506
|
+
# Videos belonging to the very first episode (for multi-camera side-by-side preview when >=2 views exist).
|
|
507
|
+
first_ep_videos: list[Path] = []
|
|
508
|
+
if insp.episodes:
|
|
509
|
+
first_ep = next(iter(insp.episodes.values()))
|
|
510
|
+
first_ep_videos = [p for p in first_ep if p.suffix.lower() in _VIDEO_EXTS]
|
|
511
|
+
if not first_ep_videos and videos:
|
|
512
|
+
first_ep_videos = [videos[0]]
|
|
513
|
+
|
|
514
|
+
has_ffmpeg = shutil.which("ffmpeg") is not None
|
|
515
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
516
|
+
cover_out: Path | None = None
|
|
517
|
+
video_out: Path | None = None
|
|
518
|
+
|
|
519
|
+
if need_cover:
|
|
520
|
+
if first_ep_videos and has_ffmpeg:
|
|
521
|
+
primary = first_ep_videos[0]
|
|
522
|
+
dur = _probe_duration(primary)
|
|
523
|
+
seek_s = min(2.0, dur * 0.3) if dur else 0.0
|
|
524
|
+
candidate = out_dir / "cover.png"
|
|
525
|
+
cmd = [
|
|
526
|
+
"ffmpeg", "-y", "-nostdin", "-hide_banner", "-loglevel", "error",
|
|
527
|
+
*(["-ss", f"{seek_s:.2f}"] if seek_s > 0 else []),
|
|
528
|
+
"-i", str(primary), "-frames:v", "1", str(candidate),
|
|
529
|
+
]
|
|
530
|
+
try:
|
|
531
|
+
subprocess.run(cmd, check=True, capture_output=True, timeout=60)
|
|
532
|
+
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
|
|
533
|
+
pass
|
|
534
|
+
if candidate.is_file() and candidate.stat().st_size > 0:
|
|
535
|
+
cover_out = candidate
|
|
536
|
+
if cover_out is None and images:
|
|
537
|
+
cover_out = images[0]
|
|
538
|
+
if cover_out is None:
|
|
539
|
+
raise DatasetError(
|
|
540
|
+
"--auto-media could not generate a thumbnail (no image files found, or ffmpeg is missing); "
|
|
541
|
+
"pass --thumbnail <file> or install ffmpeg"
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
if need_video:
|
|
545
|
+
if not first_ep_videos:
|
|
546
|
+
raise DatasetError(
|
|
547
|
+
"--auto-media could not find an MP4/WebM episode video in the folder; pass --video <file>"
|
|
548
|
+
)
|
|
549
|
+
primary = first_ep_videos[0]
|
|
550
|
+
if not has_ffmpeg:
|
|
551
|
+
# If the raw first-episode video is already under 50 MB, upload it directly; server transcodes it.
|
|
552
|
+
if primary.stat().st_size <= 50 * 1024 * 1024:
|
|
553
|
+
video_out = primary
|
|
554
|
+
else:
|
|
555
|
+
raise DatasetError(
|
|
556
|
+
"--auto-media needs ffmpeg on PATH to trim a short preview clip; "
|
|
557
|
+
"install ffmpeg or pass --video <file>"
|
|
558
|
+
)
|
|
559
|
+
else:
|
|
560
|
+
clip_dur = f"{max(1.0, min(clip_seconds, 30.0)):.2f}"
|
|
561
|
+
candidate = out_dir / "preview.mp4"
|
|
562
|
+
if len(first_ep_videos) >= 2:
|
|
563
|
+
# Try a 2-camera side-by-side 720p preview clip first (e.g. exterior + wrist view).
|
|
564
|
+
side_cmd = [
|
|
565
|
+
"ffmpeg", "-y", "-nostdin", "-hide_banner", "-loglevel", "error",
|
|
566
|
+
"-t", clip_dur, "-i", str(first_ep_videos[0]),
|
|
567
|
+
"-t", clip_dur, "-i", str(first_ep_videos[1]),
|
|
568
|
+
"-filter_complex",
|
|
569
|
+
"[0:v]scale=-2:480[v0];[1:v]scale=-2:480[v1];"
|
|
570
|
+
"[v0][v1]hstack=inputs=2,scale=w='min(1280,iw)':h='min(720,ih)':force_original_aspect_ratio=decrease,"
|
|
571
|
+
"scale=trunc(iw/2)*2:trunc(ih/2)*2[v]",
|
|
572
|
+
"-map", "[v]", "-c:v", "libx264", "-preset", "veryfast", "-crf", "26",
|
|
573
|
+
"-pix_fmt", "yuv420p", "-an", "-movflags", "+faststart", str(candidate),
|
|
574
|
+
]
|
|
575
|
+
try:
|
|
576
|
+
subprocess.run(side_cmd, check=True, capture_output=True, timeout=180)
|
|
577
|
+
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
|
|
578
|
+
candidate.unlink(missing_ok=True)
|
|
579
|
+
if not candidate.is_file() or candidate.stat().st_size == 0:
|
|
580
|
+
single_cmd = [
|
|
581
|
+
"ffmpeg", "-y", "-nostdin", "-hide_banner", "-loglevel", "error",
|
|
582
|
+
"-t", clip_dur, "-i", str(primary),
|
|
583
|
+
"-vf", "scale=w='min(1280,iw)':h='min(720,ih)':force_original_aspect_ratio=decrease,"
|
|
584
|
+
"scale=trunc(iw/2)*2:trunc(ih/2)*2",
|
|
585
|
+
"-c:v", "libx264", "-preset", "veryfast", "-crf", "26",
|
|
586
|
+
"-pix_fmt", "yuv420p", "-an", "-movflags", "+faststart", str(candidate),
|
|
587
|
+
]
|
|
588
|
+
try:
|
|
589
|
+
subprocess.run(single_cmd, check=True, capture_output=True, timeout=180)
|
|
590
|
+
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
|
|
591
|
+
raise DatasetError(f"--auto-media failed to build preview clip: {exc}") from exc
|
|
592
|
+
video_out = candidate
|
|
593
|
+
|
|
594
|
+
return cover_out, video_out
|