unidroid-cli 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unidroid_cli/__init__.py +1 -0
- unidroid_cli/__main__.py +3 -0
- unidroid_cli/_common.py +74 -0
- unidroid_cli/agent.py +392 -0
- unidroid_cli/bench.py +143 -0
- unidroid_cli/checks.py +42 -0
- unidroid_cli/cli.py +105 -0
- unidroid_cli/commands/__init__.py +1 -0
- unidroid_cli/commands/account.py +118 -0
- unidroid_cli/commands/auth.py +155 -0
- unidroid_cli/commands/bench.py +132 -0
- unidroid_cli/commands/challenge.py +97 -0
- unidroid_cli/commands/dataset.py +546 -0
- unidroid_cli/commands/submission.py +478 -0
- unidroid_cli/commands/validator.py +124 -0
- unidroid_cli/compare.py +64 -0
- unidroid_cli/config.py +85 -0
- unidroid_cli/dataset.py +414 -0
- unidroid_cli/dataset_sample.py +594 -0
- unidroid_cli/hfrepo.py +202 -0
- unidroid_cli/http.py +66 -0
- unidroid_cli/protocol.py +80 -0
- unidroid_cli/safe_extract.py +58 -0
- unidroid_cli/uploads.py +76 -0
- unidroid_cli/wallet.py +62 -0
- unidroid_cli-0.3.0.dist-info/METADATA +196 -0
- unidroid_cli-0.3.0.dist-info/RECORD +29 -0
- unidroid_cli-0.3.0.dist-info/WHEEL +4 -0
- unidroid_cli-0.3.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
"""udcli dataset: browse, buy and download datasets; upload, attach media/samples, and review them (admins)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
import tempfile
|
|
9
|
+
from collections.abc import Callable, Iterator
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Optional
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
from rich.markup import escape
|
|
16
|
+
from rich.progress import (
|
|
17
|
+
BarColumn,
|
|
18
|
+
DownloadColumn,
|
|
19
|
+
Progress,
|
|
20
|
+
TaskProgressColumn,
|
|
21
|
+
TextColumn,
|
|
22
|
+
TimeRemainingColumn,
|
|
23
|
+
TransferSpeedColumn,
|
|
24
|
+
)
|
|
25
|
+
from rich.table import Table
|
|
26
|
+
|
|
27
|
+
from .. import dataset, dataset_sample, hfrepo
|
|
28
|
+
from .._common import YES, api, confirm, console, emit_json, err, group, local_time, tao
|
|
29
|
+
from ..dataset import DatasetError, human_size, tao_to_rao
|
|
30
|
+
from ..http import ApiError
|
|
31
|
+
|
|
32
|
+
app = group("Datasets for sale. Anyone can browse; buying and downloading need a login; admins upload.")
|
|
33
|
+
|
|
34
|
+
STATE_HELP = {
|
|
35
|
+
"uploading": "upload not finished",
|
|
36
|
+
"verifying": "the server is checking the archive",
|
|
37
|
+
"pending_approval": "waiting for a super admin",
|
|
38
|
+
"approved": "on sale",
|
|
39
|
+
"rejected": "rejected",
|
|
40
|
+
"invalid": "failed verification",
|
|
41
|
+
"archived": "off sale",
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def price_label(rao: int) -> str:
|
|
46
|
+
return "free" if rao == 0 else tao(rao)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _access_line(d: dict) -> str:
|
|
50
|
+
a = d["access"]
|
|
51
|
+
if a["can_download"]:
|
|
52
|
+
return f"[green]you can download it ({a['reason']})[/green]"
|
|
53
|
+
if a["can_purchase"]:
|
|
54
|
+
return f"[yellow]purchase required: udcli dataset buy {d['id']}[/yellow]"
|
|
55
|
+
if a["reason"] == "login required":
|
|
56
|
+
return "[yellow]log in to buy and download: udcli auth login[/yellow]"
|
|
57
|
+
return f"[red]{a['reason']}[/red]"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@contextmanager
|
|
61
|
+
def _bytes_bar(label: str) -> Iterator[Callable[[int, int], None]]:
|
|
62
|
+
with Progress(TextColumn("{task.description}"), BarColumn(), TaskProgressColumn(), DownloadColumn(),
|
|
63
|
+
TransferSpeedColumn(), TimeRemainingColumn(), console=console, transient=True) as bar:
|
|
64
|
+
task = bar.add_task(label, total=None)
|
|
65
|
+
|
|
66
|
+
def update(done: int, total: int) -> None:
|
|
67
|
+
bar.update(task, completed=done, total=total)
|
|
68
|
+
|
|
69
|
+
yield update
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@contextmanager
|
|
73
|
+
def _count_bar(label: str) -> Iterator[Callable[[int, int], None]]:
|
|
74
|
+
with Progress(TextColumn("{task.description}"), BarColumn(), TaskProgressColumn(),
|
|
75
|
+
TextColumn("{task.completed}/{task.total}"), console=console, transient=True) as bar:
|
|
76
|
+
task = bar.add_task(label, total=None)
|
|
77
|
+
|
|
78
|
+
def update(done: int, total: int) -> None:
|
|
79
|
+
bar.update(task, completed=done, total=total)
|
|
80
|
+
|
|
81
|
+
yield update
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# ---------------------------------------------------------------- browse
|
|
85
|
+
|
|
86
|
+
@app.command("list")
|
|
87
|
+
def dataset_list(q: Optional[str] = typer.Option(None, "--search", "-s", help="text in the id, name or description"),
|
|
88
|
+
robot_type: Optional[str] = typer.Option(None, help="robot type"),
|
|
89
|
+
modality: Optional[str] = None,
|
|
90
|
+
free: Optional[bool] = typer.Option(None, "--free/--paid", help="only free or only paid"),
|
|
91
|
+
sort: str = typer.Option("new", help="new|popular|price_asc|price_desc|name"),
|
|
92
|
+
limit: int = typer.Option(24, min=1, max=100), offset: int = typer.Option(0, min=0),
|
|
93
|
+
as_json: bool = typer.Option(False, "--json")) -> None:
|
|
94
|
+
"""List datasets on sale. No login needed."""
|
|
95
|
+
params = {k: v for k, v in {"q": q, "robot_type": robot_type, "modality": modality, "free": free,
|
|
96
|
+
"sort": sort, "limit": limit, "offset": offset}.items() if v is not None}
|
|
97
|
+
page = api(require_key=False).get("/v1/datasets", params=params)
|
|
98
|
+
if as_json:
|
|
99
|
+
return emit_json(page)
|
|
100
|
+
table = Table("id", "name", "robot", "modality", "size", "price", "sold", "you")
|
|
101
|
+
for d in page["items"]:
|
|
102
|
+
you = "owned" if d["purchased"] else "free" if d["price_rao"] == 0 else "-"
|
|
103
|
+
table.add_row(d["id"], escape(d["name"]), escape(d["robot_type"] or d["robot"] or "-"),
|
|
104
|
+
escape(d["modality"] or "-"), human_size(d["size_bytes"]), price_label(d["price_rao"]),
|
|
105
|
+
str(d["purchase_count"]), you)
|
|
106
|
+
console.print(table)
|
|
107
|
+
shown = offset + len(page["items"])
|
|
108
|
+
console.print(f"{shown} of {page['total']}" + (f" (use --offset {shown} for more)" if shown < page["total"] else ""))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@app.command("info")
|
|
112
|
+
def dataset_info(dataset_id: str, as_json: bool = typer.Option(False, "--json")) -> None:
|
|
113
|
+
"""Price, size, license, preview media/link, and whether you can download it. No login needed."""
|
|
114
|
+
d = api(require_key=False).get(f"/v1/datasets/{dataset_id}")
|
|
115
|
+
if as_json:
|
|
116
|
+
return emit_json(d)
|
|
117
|
+
console.print(f"[bold]{escape(d['id'])}[/bold] {escape(d['name'])} ({d['status']})")
|
|
118
|
+
if d["description"]:
|
|
119
|
+
console.print(d["description"], markup=False)
|
|
120
|
+
console.print(f" price: {price_label(d['price_rao'])} sold: {d['purchase_count']}", markup=False)
|
|
121
|
+
console.print(f" size: {human_size(d['size_bytes'])} (.{d['format']}) license: {d['license'] or '-'}",
|
|
122
|
+
markup=False)
|
|
123
|
+
robot = " ".join(x for x in (d["robot"], d["robot_type"]) if x)
|
|
124
|
+
if robot or d["modality"]:
|
|
125
|
+
console.print(f" robot: {robot or '-'} modality: {d['modality'] or '-'}", markup=False)
|
|
126
|
+
for key, value in (d.get("info") or {}).items():
|
|
127
|
+
console.print(f" {key}: {value}", markup=False)
|
|
128
|
+
if d.get("preview_url"):
|
|
129
|
+
console.print(f" free preview: {d['preview_url']}", markup=False)
|
|
130
|
+
if d.get("cover_url"):
|
|
131
|
+
console.print(f" thumbnail: {d['cover_url']}", markup=False)
|
|
132
|
+
if d.get("video_url"):
|
|
133
|
+
console.print(f" preview clip: {d['video_url']}", markup=False)
|
|
134
|
+
console.print(f" sha256 {d['sha256']}", markup=False)
|
|
135
|
+
if d.get("review_note"):
|
|
136
|
+
console.print(f" note: {d['review_note']}", markup=False)
|
|
137
|
+
console.print(f" {_access_line(d)}")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@app.command("mine")
|
|
141
|
+
def dataset_mine(as_json: bool = typer.Option(False, "--json")) -> None:
|
|
142
|
+
"""Datasets you bought."""
|
|
143
|
+
rows = api().get("/v1/account/datasets")
|
|
144
|
+
if as_json:
|
|
145
|
+
return emit_json(rows)
|
|
146
|
+
table = Table("id", "name", "size", "paid", "bought")
|
|
147
|
+
for d in rows:
|
|
148
|
+
table.add_row(d["id"], escape(d["name"]), human_size(d["size_bytes"]), price_label(d["paid_rao"]),
|
|
149
|
+
local_time(d["purchased_at"]))
|
|
150
|
+
console.print(table)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# ---------------------------------------------------------------- buy and download
|
|
154
|
+
|
|
155
|
+
@app.command("buy")
|
|
156
|
+
def dataset_buy(dataset_id: str, yes: bool = typer.Option(False, "--yes", "-y", help="do not ask to confirm")) -> None:
|
|
157
|
+
"""Buy a dataset with your account balance (login required)."""
|
|
158
|
+
client = api()
|
|
159
|
+
d = client.get(f"/v1/datasets/{dataset_id}")
|
|
160
|
+
access = d["access"]
|
|
161
|
+
if access["purchased"]:
|
|
162
|
+
return console.print(f"You already own [bold]{dataset_id}[/bold]: udcli dataset download {dataset_id}")
|
|
163
|
+
if not access["can_purchase"]:
|
|
164
|
+
raise DatasetError(f"cannot buy {dataset_id}: {access['reason']}")
|
|
165
|
+
price = d["price_rao"]
|
|
166
|
+
balance = client.get("/v1/account/balance")["balance_rao"]
|
|
167
|
+
console.print(f"{escape(d['name'])} ({dataset_id}): [bold]{price_label(price)}[/bold] your balance: {tao(balance)}")
|
|
168
|
+
if price > balance:
|
|
169
|
+
raise DatasetError(f"not enough balance: {tao(price - balance)} short. See `udcli account balance`")
|
|
170
|
+
if not yes:
|
|
171
|
+
typer.confirm("Buy this dataset?", abort=True)
|
|
172
|
+
done = client.post(f"/v1/datasets/{dataset_id}/purchase", json={"expected_price_rao": price})
|
|
173
|
+
console.print(f"[green]Bought[/green] {dataset_id}. Balance: {tao(done['balance_rao'])}")
|
|
174
|
+
console.print(f"Download it: udcli dataset download {dataset_id}")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@app.command("download")
|
|
178
|
+
def dataset_download(dataset_id: str,
|
|
179
|
+
out: Path = typer.Option(Path("."), "--out", "-o", help="folder to save into"),
|
|
180
|
+
extract: bool = typer.Option(False, "--extract", help="also unpack into <out>/<id>/"),
|
|
181
|
+
force: bool = typer.Option(False, "--force", help="download again even if intact")) -> None:
|
|
182
|
+
"""Download a dataset you own (login required). Resumable and sha256-verified."""
|
|
183
|
+
client = api()
|
|
184
|
+
with _bytes_bar(f"downloading {dataset_id}") as progress:
|
|
185
|
+
path = dataset.download(client, dataset_id, out, force=force, progress=progress)
|
|
186
|
+
console.print(f"[green]{dataset_id}[/green] -> {path} (sha256 verified)")
|
|
187
|
+
if extract:
|
|
188
|
+
folder = dataset.extract(path, out / dataset_id)
|
|
189
|
+
console.print(f"unpacked to {folder}")
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
# ---------------------------------------------------------------- upload and media (admin)
|
|
193
|
+
|
|
194
|
+
def _slug(text: str) -> str:
|
|
195
|
+
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:64]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _info_items(items: list[str]) -> dict:
|
|
199
|
+
out: dict = {}
|
|
200
|
+
for item in items:
|
|
201
|
+
key, sep, raw = item.partition("=")
|
|
202
|
+
if not sep or not key.strip():
|
|
203
|
+
raise DatasetError(f"--info expects key=value, got {item!r}")
|
|
204
|
+
try:
|
|
205
|
+
out[key.strip()] = json.loads(raw)
|
|
206
|
+
except ValueError:
|
|
207
|
+
out[key.strip()] = raw
|
|
208
|
+
return out
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _resolve_hf_write_token(explicit: str | None) -> str:
|
|
212
|
+
token = hfrepo.local_token(explicit)
|
|
213
|
+
if not token and sys.stdin.isatty():
|
|
214
|
+
token = typer.prompt("Hugging Face write token", hide_input=True).strip()
|
|
215
|
+
if not token:
|
|
216
|
+
raise DatasetError("pass --hf-token, set HF_TOKEN, or run `hf auth login`")
|
|
217
|
+
hfrepo.verify_write_token(token)
|
|
218
|
+
return token
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
@app.command("upload")
|
|
222
|
+
def dataset_upload(
|
|
223
|
+
source: Path = typer.Argument(..., help="a dataset folder, or a .zip / .tar / .tar.gz archive"),
|
|
224
|
+
license: str = typer.Option(..., "--license", help="terms of use, e.g. CC-BY-4.0"),
|
|
225
|
+
price: str = typer.Option(..., "--price", help="price in TAO (0 for free)"),
|
|
226
|
+
dataset_id: Optional[str] = typer.Option(None, "--id", help="default: the folder or file name"),
|
|
227
|
+
name: Optional[str] = typer.Option(None, "--name", help="display name; default: the id"),
|
|
228
|
+
description: str = typer.Option("", "--description"),
|
|
229
|
+
robot: str = typer.Option("", "--robot"),
|
|
230
|
+
robot_type: str = typer.Option("", "--robot-type"),
|
|
231
|
+
modality: str = typer.Option("", "--modality"),
|
|
232
|
+
preview_url: Optional[str] = typer.Option(
|
|
233
|
+
None, "--preview-url", help="existing Hugging Face dataset page (set automatically when --hf-repo is used)"),
|
|
234
|
+
hf_repo: Optional[str] = typer.Option(
|
|
235
|
+
None, "--hf-repo", help="Hugging Face dataset repo (<org>/<name>) to push a free sample preview"),
|
|
236
|
+
hf_token: Optional[str] = typer.Option(
|
|
237
|
+
None, "--hf-token", envvar="HF_TOKEN",
|
|
238
|
+
help="Hugging Face write token (defaults to HF_TOKEN or `hf auth login`; never sent to Unidroid)"),
|
|
239
|
+
sample_episodes: int = typer.Option(
|
|
240
|
+
1, "--sample-episodes", "-n", min=1,
|
|
241
|
+
help="number of episodes (N) to upload to --hf-repo as a sample preview"),
|
|
242
|
+
sample_frames: Optional[int] = typer.Option(
|
|
243
|
+
None, "--sample-frames", min=1,
|
|
244
|
+
help="optional max frames per sampled episode (uploads a portion of each sampled episode)"),
|
|
245
|
+
thumbnail: Optional[Path] = typer.Option(
|
|
246
|
+
None, "--thumbnail", help="cover image (PNG, JPEG or WebP, up to 5 MB)"),
|
|
247
|
+
video: Optional[Path] = typer.Option(
|
|
248
|
+
None, "--video", help="short preview clip (MP4 or WebM, up to 50 MB)"),
|
|
249
|
+
auto_media: bool = typer.Option(
|
|
250
|
+
False, "--auto-media", help="extract missing thumbnail and/or preview clip automatically from the folder"),
|
|
251
|
+
clip_seconds: float = typer.Option(
|
|
252
|
+
15.0, "--clip-seconds", min=1.0, max=30.0, help="clip length in seconds for --auto-media"),
|
|
253
|
+
work_dir: Optional[Path] = typer.Option(
|
|
254
|
+
None, "--work-dir", help="folder where the staged .zip is built (default: parent of source folder)"),
|
|
255
|
+
info: list[str] = typer.Option([], "--info", help="key=value facts such as episodes=120 (repeatable)"),
|
|
256
|
+
jobs: int = typer.Option(4, "--jobs", min=1, max=16, help="parallel part uploads (memory: jobs x 128 MB)"),
|
|
257
|
+
confirm_rights: bool = typer.Option(False, "--confirm-rights", help="skip the rights question"),
|
|
258
|
+
yes: bool = typer.Option(False, "--yes", "-y", help="also skip the question before the sample goes public"),
|
|
259
|
+
) -> None:
|
|
260
|
+
"""Upload a dataset, optionally pushing an N-episode Hugging Face sample and attaching a thumbnail & clip.
|
|
261
|
+
|
|
262
|
+
Run the same command again to resume an interrupted upload. A super admin's dataset goes on sale once it
|
|
263
|
+
verifies; an admin's waits for a super admin to approve it.
|
|
264
|
+
"""
|
|
265
|
+
if not source.exists():
|
|
266
|
+
raise DatasetError(f"{source} not found")
|
|
267
|
+
if (hf_repo or auto_media) and not source.is_dir():
|
|
268
|
+
raise DatasetError("--hf-repo and --auto-media require a dataset folder rather than a pre-built archive")
|
|
269
|
+
price_rao = tao_to_rao(price)
|
|
270
|
+
explicit_info = _info_items(info)
|
|
271
|
+
|
|
272
|
+
normalized_hf: str | None = None
|
|
273
|
+
resolved_hf_token: str | None = None
|
|
274
|
+
if hf_repo:
|
|
275
|
+
normalized_hf = hfrepo.normalize_dataset_repo(hf_repo)
|
|
276
|
+
expected_url = hfrepo.dataset_url(normalized_hf)
|
|
277
|
+
if preview_url and preview_url.strip() and preview_url.strip().rstrip("/") != expected_url:
|
|
278
|
+
raise DatasetError("pass either --hf-repo (to upload a sample) or --preview-url, not both")
|
|
279
|
+
preview_url = expected_url
|
|
280
|
+
|
|
281
|
+
if thumbnail is not None:
|
|
282
|
+
dataset.detect_media_type("cover", thumbnail)
|
|
283
|
+
if video is not None:
|
|
284
|
+
dataset.detect_media_type("video", video)
|
|
285
|
+
|
|
286
|
+
client = api()
|
|
287
|
+
dataset_id = dataset_id or _slug(source.resolve().name.split(".")[0])
|
|
288
|
+
display_name = name or dataset_id
|
|
289
|
+
if client.get("/v1/auth/me")["role"] not in ("admin", "super_admin"): # before hashing gigabytes
|
|
290
|
+
raise DatasetError("only admins and super admins can upload datasets")
|
|
291
|
+
if normalized_hf:
|
|
292
|
+
resolved_hf_token = _resolve_hf_write_token(hf_token)
|
|
293
|
+
|
|
294
|
+
if not confirm_rights:
|
|
295
|
+
typer.confirm("Do you confirm you have the right to sell this data and that it holds no personal "
|
|
296
|
+
"information its subjects have not agreed to share?", abort=True)
|
|
297
|
+
|
|
298
|
+
# Exclude explicit thumbnail/video files if the admin placed them inside the dataset folder.
|
|
299
|
+
excluded_media = {p.resolve() for p in (thumbnail, video) if p is not None}
|
|
300
|
+
insp = dataset_sample.inspect_folder(source, exclude=excluded_media) if source.is_dir() else None
|
|
301
|
+
merged_info = {**(insp.auto_info if insp else {}), **explicit_info}
|
|
302
|
+
inferred_robot_type = robot_type or (
|
|
303
|
+
str(insp.info_json["robot_type"])
|
|
304
|
+
if insp and isinstance(insp.info_json.get("robot_type"), str) and not robot
|
|
305
|
+
else ""
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
with tempfile.TemporaryDirectory(prefix="udcli-preview-") as preview_tmp:
|
|
309
|
+
tmp_root = Path(preview_tmp)
|
|
310
|
+
cover_path, video_path = thumbnail, video
|
|
311
|
+
|
|
312
|
+
if auto_media and insp is not None and (cover_path is None or video_path is None):
|
|
313
|
+
gen_cover, gen_video = dataset_sample.auto_extract_media(
|
|
314
|
+
source, tmp_root / "media", inspection=insp, clip_seconds=clip_seconds,
|
|
315
|
+
need_cover=cover_path is None, need_video=video_path is None, exclude=excluded_media,
|
|
316
|
+
)
|
|
317
|
+
cover_path = cover_path or gen_cover
|
|
318
|
+
video_path = video_path or gen_video
|
|
319
|
+
if cover_path is not None:
|
|
320
|
+
dataset.detect_media_type("cover", cover_path)
|
|
321
|
+
if video_path is not None:
|
|
322
|
+
dataset.detect_media_type("video", video_path)
|
|
323
|
+
|
|
324
|
+
if normalized_hf and resolved_hf_token and insp is not None:
|
|
325
|
+
raw_folder_bytes = sum(p.stat().st_size for p in insp.files)
|
|
326
|
+
sample = dataset_sample.build_sample(
|
|
327
|
+
source, tmp_root / "sample",
|
|
328
|
+
sample_episodes=sample_episodes,
|
|
329
|
+
sample_frames=sample_frames,
|
|
330
|
+
inspection=insp,
|
|
331
|
+
dataset_id=dataset_id,
|
|
332
|
+
name=display_name,
|
|
333
|
+
description=description,
|
|
334
|
+
license_name=license,
|
|
335
|
+
robot=robot,
|
|
336
|
+
robot_type=inferred_robot_type,
|
|
337
|
+
modality=modality,
|
|
338
|
+
info=merged_info,
|
|
339
|
+
full_size_bytes=raw_folder_bytes,
|
|
340
|
+
exclude=excluded_media,
|
|
341
|
+
)
|
|
342
|
+
merged_info.setdefault("sample_episodes", sample.sampled_episodes)
|
|
343
|
+
if sample_frames is not None:
|
|
344
|
+
merged_info.setdefault("sample_frames", sample_frames)
|
|
345
|
+
if price_rao > 0 and sample.sampled_episodes >= sample.total_episodes and sample_frames is None:
|
|
346
|
+
raise DatasetError(
|
|
347
|
+
f"--sample-episodes {sample.sampled_episodes} covers every episode of this paid dataset; the "
|
|
348
|
+
"free public sample would hold all of it. Lower --sample-episodes or pass --sample-frames.")
|
|
349
|
+
console.print(
|
|
350
|
+
f"Sample for Hugging Face {normalized_hf} (public): {sample.sampled_episodes} of "
|
|
351
|
+
f"{sample.total_episodes} episodes, {len(sample.files)} files, {human_size(sample.total_bytes)}")
|
|
352
|
+
for rel in sorted(sample.files)[:15]:
|
|
353
|
+
console.print(f" {rel}", markup=False, highlight=False)
|
|
354
|
+
if len(sample.files) > 15:
|
|
355
|
+
console.print(f" ... and {len(sample.files) - 15} more")
|
|
356
|
+
if not yes and not confirm_rights:
|
|
357
|
+
typer.confirm("Publish this sample on Hugging Face?", abort=True)
|
|
358
|
+
console.print(f"Uploading the sample to Hugging Face {normalized_hf}...")
|
|
359
|
+
sha, hf_url = hfrepo.push_dataset_sample(
|
|
360
|
+
normalized_hf, sample.files, resolved_hf_token, private=False,
|
|
361
|
+
message=f"Upload {sample.sampled_episodes}-episode preview sample for {dataset_id} with udcli",
|
|
362
|
+
)
|
|
363
|
+
preview_url = hf_url
|
|
364
|
+
console.print(f"[green]Sample preview[/green] -> {hf_url} (commit {sha[:8]})", highlight=False)
|
|
365
|
+
|
|
366
|
+
meta = {
|
|
367
|
+
"id": dataset_id,
|
|
368
|
+
"name": display_name,
|
|
369
|
+
"description": description,
|
|
370
|
+
"license": license,
|
|
371
|
+
"robot": robot,
|
|
372
|
+
"robot_type": inferred_robot_type,
|
|
373
|
+
"modality": modality,
|
|
374
|
+
"preview_url": preview_url,
|
|
375
|
+
"info": merged_info,
|
|
376
|
+
"price_rao": price_rao,
|
|
377
|
+
"rights_confirmed": True,
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
with _count_bar("packaging dataset folder") as pack_bar:
|
|
381
|
+
with dataset.prepared_archive(
|
|
382
|
+
source, dataset_id, work_dir=work_dir, exclude=excluded_media, progress=pack_bar,
|
|
383
|
+
) as prep:
|
|
384
|
+
if prep.reused_staged:
|
|
385
|
+
console.print(f"Reusing staged archive {prep.path.name}")
|
|
386
|
+
console.print(
|
|
387
|
+
f"Uploading {dataset_id}: {human_size(prep.path.stat().st_size)}, "
|
|
388
|
+
f"price {price_label(meta['price_rao'])}"
|
|
389
|
+
)
|
|
390
|
+
with _count_bar("uploading archive parts") as upload_bar:
|
|
391
|
+
result = dataset.upload(
|
|
392
|
+
client, prep.path, meta, sha256=prep.sha256, jobs=jobs, progress=upload_bar,
|
|
393
|
+
on_resume=lambda have, total: console.print(f"Resuming: {have}/{total} parts already stored"),
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
for kind, media_file, flag in (("cover", cover_path, "--thumbnail"), ("video", video_path, "--video")):
|
|
397
|
+
if media_file is None:
|
|
398
|
+
continue
|
|
399
|
+
label = "thumbnail" if kind == "cover" else "preview clip"
|
|
400
|
+
try:
|
|
401
|
+
result = dataset.upload_media(client, dataset_id, kind, media_file)
|
|
402
|
+
console.print(f"[green]Attached {label}[/green] ({human_size(media_file.stat().st_size)})")
|
|
403
|
+
except (ApiError, DatasetError) as exc:
|
|
404
|
+
_print_outcome(result)
|
|
405
|
+
err.print(
|
|
406
|
+
f"[red]Warning:[/red] the archive is uploaded, but {label} failed ({exc}). "
|
|
407
|
+
f"Retry just the media with: udcli dataset media {dataset_id} {flag} {media_file}"
|
|
408
|
+
)
|
|
409
|
+
raise SystemExit(1) from None
|
|
410
|
+
|
|
411
|
+
_print_outcome(result)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
@app.command("media")
|
|
415
|
+
def dataset_media(
|
|
416
|
+
dataset_id: str,
|
|
417
|
+
thumbnail: Optional[Path] = typer.Option(None, "--thumbnail", help="upload or replace cover (PNG, JPEG or WebP)"),
|
|
418
|
+
video: Optional[Path] = typer.Option(None, "--video", help="upload or replace preview clip (MP4 or WebM)"),
|
|
419
|
+
remove_thumbnail: bool = typer.Option(False, "--remove-thumbnail", help="delete the cover image"),
|
|
420
|
+
remove_video: bool = typer.Option(False, "--remove-video", help="delete the preview clip"),
|
|
421
|
+
) -> None:
|
|
422
|
+
"""Upload, replace or remove a dataset's thumbnail image and short preview video clip (admin)."""
|
|
423
|
+
if thumbnail and remove_thumbnail:
|
|
424
|
+
raise DatasetError("pass either --thumbnail or --remove-thumbnail, not both")
|
|
425
|
+
if video and remove_video:
|
|
426
|
+
raise DatasetError("pass either --video or --remove-video, not both")
|
|
427
|
+
if not any((thumbnail, video, remove_thumbnail, remove_video)):
|
|
428
|
+
raise DatasetError("nothing to change: pass --thumbnail, --video, --remove-thumbnail or --remove-video")
|
|
429
|
+
if thumbnail is not None:
|
|
430
|
+
dataset.detect_media_type("cover", thumbnail)
|
|
431
|
+
if video is not None:
|
|
432
|
+
dataset.detect_media_type("video", video)
|
|
433
|
+
|
|
434
|
+
client = api()
|
|
435
|
+
d: dict | None = None
|
|
436
|
+
if remove_thumbnail:
|
|
437
|
+
d = dataset.delete_media(client, dataset_id, "cover")
|
|
438
|
+
console.print(f"Removed thumbnail from {dataset_id}.")
|
|
439
|
+
if thumbnail is not None:
|
|
440
|
+
d = dataset.upload_media(client, dataset_id, "cover", thumbnail)
|
|
441
|
+
console.print(f"[green]Uploaded thumbnail[/green] for {dataset_id}: {d.get('cover_url')}", highlight=False)
|
|
442
|
+
if remove_video:
|
|
443
|
+
d = dataset.delete_media(client, dataset_id, "video")
|
|
444
|
+
console.print(f"Removed preview clip from {dataset_id}.")
|
|
445
|
+
if video is not None:
|
|
446
|
+
d = dataset.upload_media(client, dataset_id, "video", video)
|
|
447
|
+
console.print(f"[green]Uploaded preview clip[/green] for {dataset_id}: {d.get('video_url')}", highlight=False)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _print_outcome(d: dict) -> None:
|
|
451
|
+
status = d["status"]
|
|
452
|
+
colour = "green" if status in ("approved", "pending_approval", "verifying") else "red"
|
|
453
|
+
console.print(f"[{colour}]{d['id']}: {status}[/{colour}] ({STATE_HELP.get(status, status)})"
|
|
454
|
+
+ (f": {escape(d['review_note'])}" if d.get("review_note") and status != "approved" else ""))
|
|
455
|
+
if status == "verifying":
|
|
456
|
+
console.print("Check on it with `udcli dataset uploads`.")
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
@app.command("uploads")
|
|
460
|
+
def dataset_uploads(as_json: bool = typer.Option(False, "--json")) -> None:
|
|
461
|
+
"""Your uploads and where each one stands (admin)."""
|
|
462
|
+
rows = api().get("/v1/account/dataset-uploads")
|
|
463
|
+
if as_json:
|
|
464
|
+
return emit_json(rows)
|
|
465
|
+
table = Table("id", "status", "price", "size", "sold", "note")
|
|
466
|
+
for d in rows:
|
|
467
|
+
table.add_row(d["id"], f"{d['status']} ({STATE_HELP.get(d['status'], '')})", price_label(d["price_rao"]),
|
|
468
|
+
human_size(d["size_bytes"]), str(d["purchase_count"]), escape(d.get("review_note", "")))
|
|
469
|
+
console.print(table)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
@app.command("discard")
|
|
473
|
+
def dataset_discard(dataset_id: str, yes: bool = YES) -> None:
|
|
474
|
+
"""Throw away your unfinished, invalid or rejected upload (admin)."""
|
|
475
|
+
confirm(f"Discard the upload {dataset_id}?", yes)
|
|
476
|
+
api().delete(f"/v1/datasets/uploads/{dataset_id}")
|
|
477
|
+
console.print(f"Discarded {dataset_id}.")
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
@app.command("set-price")
|
|
481
|
+
def dataset_set_price(dataset_id: str, price: str = typer.Argument(..., help="price in TAO (0 for free)")) -> None:
|
|
482
|
+
"""Change a dataset's price (admin). People who already bought it keep their copy."""
|
|
483
|
+
d = api().put(f"/v1/datasets/{dataset_id}/price", json={"price_rao": tao_to_rao(price)})
|
|
484
|
+
console.print(f"{d['id']}: {price_label(d['price_rao'])}")
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
@app.command("edit")
|
|
488
|
+
def dataset_edit(dataset_id: str, name: Optional[str] = None, description: Optional[str] = None,
|
|
489
|
+
license: Optional[str] = typer.Option(None, "--license"), robot: Optional[str] = None,
|
|
490
|
+
robot_type: Optional[str] = None, modality: Optional[str] = None,
|
|
491
|
+
preview_url: Optional[str] = typer.Option(None, "--preview-url", help="'' removes the link"),
|
|
492
|
+
info: list[str] = typer.Option([], "--info", help="key=value; replaces the facts")) -> None:
|
|
493
|
+
"""Edit catalog text (a super admin any dataset; an admin their own until it is approved)."""
|
|
494
|
+
fields = {"name": name, "description": description, "license": license, "robot": robot,
|
|
495
|
+
"robot_type": robot_type, "modality": modality, "preview_url": preview_url}
|
|
496
|
+
body = {k: v for k, v in fields.items() if v is not None}
|
|
497
|
+
if info:
|
|
498
|
+
body["info"] = _info_items(info)
|
|
499
|
+
if not body:
|
|
500
|
+
raise DatasetError("nothing to change: pass at least one option")
|
|
501
|
+
d = api().request("PATCH", f"/v1/datasets/{dataset_id}", json=body)
|
|
502
|
+
console.print(f"Updated {d['id']}: {', '.join(sorted(body))}")
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
# ---------------------------------------------------------------- review (super admin)
|
|
506
|
+
|
|
507
|
+
@app.command("pending")
|
|
508
|
+
def dataset_pending(as_json: bool = typer.Option(False, "--json")) -> None:
|
|
509
|
+
"""Datasets waiting for approval (admin can look; a super admin decides)."""
|
|
510
|
+
page = api().get("/v1/admin/datasets", params={"status": "pending_approval", "limit": 200})
|
|
511
|
+
if as_json:
|
|
512
|
+
return emit_json(page)
|
|
513
|
+
table = Table("id", "name", "uploaded by", "size", "price", "license")
|
|
514
|
+
for d in page["items"]:
|
|
515
|
+
table.add_row(d["id"], escape(d["name"]), d.get("uploaded_by") or "-", human_size(d["size_bytes"]),
|
|
516
|
+
price_label(d["price_rao"]), escape(d["license"]))
|
|
517
|
+
console.print(table)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
@app.command("approve")
|
|
521
|
+
def dataset_approve(dataset_id: str, note: str = typer.Option("", "--note")) -> None:
|
|
522
|
+
"""Put a verified upload on sale (super admin only)."""
|
|
523
|
+
d = api().post(f"/v1/admin/datasets/{dataset_id}/approve", json={"note": note})
|
|
524
|
+
console.print(f"[green]{d['id']}: {d['status']}[/green] at {price_label(d['price_rao'])}")
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
@app.command("reject")
|
|
528
|
+
def dataset_reject(dataset_id: str, reason: str = typer.Option(..., "--reason")) -> None:
|
|
529
|
+
"""Reject a pending upload with a reason the uploader will see (super admin only)."""
|
|
530
|
+
d = api().post(f"/v1/admin/datasets/{dataset_id}/reject", json={"reason": reason})
|
|
531
|
+
console.print(f"{d['id']}: {d['status']}")
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
@app.command("archive")
|
|
535
|
+
def dataset_archive(dataset_id: str, yes: bool = YES) -> None:
|
|
536
|
+
"""Take a dataset off sale. Buyers keep their access (super admin only)."""
|
|
537
|
+
confirm(f"Take {dataset_id} off sale?", yes)
|
|
538
|
+
d = api().post(f"/v1/admin/datasets/{dataset_id}/archive")
|
|
539
|
+
console.print(f"{d['id']}: {d['status']}")
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
@app.command("restore")
|
|
543
|
+
def dataset_restore(dataset_id: str) -> None:
|
|
544
|
+
"""Put an archived dataset back on sale (super admin only)."""
|
|
545
|
+
d = api().post(f"/v1/admin/datasets/{dataset_id}/restore")
|
|
546
|
+
console.print(f"{d['id']}: {d['status']}")
|