unidroid-cli 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unidroid_cli/__init__.py +1 -0
- unidroid_cli/__main__.py +3 -0
- unidroid_cli/_common.py +74 -0
- unidroid_cli/agent.py +392 -0
- unidroid_cli/bench.py +143 -0
- unidroid_cli/checks.py +42 -0
- unidroid_cli/cli.py +105 -0
- unidroid_cli/commands/__init__.py +1 -0
- unidroid_cli/commands/account.py +118 -0
- unidroid_cli/commands/auth.py +155 -0
- unidroid_cli/commands/bench.py +132 -0
- unidroid_cli/commands/challenge.py +97 -0
- unidroid_cli/commands/dataset.py +546 -0
- unidroid_cli/commands/submission.py +478 -0
- unidroid_cli/commands/validator.py +124 -0
- unidroid_cli/compare.py +64 -0
- unidroid_cli/config.py +85 -0
- unidroid_cli/dataset.py +414 -0
- unidroid_cli/dataset_sample.py +594 -0
- unidroid_cli/hfrepo.py +202 -0
- unidroid_cli/http.py +66 -0
- unidroid_cli/protocol.py +80 -0
- unidroid_cli/safe_extract.py +58 -0
- unidroid_cli/uploads.py +76 -0
- unidroid_cli/wallet.py +62 -0
- unidroid_cli-0.3.0.dist-info/METADATA +196 -0
- unidroid_cli-0.3.0.dist-info/RECORD +29 -0
- unidroid_cli-0.3.0.dist-info/WHEEL +4 -0
- unidroid_cli-0.3.0.dist-info/entry_points.txt +2 -0
unidroid_cli/compare.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Compare a local `udcli submission eval` run with what validators reported on a submission."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import math
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def local_config_sha256(results_path: Path) -> str | None:
|
|
12
|
+
"""sha256 of the eval.yaml a `udcli submission eval` run used: <run>/config/eval.yaml next to <run>/out/."""
|
|
13
|
+
cfg = results_path.parent.parent / "config" / "eval.yaml"
|
|
14
|
+
return hashlib.sha256(cfg.read_bytes()).hexdigest() if cfg.is_file() else None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _number(value) -> float | None:
|
|
18
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
|
19
|
+
return None
|
|
20
|
+
return float(value)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class ValidatorRow:
|
|
25
|
+
name: str
|
|
26
|
+
score: float
|
|
27
|
+
success_rate: float | None
|
|
28
|
+
episodes: int | None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class Comparison:
|
|
33
|
+
local_score: float | None
|
|
34
|
+
local_success_rate: float | None
|
|
35
|
+
local_episodes: int | None
|
|
36
|
+
platform_score: float | None
|
|
37
|
+
validators: list[ValidatorRow]
|
|
38
|
+
config: str # same | different | unknown
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def gap(self) -> float | None:
|
|
42
|
+
if self.local_score is None or self.platform_score is None:
|
|
43
|
+
return None
|
|
44
|
+
return self.local_score - self.platform_score
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def compare(local: dict, phase: dict, local_sha: str | None) -> Comparison:
|
|
48
|
+
"""``phase`` is one entry of ``evaluation.phases`` from GET /v1/submissions/{id}."""
|
|
49
|
+
metrics = local.get("metrics") if isinstance(local.get("metrics"), dict) else {}
|
|
50
|
+
episodes = _number(metrics.get("episodes"))
|
|
51
|
+
scored = [r for r in phase.get("results") or [] if r.get("status") == "scored" and r.get("score") is not None]
|
|
52
|
+
rows = []
|
|
53
|
+
for r in scored:
|
|
54
|
+
name = r.get("name") or (f"uid {r['uid']}" if r.get("uid") is not None else r["hotkey"][:10])
|
|
55
|
+
m = r.get("metrics") or {}
|
|
56
|
+
count = _number(m.get("episodes"))
|
|
57
|
+
rows.append(ValidatorRow(name, float(r["score"]), _number(m.get("success_rate")),
|
|
58
|
+
int(count) if count is not None else None))
|
|
59
|
+
if local_sha is None:
|
|
60
|
+
config = "unknown"
|
|
61
|
+
else:
|
|
62
|
+
config = "same" if local_sha == phase.get("config_sha256") else "different"
|
|
63
|
+
return Comparison(_number(local.get("score")), _number(metrics.get("success_rate")),
|
|
64
|
+
int(episodes) if episodes is not None else None, phase.get("score"), rows, config)
|
unidroid_cli/config.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Local credentials and paths (~/.unidroid)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import subprocess
|
|
8
|
+
import warnings
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
DEFAULT_API_URL = "https://api.unidroid.ai"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def home() -> Path:
|
|
15
|
+
return Path(os.environ.get("UNIDROID_HOME", Path.home() / ".unidroid"))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def credentials_path() -> Path:
|
|
19
|
+
return home() / "credentials.json"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def load_credentials() -> dict:
|
|
23
|
+
path = credentials_path()
|
|
24
|
+
if not path.is_file():
|
|
25
|
+
return {}
|
|
26
|
+
try:
|
|
27
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
28
|
+
except (OSError, ValueError):
|
|
29
|
+
return {}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _restrict_to_current_user(path: Path) -> bool:
|
|
33
|
+
"""Make ``path`` readable by the current user only. Returns False when that could not be done."""
|
|
34
|
+
if os.name != "nt":
|
|
35
|
+
try:
|
|
36
|
+
path.chmod(0o600)
|
|
37
|
+
return True
|
|
38
|
+
except OSError:
|
|
39
|
+
return False
|
|
40
|
+
user = os.environ.get("USERNAME")
|
|
41
|
+
if not user:
|
|
42
|
+
return False
|
|
43
|
+
domain = os.environ.get("USERDOMAIN")
|
|
44
|
+
account = f"{domain}\\{user}" if domain else user
|
|
45
|
+
try:
|
|
46
|
+
done = subprocess.run(["icacls", str(path), "/inheritance:r", "/grant:r", f"{account}:F"],
|
|
47
|
+
capture_output=True, timeout=30, check=False)
|
|
48
|
+
except (OSError, subprocess.SubprocessError):
|
|
49
|
+
return False
|
|
50
|
+
return done.returncode == 0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def save_credentials(data: dict) -> Path:
|
|
54
|
+
"""Write credentials.json through a private temp file and an atomic replace, so a crash never leaves a
|
|
55
|
+
half-written file and the key is never readable by others, even briefly."""
|
|
56
|
+
path = credentials_path()
|
|
57
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
58
|
+
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
|
59
|
+
try:
|
|
60
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
61
|
+
os.close(fd)
|
|
62
|
+
if not _restrict_to_current_user(tmp):
|
|
63
|
+
warnings.warn(f"could not restrict {path} to your user; check its permissions", stacklevel=2)
|
|
64
|
+
tmp.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
|
65
|
+
os.replace(tmp, path)
|
|
66
|
+
finally:
|
|
67
|
+
tmp.unlink(missing_ok=True)
|
|
68
|
+
return path
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def clear_credentials() -> None:
|
|
72
|
+
credentials_path().unlink(missing_ok=True)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def api_url(override: str | None = None) -> str:
|
|
76
|
+
return (override or os.environ.get("UNIDROID_API_URL") or load_credentials().get("api_url")
|
|
77
|
+
or DEFAULT_API_URL).rstrip("/")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def api_key(override: str | None = None) -> str | None:
|
|
81
|
+
return override or os.environ.get("UNIDROID_API_KEY") or load_credentials().get("api_key")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def bench_cache_dir(override: str | None = None) -> Path:
|
|
85
|
+
return Path(override or os.environ.get("UNIDROID_CACHE_DIR") or home() / "bench-cache")
|
unidroid_cli/dataset.py
ADDED
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
"""Dataset client: resumable, sha256-verified downloads, resumable multipart uploads, and media uploads."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import time
|
|
8
|
+
import zipfile
|
|
9
|
+
from collections.abc import Callable, Iterator
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from decimal import Decimal, InvalidOperation
|
|
13
|
+
from fnmatch import fnmatch
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import httpx
|
|
17
|
+
|
|
18
|
+
from . import uploads
|
|
19
|
+
from .safe_extract import UnsafeArchive, safe_extract
|
|
20
|
+
from .http import Api, ApiError, raw_client
|
|
21
|
+
|
|
22
|
+
TAO = 1_000_000_000
|
|
23
|
+
FORMATS = {".tar.gz": "tar.gz", ".tgz": "tar.gz", ".tar": "tar", ".zip": "zip"}
|
|
24
|
+
CHUNK = 8 << 20
|
|
25
|
+
IMAGE_MAX_BYTES = 5 * 1024 * 1024
|
|
26
|
+
VIDEO_MAX_BYTES = 50 * 1024 * 1024
|
|
27
|
+
JUNK_NAMES = (".DS_Store", "Thumbs.db", "*.swp", "*.swo", "*~")
|
|
28
|
+
JUNK_DIRS = {".cache", ".git", "__pycache__"}
|
|
29
|
+
STAGING_PREFIX = ".udcli-upload-"
|
|
30
|
+
FIXED_ZIP_TIME = (1980, 1, 1, 0, 0, 0)
|
|
31
|
+
Progress = Callable[[int, int], None]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class DatasetError(Exception):
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class PreparedArchive:
|
|
40
|
+
path: Path
|
|
41
|
+
sha256: str | None = None
|
|
42
|
+
reused_staged: bool = False
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def tao_to_rao(text: str) -> int:
|
|
46
|
+
"""Exact conversion of a TAO amount typed by a person ("0.25") to rao."""
|
|
47
|
+
try:
|
|
48
|
+
value = Decimal(str(text).strip())
|
|
49
|
+
except InvalidOperation:
|
|
50
|
+
raise DatasetError(f"{text!r} is not a number of TAO") from None
|
|
51
|
+
if not value.is_finite() or value < 0:
|
|
52
|
+
raise DatasetError("the price must be zero or more TAO")
|
|
53
|
+
rao = value * TAO
|
|
54
|
+
if rao != rao.to_integral_value():
|
|
55
|
+
raise DatasetError("the price has more than 9 decimal places")
|
|
56
|
+
return int(rao)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def human_size(n: int) -> str:
|
|
60
|
+
size = float(n)
|
|
61
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
62
|
+
if size < 1000 or unit == "TB":
|
|
63
|
+
return f"{size:.0f} {unit}" if unit == "B" else f"{size:.2f} {unit}"
|
|
64
|
+
size /= 1000
|
|
65
|
+
return f"{n} B"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def sha256_file(path: Path, progress: Progress | None = None) -> str:
|
|
69
|
+
digest, done, total = hashlib.sha256(), 0, path.stat().st_size
|
|
70
|
+
with path.open("rb") as fh:
|
|
71
|
+
for chunk in iter(lambda: fh.read(CHUNK), b""):
|
|
72
|
+
digest.update(chunk)
|
|
73
|
+
done += len(chunk)
|
|
74
|
+
if progress:
|
|
75
|
+
progress(done, total)
|
|
76
|
+
return digest.hexdigest()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# ---------------------------------------------------------------- download
|
|
80
|
+
|
|
81
|
+
def _mint(api: Api, dataset_id: str) -> dict:
|
|
82
|
+
try:
|
|
83
|
+
return api.get(f"/v1/datasets/{dataset_id}/download")
|
|
84
|
+
except ApiError as exc:
|
|
85
|
+
if exc.status == 403 and exc.detail == "purchase required":
|
|
86
|
+
raise DatasetError(f"purchase required: run `udcli dataset buy {dataset_id}` first") from None
|
|
87
|
+
raise
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _hash_prefix(path: Path, digest) -> int:
|
|
91
|
+
done = 0
|
|
92
|
+
with path.open("rb") as fh:
|
|
93
|
+
for chunk in iter(lambda: fh.read(CHUNK), b""):
|
|
94
|
+
digest.update(chunk)
|
|
95
|
+
done += len(chunk)
|
|
96
|
+
return done
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _fetch(api: Api, dataset_id: str, info: dict, dest: Path, progress: Progress | None, attempts: int) -> None:
|
|
100
|
+
"""Stream the archive into ``<name>.part``, resuming after a dropped connection or an earlier run."""
|
|
101
|
+
part = dest.with_name(dest.name + ".part")
|
|
102
|
+
total, digest, offset, failures = int(info["size_bytes"]), hashlib.sha256(), 0, 0
|
|
103
|
+
if part.is_file() and part.stat().st_size <= total:
|
|
104
|
+
offset = _hash_prefix(part, digest)
|
|
105
|
+
else:
|
|
106
|
+
part.unlink(missing_ok=True)
|
|
107
|
+
with raw_client() as client:
|
|
108
|
+
while offset < total:
|
|
109
|
+
before = offset
|
|
110
|
+
try:
|
|
111
|
+
with client.stream("GET", info["url"], headers={"Range": f"bytes={offset}-"} if offset else {}) as resp:
|
|
112
|
+
if resp.status_code in (401, 403): # the link expired: ask for a new one
|
|
113
|
+
info = _mint(api, dataset_id)
|
|
114
|
+
elif resp.status_code >= 400:
|
|
115
|
+
raise DatasetError(f"download failed with HTTP {resp.status_code}")
|
|
116
|
+
else:
|
|
117
|
+
if offset and resp.status_code != 206: # the server ignored the range: start over
|
|
118
|
+
digest, offset = hashlib.sha256(), 0
|
|
119
|
+
with part.open("ab" if offset else "wb") as fh:
|
|
120
|
+
for chunk in resp.iter_bytes(CHUNK):
|
|
121
|
+
fh.write(chunk)
|
|
122
|
+
digest.update(chunk)
|
|
123
|
+
offset += len(chunk)
|
|
124
|
+
if progress:
|
|
125
|
+
progress(offset, total)
|
|
126
|
+
except httpx.HTTPError:
|
|
127
|
+
pass # counted below: no progress means a failure
|
|
128
|
+
except OSError as exc:
|
|
129
|
+
raise DatasetError(f"cannot write {part}: {exc}") from None
|
|
130
|
+
if offset == before:
|
|
131
|
+
failures += 1
|
|
132
|
+
if failures >= attempts:
|
|
133
|
+
raise DatasetError(f"download stalled at {human_size(offset)} of {human_size(total)}; "
|
|
134
|
+
"run the command again to resume")
|
|
135
|
+
time.sleep(min(2 ** failures, 20))
|
|
136
|
+
else:
|
|
137
|
+
failures = 0
|
|
138
|
+
if offset != total or digest.hexdigest() != info["sha256"]:
|
|
139
|
+
part.unlink(missing_ok=True)
|
|
140
|
+
raise DatasetError("sha256 mismatch: the downloaded archive is corrupted; run the command again")
|
|
141
|
+
part.replace(dest)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def download(api: Api, dataset_id: str, out_dir: Path, *, force: bool = False, progress: Progress | None = None,
|
|
145
|
+
attempts: int = 5) -> Path:
|
|
146
|
+
"""Download the archive into ``out_dir`` and return its path. An intact earlier copy is reused."""
|
|
147
|
+
info = _mint(api, dataset_id)
|
|
148
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
149
|
+
dest = out_dir / Path(info["filename"]).name
|
|
150
|
+
if not force and dest.is_file() and dest.stat().st_size == info["size_bytes"] \
|
|
151
|
+
and sha256_file(dest) == info["sha256"]:
|
|
152
|
+
return dest
|
|
153
|
+
_fetch(api, dataset_id, info, dest, progress, attempts)
|
|
154
|
+
return dest
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def extract(archive: Path, target: Path) -> Path:
|
|
158
|
+
"""Unpack into ``target`` (created if needed), refusing anything that would land outside it."""
|
|
159
|
+
try:
|
|
160
|
+
return safe_extract(archive, target)
|
|
161
|
+
except UnsafeArchive as exc:
|
|
162
|
+
raise DatasetError(str(exc)) from None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# ---------------------------------------------------------------- archive & upload
|
|
166
|
+
|
|
167
|
+
def detect_format(path: Path) -> str:
|
|
168
|
+
name = path.name.lower()
|
|
169
|
+
for suffix, fmt in FORMATS.items():
|
|
170
|
+
if name.endswith(suffix):
|
|
171
|
+
return fmt
|
|
172
|
+
raise DatasetError(f"{path.name}: expected a folder or a .zip, .tar or .tar.gz archive")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _is_junk(rel_parts: tuple[str, ...]) -> bool:
|
|
176
|
+
if any(part in JUNK_DIRS or part.startswith(STAGING_PREFIX) for part in rel_parts):
|
|
177
|
+
return True
|
|
178
|
+
name = rel_parts[-1]
|
|
179
|
+
return any(fnmatch(name, pat) for pat in JUNK_NAMES)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def iter_dataset_files(folder: Path, exclude: set[Path] | None = None) -> list[Path]:
|
|
183
|
+
"""Sorted files inside ``folder``, skipping OS/editor/VCS junk and any explicit exclusions."""
|
|
184
|
+
excluded = {p.resolve() for p in (exclude or set())}
|
|
185
|
+
files: list[Path] = []
|
|
186
|
+
for path in folder.rglob("*"):
|
|
187
|
+
if not path.is_file():
|
|
188
|
+
continue
|
|
189
|
+
rel_parts = path.relative_to(folder).parts
|
|
190
|
+
if _is_junk(rel_parts):
|
|
191
|
+
continue
|
|
192
|
+
if excluded and path.resolve() in excluded:
|
|
193
|
+
continue
|
|
194
|
+
files.append(path)
|
|
195
|
+
return sorted(files, key=lambda p: p.relative_to(folder).as_posix())
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class _HashingStreamWriter:
|
|
199
|
+
"""Non-seekable sequential writer that computes SHA-256 while ZipFile writes the archive once."""
|
|
200
|
+
|
|
201
|
+
def __init__(self, fh):
|
|
202
|
+
self._fh = fh
|
|
203
|
+
self._pos = 0
|
|
204
|
+
self.digest = hashlib.sha256()
|
|
205
|
+
|
|
206
|
+
def write(self, data: bytes | memoryview) -> int:
|
|
207
|
+
raw = bytes(data)
|
|
208
|
+
n = self._fh.write(raw)
|
|
209
|
+
self.digest.update(raw[:n])
|
|
210
|
+
self._pos += n
|
|
211
|
+
return n
|
|
212
|
+
|
|
213
|
+
def tell(self) -> int:
|
|
214
|
+
return self._pos
|
|
215
|
+
|
|
216
|
+
def flush(self) -> None:
|
|
217
|
+
self._fh.flush()
|
|
218
|
+
|
|
219
|
+
def seekable(self) -> bool:
|
|
220
|
+
return False
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def build_archive(folder: Path, out: Path, progress: Callable[[int, int], None] | None = None,
|
|
224
|
+
*, exclude: set[Path] | None = None) -> str:
|
|
225
|
+
"""Zip a folder deterministically without recompressing it, returning its SHA-256 computed in one pass."""
|
|
226
|
+
files = iter_dataset_files(folder, exclude=exclude)
|
|
227
|
+
if not files:
|
|
228
|
+
raise DatasetError(f"{folder} contains no files")
|
|
229
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
230
|
+
with out.open("wb") as raw_fh:
|
|
231
|
+
writer = _HashingStreamWriter(raw_fh)
|
|
232
|
+
with zipfile.ZipFile(writer, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as zf: # type: ignore[arg-type]
|
|
233
|
+
for i, path in enumerate(files, 1):
|
|
234
|
+
rel = path.relative_to(folder).as_posix()
|
|
235
|
+
size = path.stat().st_size
|
|
236
|
+
zinfo = zipfile.ZipInfo(filename=rel, date_time=FIXED_ZIP_TIME)
|
|
237
|
+
zinfo.compress_type = zipfile.ZIP_STORED
|
|
238
|
+
zinfo.external_attr = 0o644 << 16
|
|
239
|
+
with path.open("rb") as src, zf.open(zinfo, "w", force_zip64=size >= 0xFFFFFFFF) as dst:
|
|
240
|
+
for chunk in iter(lambda: src.read(CHUNK), b""):
|
|
241
|
+
dst.write(chunk)
|
|
242
|
+
if progress:
|
|
243
|
+
progress(i, len(files))
|
|
244
|
+
writer.flush()
|
|
245
|
+
return writer.digest.hexdigest()
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _folder_fingerprint(folder: Path, files: list[Path]) -> str:
|
|
249
|
+
digest = hashlib.sha256()
|
|
250
|
+
for path in files:
|
|
251
|
+
st = path.stat()
|
|
252
|
+
digest.update(f"{path.relative_to(folder).as_posix()}\0{st.st_size}\0{st.st_mtime_ns}\n".encode())
|
|
253
|
+
return digest.hexdigest()
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _resumable(api: Api, dataset_id: str, sha: str, size: int) -> dict | None:
|
|
257
|
+
"""The open upload of this very archive, if there is one: the stored parts and the part size."""
|
|
258
|
+
try:
|
|
259
|
+
status = api.get(f"/v1/datasets/uploads/{dataset_id}")
|
|
260
|
+
except ApiError as exc:
|
|
261
|
+
if exc.status == 404:
|
|
262
|
+
return None
|
|
263
|
+
raise
|
|
264
|
+
d = status["dataset"]
|
|
265
|
+
if d["status"] == "uploading" and d["sha256"] == sha and d["size_bytes"] == size:
|
|
266
|
+
return status
|
|
267
|
+
return None
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def upload(api: Api, archive: Path, meta: dict, *, sha256: str | None = None, jobs: int = 4, attempts: int = 5,
|
|
271
|
+
progress: Progress | None = None, on_resume: Callable[[int, int], None] | None = None) -> dict:
|
|
272
|
+
"""Upload one archive. Running it again after an interruption continues with the parts still missing.
|
|
273
|
+
|
|
274
|
+
``meta`` holds the catalog fields (id, name, license, price_rao, ...). Memory use is about
|
|
275
|
+
``jobs`` times the part size (128 MB by default).
|
|
276
|
+
"""
|
|
277
|
+
size = archive.stat().st_size
|
|
278
|
+
sha = sha256 or sha256_file(archive)
|
|
279
|
+
dataset_id = meta["id"]
|
|
280
|
+
body = {**meta, "format": detect_format(archive), "sha256": sha, "size_bytes": size}
|
|
281
|
+
status = _resumable(api, dataset_id, sha, size)
|
|
282
|
+
stored: dict[int, str] = {}
|
|
283
|
+
if status is not None:
|
|
284
|
+
part_size, count = status["part_size"], status["part_count"]
|
|
285
|
+
stored = {p["part_number"]: p["etag"] for p in status["uploaded_parts"]}
|
|
286
|
+
todo = [n for n in range(1, count + 1) if n not in stored]
|
|
287
|
+
urls = {p["part_number"]: p["url"] for p in api.post(
|
|
288
|
+
f"/v1/datasets/uploads/{dataset_id}/parts", json={"part_numbers": todo})["parts"]} if todo else {}
|
|
289
|
+
if on_resume:
|
|
290
|
+
on_resume(len(stored), count)
|
|
291
|
+
else:
|
|
292
|
+
info = api.post("/v1/datasets/uploads", json=body)
|
|
293
|
+
part_size, count = info["part_size"], info["part_count"]
|
|
294
|
+
urls = {p["part_number"]: p["url"] for p in info["parts"]}
|
|
295
|
+
todo = list(range(1, count + 1))
|
|
296
|
+
|
|
297
|
+
def refresh(number: int) -> str:
|
|
298
|
+
return api.post(f"/v1/datasets/uploads/{dataset_id}/parts", json={"part_numbers": [number]})["parts"][0]["url"]
|
|
299
|
+
|
|
300
|
+
try:
|
|
301
|
+
parts = uploads.upload_parts(archive, part_size, urls, stored=stored, jobs=jobs, attempts=attempts,
|
|
302
|
+
refresh=refresh, progress=progress)
|
|
303
|
+
except uploads.PartUploadError as exc:
|
|
304
|
+
raise DatasetError(f"{exc}; run the command again to resume") from None
|
|
305
|
+
return api.post(f"/v1/datasets/uploads/{dataset_id}/complete", json={"parts": parts})
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
@contextmanager
|
|
309
|
+
def prepared_archive(
|
|
310
|
+
source: Path,
|
|
311
|
+
dataset_id: str,
|
|
312
|
+
*,
|
|
313
|
+
work_dir: Path | None = None,
|
|
314
|
+
exclude: set[Path] | None = None,
|
|
315
|
+
progress: Callable[[int, int], None] | None = None,
|
|
316
|
+
) -> Iterator[PreparedArchive]:
|
|
317
|
+
"""Prepare an archive with single-pass SHA-256 and resumable staging on interruptions.
|
|
318
|
+
|
|
319
|
+
When ``source`` is a directory, the ZIP is staged next to the folder (or in ``work_dir``) rather than
|
|
320
|
+
``/tmp``. If the upload fails mid-way, the staged ZIP and its SHA-256 sidecar are kept so re-running the
|
|
321
|
+
command resumes immediately without re-zipping or re-hashing; on success, both are deleted.
|
|
322
|
+
"""
|
|
323
|
+
if source.is_file():
|
|
324
|
+
detect_format(source)
|
|
325
|
+
yield PreparedArchive(path=source, sha256=None, reused_staged=False)
|
|
326
|
+
return
|
|
327
|
+
if not source.is_dir():
|
|
328
|
+
raise DatasetError(f"{source} not found")
|
|
329
|
+
|
|
330
|
+
files = iter_dataset_files(source, exclude=exclude)
|
|
331
|
+
if not files:
|
|
332
|
+
raise DatasetError(f"{source} contains no files")
|
|
333
|
+
fingerprint = _folder_fingerprint(source, files)
|
|
334
|
+
|
|
335
|
+
stage_root = (work_dir or source.resolve().parent)
|
|
336
|
+
stage_root.mkdir(parents=True, exist_ok=True)
|
|
337
|
+
zip_path = stage_root / f"{STAGING_PREFIX}{dataset_id}.zip"
|
|
338
|
+
meta_path = stage_root / f"{STAGING_PREFIX}{dataset_id}.json"
|
|
339
|
+
|
|
340
|
+
reused = False
|
|
341
|
+
sha: str | None = None
|
|
342
|
+
if zip_path.is_file() and meta_path.is_file():
|
|
343
|
+
try:
|
|
344
|
+
saved = json.loads(meta_path.read_text(encoding="utf-8"))
|
|
345
|
+
st = zip_path.stat()
|
|
346
|
+
if (saved.get("fingerprint") == fingerprint
|
|
347
|
+
and saved.get("size_bytes") == st.st_size
|
|
348
|
+
and saved.get("mtime_ns") == st.st_mtime_ns
|
|
349
|
+
and isinstance(saved.get("sha256"), str)
|
|
350
|
+
and len(saved["sha256"]) == 64):
|
|
351
|
+
sha = saved["sha256"]
|
|
352
|
+
reused = True
|
|
353
|
+
except (OSError, ValueError, TypeError):
|
|
354
|
+
reused = False
|
|
355
|
+
|
|
356
|
+
if not reused:
|
|
357
|
+
part_zip = stage_root / f"{STAGING_PREFIX}{dataset_id}.zip.tmp"
|
|
358
|
+
try:
|
|
359
|
+
sha = build_archive(source, part_zip, progress, exclude=exclude)
|
|
360
|
+
part_zip.replace(zip_path)
|
|
361
|
+
st = zip_path.stat()
|
|
362
|
+
meta_path.write_text(
|
|
363
|
+
json.dumps({"fingerprint": fingerprint, "sha256": sha,
|
|
364
|
+
"size_bytes": st.st_size, "mtime_ns": st.st_mtime_ns}),
|
|
365
|
+
encoding="utf-8",
|
|
366
|
+
)
|
|
367
|
+
except BaseException:
|
|
368
|
+
part_zip.unlink(missing_ok=True)
|
|
369
|
+
raise
|
|
370
|
+
|
|
371
|
+
yield PreparedArchive(path=zip_path, sha256=sha, reused_staged=reused)
|
|
372
|
+
# Reached only when the caller's upload succeeded without raising: clean up the staged archive.
|
|
373
|
+
zip_path.unlink(missing_ok=True)
|
|
374
|
+
meta_path.unlink(missing_ok=True)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
# ---------------------------------------------------------------- preview media (thumbnail & short video clip)
|
|
378
|
+
|
|
379
|
+
def detect_media_type(kind: str, path: Path) -> str:
|
|
380
|
+
"""Validate file existence, size cap and magic bytes for ``cover`` or ``video``; return Content-Type."""
|
|
381
|
+
label = "thumbnail" if kind == "cover" else "preview video"
|
|
382
|
+
if not path.is_file():
|
|
383
|
+
raise DatasetError(f"{label} not found: {path}")
|
|
384
|
+
size = path.stat().st_size
|
|
385
|
+
if size == 0:
|
|
386
|
+
raise DatasetError(f"{label} is empty: {path}")
|
|
387
|
+
limit = VIDEO_MAX_BYTES if kind == "video" else IMAGE_MAX_BYTES
|
|
388
|
+
if size > limit:
|
|
389
|
+
raise DatasetError(f"{label} {path.name} is {human_size(size)}; maximum is {limit // (1024 * 1024)} MB")
|
|
390
|
+
with path.open("rb") as fh:
|
|
391
|
+
head = fh.read(16)
|
|
392
|
+
if kind == "cover":
|
|
393
|
+
if head.startswith(b"\x89PNG\r\n\x1a\n"):
|
|
394
|
+
return "image/png"
|
|
395
|
+
if head.startswith(b"\xff\xd8\xff"):
|
|
396
|
+
return "image/jpeg"
|
|
397
|
+
if len(head) >= 12 and head[:4] == b"RIFF" and head[8:12] == b"WEBP":
|
|
398
|
+
return "image/webp"
|
|
399
|
+
raise DatasetError(f"{path.name}: thumbnail must be a PNG, JPEG or WebP image")
|
|
400
|
+
if len(head) >= 8 and head[4:8] == b"ftyp":
|
|
401
|
+
return "video/mp4"
|
|
402
|
+
if head.startswith(b"\x1a\x45\xdf\xa3"):
|
|
403
|
+
return "video/webm"
|
|
404
|
+
raise DatasetError(f"{path.name}: preview video must be an MP4 or WebM clip")
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def upload_media(api: Api, dataset_id: str, kind: str, path: Path) -> dict:
|
|
408
|
+
content_type = detect_media_type(kind, path)
|
|
409
|
+
return api.put(f"/v1/datasets/{dataset_id}/media/{kind}", content=path.read_bytes(),
|
|
410
|
+
headers={"Content-Type": content_type})
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def delete_media(api: Api, dataset_id: str, kind: str) -> dict:
|
|
414
|
+
return api.delete(f"/v1/datasets/{dataset_id}/media/{kind}")
|