unidroid-cli 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,478 @@
1
+ """udcli submission: prepare, check, evaluate, submit and track your entries."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import secrets
9
+ import shlex
10
+ import shutil
11
+ import subprocess
12
+ import time
13
+ from pathlib import Path
14
+ from typing import Optional
15
+
16
+ import typer
17
+ from rich.table import Table
18
+
19
+ from .. import agent, bench, checks, compare, config, hfrepo, protocol, uploads, wallet
20
+ from .._common import api, console, emit_json, err, group, hash_progress, local_time, pick_challenge, report, tao
21
+ from ..http import Api, ApiError
22
+
23
+ app = group("Your entries, in order: hash, push, check, eval, submit, then list and show.")
24
+
25
+
26
+ # ------------------------------------------------------------------ prepare a submission folder
27
+
28
+ def _check_submission_folder(path: Path, *, hash_weights: bool = True) -> agent.Report:
29
+ report_ = agent.check(path, hash_weights_too=hash_weights, progress=hash_progress)
30
+ for name in report_.skipped:
31
+ console.print(f"[yellow]not in the bundle:[/yellow] {name}", highlight=False)
32
+ if report_.bundle:
33
+ size = sum(p.stat().st_size for _, p in report_.bundle)
34
+ weights = sum(w.size for w in report_.local.values())
35
+ console.print(f"Bundle: {len(report_.bundle)} files, {size / 1e6:.2f} MB. Weights: {len(report_.local)} files, "
36
+ f"{weights / 1e9:.2f} GB.")
37
+ return report_
38
+
39
+
40
+ @app.command("hash")
41
+ def submission_hash(
42
+ path: Path = typer.Option(Path("."), "--path", help="The submission folder (agent.yaml, brain/, weights/)."),
43
+ check_only: bool = typer.Option(False, "--check", help="Only compare; exit 1 when agent.yaml is out of date."),
44
+ ) -> None:
45
+ """Hash every file in weights/ and write the sha256 map into agent.yaml's `weights:` block.
46
+
47
+ Only that block is rewritten (sorted by path); every other line and comment stays as it is.
48
+ """
49
+ text, cfg = agent.read_agent(path)
50
+ local = agent.hash_weights(path, hash_progress)
51
+ declared = agent.declared_weights(cfg)
52
+ diff = agent.weights_diff(declared, local)
53
+ if check_only:
54
+ report(diff)
55
+ return
56
+ if not diff:
57
+ console.print(f"[green]agent.yaml is up to date[/green] ({len(local)} weights files).")
58
+ return
59
+ for line in diff:
60
+ console.print(f" {line}", markup=False)
61
+ new_text = agent.rewrite_weights_block(text, {p: w.sha256 for p, w in local.items()})
62
+ (path / agent.AGENT_YAML).write_bytes(new_text.encode("utf-8"))
63
+ console.print(f"[green]Wrote[/green] {len(local)} weights hashes to {path / agent.AGENT_YAML}. "
64
+ f"weights_digest {protocol.weights_digest({p: w.sha256 for p, w in local.items()})}")
65
+
66
+
67
+ @app.command("push")
68
+ def submission_push(
69
+ path: Path = typer.Option(Path("."), "--path", help="Submission folder; its weights/ is pushed."),
70
+ hf_repo: str = typer.Option(..., "--hf-repo", help="owner/name on Hugging Face (created private if missing)."),
71
+ hf_token: Optional[str] = typer.Option(None, "--hf-token", envvar="HF_TOKEN",
72
+ help="Write token (defaults to `hf auth login`). Stays on this machine."),
73
+ public_repo: bool = typer.Option(False, "--public-repo"),
74
+ ) -> None:
75
+ """Push weights/ to Hugging Face so the repo holds exactly the files hashed in agent.yaml."""
76
+ _, cfg = agent.read_agent(path)
77
+ local = agent.hash_weights(path, hash_progress)
78
+ diff = agent.weights_diff(agent.declared_weights(cfg), local)
79
+ if diff:
80
+ report(diff + ["run `udcli submission hash` first"])
81
+ token = hfrepo.local_token(hf_token)
82
+ if not token:
83
+ raise ApiError(401, "pass --hf-token, set HF_TOKEN or run `hf auth login`")
84
+ sha = hfrepo.push_weights(hf_repo, {p: path / agent.WEIGHTS_DIR / p for p in local}, token,
85
+ private=not public_repo)
86
+ console.print(f"[green]Pushed[/green] {len(local)} files to {hf_repo} at commit {sha}.")
87
+
88
+
89
+ @app.command("check")
90
+ def submission_check(
91
+ path: Path = typer.Option(Path("."), "--path", help="brain-comm submission folder (agent.yaml, brain/, weights/)."),
92
+ challenge: Optional[str] = typer.Option(None, "--challenge", help="Also confirm this challenge is open."),
93
+ offline: bool = typer.Option(False, "--offline", help="Skip the challenge lookup."),
94
+ ) -> None:
95
+ """Check a submission folder: layout, schema, weight hashes and the bundle rules."""
96
+ if challenge and not offline:
97
+ ch = pick_challenge(api(require_key=False), challenge)
98
+ if not ch["accepting_submissions"]:
99
+ if ch.get("test_mode") and ch["current_period"] == "contest":
100
+ raise ApiError(403, f"{ch['id']} is in test mode; only whitelisted accounts can submit")
101
+ raise ApiError(409, f"{ch['id']} is in its {ch['current_period']} period; uploads are closed")
102
+ report(_check_submission_folder(path).problems)
103
+
104
+
105
+ # ------------------------------------------------------------------ local evaluation
106
+
107
+ def docker_command(image: str, model: Path, benches: dict[str, Path], out: Path, cfg: Path, gpus: str) -> list[str]:
108
+ """Same sandbox as the validator."""
109
+ cmd = ["docker", "run", "--rm", "--network", "none", "--cap-drop", "ALL", "--security-opt", "no-new-privileges",
110
+ "--read-only", "--tmpfs", "/tmp:rw,exec,size=16g", "--shm-size", "8g", "-e", "HOME=/tmp"]
111
+ mounts = [f"{model.resolve()}:/model:ro", f"{out.resolve()}:/out", f"{cfg.resolve()}:/config:ro",
112
+ *(f"{path.resolve()}:/benchmark/{bench_id}:ro" for bench_id, path in benches.items())]
113
+ for m in mounts:
114
+ cmd += ["-v", m]
115
+ if gpus:
116
+ cmd += ["--gpus", gpus]
117
+ return cmd + [image]
118
+
119
+
120
+ @app.command("eval")
121
+ def submission_eval(
122
+ challenge: Optional[str] = typer.Option(None, "--challenge"),
123
+ model_dir: Optional[Path] = typer.Option(
124
+ None, "--model-dir", help="Local model folder, mounted at /model (not the submission folder)."),
125
+ path: Optional[Path] = typer.Option(None, "--path", hidden=True, help="Old name of --model-dir."),
126
+ hf_repo: Optional[str] = typer.Option(None, "--hf-repo"),
127
+ revision: str = typer.Option("main", "--revision"),
128
+ hf_token: Optional[str] = typer.Option(None, "--hf-token", envvar="HF_TOKEN"),
129
+ image: Optional[str] = typer.Option(None, "--image", help="Evaluator container image."),
130
+ gpus: str = typer.Option("all", "--gpus", help="Docker --gpus value; empty for CPU."),
131
+ out_dir: Optional[Path] = typer.Option(None, "--out"),
132
+ dry_run: bool = typer.Option(False, "--dry-run", help="Print the docker command only."),
133
+ ) -> None:
134
+ """Evaluate a model locally: run an evaluator image with the challenge's public eval config.
135
+
136
+ Linked public benchmarks are mounted for the image, in the same sandbox as validators.
137
+ """
138
+ client = api()
139
+ ch = pick_challenge(client, challenge)
140
+ public_eval = ch.get("public_eval") or ""
141
+ if not isinstance(public_eval, str) or not public_eval.strip():
142
+ raise ApiError(400, f"{ch['id']} has no public evaluation to run locally")
143
+ public = [b for b in ch.get("benchmarks") or [] if b.get("visibility") == "public"]
144
+ if not image:
145
+ raise ApiError(400, "pass --image; the challenge stores an eval repo, not a container image")
146
+ benches = {b["id"]: bench.download(client, b["id"],
147
+ config.bench_cache_dir()) for b in public}
148
+ if path and not model_dir:
149
+ err.print("[yellow]--path is now --model-dir for `submission eval`; the old name will be removed.[/yellow]")
150
+ model_dir = path
151
+ if model_dir:
152
+ model = model_dir
153
+ elif hf_repo:
154
+ from huggingface_hub import snapshot_download
155
+
156
+ token = hfrepo.local_token(hf_token)
157
+ snap = hfrepo.resolve(hf_repo, revision, token)
158
+ model = config.home() / "models" / hf_repo.replace("/", "__") / snap.commit_sha
159
+ snapshot_download(hf_repo, revision=snap.commit_sha, token=token or False, local_dir=model)
160
+ else:
161
+ raise ApiError(400, "pass --model-dir or --hf-repo")
162
+ out = out_dir or Path(f"udcli-eval-{ch['id']}-{int(time.time())}")
163
+ cfg = out / "config"
164
+ (out / "out").mkdir(parents=True, exist_ok=True)
165
+ cfg.mkdir(parents=True, exist_ok=True)
166
+ # Bytes, not text: the config sha256 must match the platform's on Windows too.
167
+ (cfg / "eval.yaml").write_bytes(public_eval.encode("utf-8"))
168
+ cmd = docker_command(image, model, benches, out / "out", cfg, gpus)
169
+ typer.echo(subprocess.list2cmdline(cmd) if os.name == "nt" else shlex.join(cmd))
170
+ if dry_run:
171
+ return
172
+ if not shutil.which("docker"):
173
+ raise ApiError(400, "docker is not installed or not on PATH")
174
+ code = subprocess.call(cmd)
175
+ results_path = out / "out" / "results.json"
176
+ if code != 0 or not results_path.is_file():
177
+ raise ApiError(1, f"evaluator failed (exit code {code}); see the output above")
178
+ results = json.loads(results_path.read_text(encoding="utf-8"))
179
+ console.print(f"[green]Score {results.get('score')}[/green] metrics={results.get('metrics')} ({results_path})")
180
+ console.print(f"Compare with validators: udcli submission show <submission-id> --compare {results_path}",
181
+ markup=False)
182
+
183
+
184
+ # ------------------------------------------------------------------ submit
185
+
186
+ def _terms_and_fee(client: Api, ch: dict, yes: bool) -> None:
187
+ terms = client.get("/v1/platform")["terms"]
188
+ console.print(f"\n[bold]Publication terms[/bold] ({terms['id']}):\n{terms['text']}\n")
189
+ if not yes and not typer.confirm("Accept the terms and submit?"):
190
+ raise typer.Exit(1)
191
+ fs = ch.get("fee_schedule") or {}
192
+ if fs and ch["fee_rao"] > fs["base_rao"]:
193
+ note = f" It falls back to {tao(fs['base_rao'])} {fs['decay_hours']:g}h after your last upload to this challenge."
194
+ elif fs and fs["max_rao"] > fs["base_rao"]:
195
+ note = f" Uploading again within {fs['decay_hours']:g}h costs up to {tao(fs['max_rao'])}."
196
+ else:
197
+ note = ""
198
+ console.print(f"Submission fee: [bold]{tao(ch['fee_rao'])}[/bold] (charged from your platform balance).{note}")
199
+
200
+
201
+ def _open_challenge(client: Api, challenge: str | None) -> dict:
202
+ ch = pick_challenge(client, challenge)
203
+ if not ch["accepting_submissions"]:
204
+ if ch.get("test_mode") and ch["current_period"] == "contest":
205
+ raise ApiError(403, f"{ch['id']} is in test mode; only whitelisted accounts can submit")
206
+ raise ApiError(409, f"{ch['id']} is in its {ch['current_period']} period; uploads are closed")
207
+ return ch
208
+
209
+
210
+ @app.command("submit")
211
+ def submission_submit(
212
+ hf_repo: str = typer.Option(..., "--hf-repo", help="owner/name on Hugging Face for weights/."),
213
+ path: Path = typer.Option(Path("."), "--path", help="brain-comm submission folder (agent.yaml, brain/, train/, weights/). "
214
+ "weights/ goes to --hf-repo; the rest is uploaded to Unidroid as a hashed ZIP."),
215
+ revision: str = typer.Option("main", "--revision",
216
+ help="Branch, tag or commit; pinned to a commit SHA. Anything other than main "
217
+ "needs --skip-upload: a normal submit always pushes to main."),
218
+ challenge: Optional[str] = typer.Option(None, "--challenge"),
219
+ skip_upload: bool = typer.Option(False, "--skip-upload",
220
+ help="weights/ is already at --revision; do not push it."),
221
+ write_hashes: bool = typer.Option(False, "--write-hashes",
222
+ help="Update agent.yaml's weights hashes without asking."),
223
+ public_repo: bool = typer.Option(False, "--public-repo", help="Create --hf-repo as public (default private)."),
224
+ wallet_name: str = typer.Option("miner", "--wallet", help="Bittensor wallet name."),
225
+ wallet_hotkey: str = typer.Option("default", "--hotkey", help="Hotkey name in that wallet."),
226
+ wallet_path: Optional[str] = typer.Option(None, "--wallet-path"),
227
+ yes: bool = typer.Option(False, "--yes", "-y", help="Accept the publication terms without prompting."),
228
+ ) -> None:
229
+ """Submit a brain-comm folder: hash weights/ into agent.yaml, push weights/ to --hf-repo, pin the
230
+ commit, check it against agent.yaml, then upload the rest as a deterministic ZIP.
231
+
232
+ Pushing uses `hf auth login` / HF_TOKEN locally (never sent). Validators use the token from `udcli auth hf set`.
233
+ """
234
+ if revision != "main" and not skip_upload:
235
+ raise ApiError(400, f"--revision {revision} is only used with --skip-upload; without it, weights/ is "
236
+ "pushed to main and that commit is submitted")
237
+ signer = wallet.hotkey_signer(wallet_name, wallet_hotkey, wallet_path)
238
+ console.print(f"Signing with [bold]--wallet {wallet_name} --hotkey {wallet_hotkey}[/bold] ({signer.ss58})")
239
+ client = api()
240
+ ch = _open_challenge(client, challenge)
241
+ _submit_bundle(client, ch, path, hf_repo, revision, skip_upload=skip_upload,
242
+ write_hashes=write_hashes, private=not public_repo, signer=signer, yes=yes)
243
+
244
+
245
+ def _uploads_dir() -> Path:
246
+ path = config.home() / "uploads"
247
+ path.mkdir(parents=True, exist_ok=True)
248
+ return path
249
+
250
+
251
+ def _refresh_hashes(path: Path, write: bool) -> agent.Report:
252
+ report_ = _check_submission_folder(path)
253
+ local = report_.local
254
+ stale = agent.weights_diff(agent.declared_weights(report_.config), local) if local else []
255
+ missing_map = [p for p in report_.problems if p.startswith("weights must be a mapping")]
256
+ others = [p for p in report_.problems if p not in stale and p not in missing_map]
257
+ if others:
258
+ report(others)
259
+ if stale or missing_map:
260
+ for line in stale:
261
+ console.print(f" {line}", markup=False)
262
+ if not write and not typer.confirm("agent.yaml weights hashes are out of date. Rewrite them?"):
263
+ raise typer.Exit(1)
264
+ new_text = agent.rewrite_weights_block(report_.agent_text, {p: w.sha256 for p, w in local.items()})
265
+ (path / agent.AGENT_YAML).write_bytes(new_text.encode("utf-8"))
266
+ console.print(f"[green]Updated[/green] the weights hashes in {path / agent.AGENT_YAML}.")
267
+ report_ = agent.check(path, hash_weights_too=False)
268
+ report_.local = local
269
+ report(report_.problems + agent.weights_diff(agent.declared_weights(report_.config), local))
270
+ return report_
271
+
272
+
273
+ def _upload_parts(info: dict, archive: Path, state_file: Path) -> list[dict]:
274
+ """Upload the parts, retrying each one. Finished parts are saved in ``state_file``, so a resume skips them."""
275
+ state = json.loads(state_file.read_text(encoding="utf-8")) if state_file.is_file() else {}
276
+ stored = {int(n): etag for n, etag in (state.get("parts") or {}).items()}
277
+
278
+ def save(part: dict) -> None:
279
+ state.setdefault("parts", {})[str(part["part_number"])] = part["etag"]
280
+ state_file.write_text(json.dumps(state), encoding="utf-8")
281
+
282
+ try:
283
+ return uploads.upload_parts(
284
+ archive, info["part_size"], {p["part_number"]: p["url"] for p in info["parts"]},
285
+ stored={n: e for n, e in stored.items() if n in {p["part_number"] for p in info["parts"]}},
286
+ on_part=save, progress=lambda done, total: console.print(f" uploaded part {done}/{total}"))
287
+ except uploads.PartUploadError as exc:
288
+ raise ApiError(0, f"{exc}; resume with `udcli submission resume {info['upload_id']}`") from None
289
+
290
+
291
+ def _finish_upload(client: Api, info: dict, archive: Path, state_file: Path) -> dict:
292
+ parts = _upload_parts(info, archive, state_file)
293
+ result = client.post(f"/v1/submissions/bundles/{info['upload_id']}/complete", json={"parts": parts})
294
+ state_file.unlink(missing_ok=True)
295
+ archive.unlink(missing_ok=True)
296
+ console.print(f"[green]Submitted[/green] #{result['id']} to {result['challenge_id']} as {result['hotkey']}; "
297
+ f"fee {tao(result['fee_rao'])}. Bundle sha256 {result['bundle']['sha256']}. Keep "
298
+ f"{result['hf_repo']}@{result['commit_sha'][:10]} available and unchanged until Review is done.")
299
+ return result
300
+
301
+
302
+ def _announce_deletes(paths: list[str]) -> None:
303
+ console.print(f"[yellow]Removing {len(paths)} file(s) from main that agent.yaml does not declare:[/yellow]")
304
+ for p in paths[:20]:
305
+ console.print(f" {p}", markup=False)
306
+ if len(paths) > 20:
307
+ console.print(f" ... and {len(paths) - 20} more")
308
+
309
+
310
+ def _submit_bundle(client: Api, ch: dict, path: Path, hf_repo: str, revision: str, *, skip_upload: bool,
311
+ write_hashes: bool, private: bool, signer: wallet.Signer, yes: bool) -> None:
312
+ report_ = _refresh_hashes(path, write_hashes)
313
+ declared = agent.declared_weights(report_.config)
314
+ token = hfrepo.local_token()
315
+ # Terms and fee come first: nothing is pushed to Hugging Face until the miner has agreed.
316
+ _terms_and_fee(client, ch, yes)
317
+
318
+ if skip_upload:
319
+ snap = hfrepo.resolve(hf_repo, revision, token)
320
+ else:
321
+ if not token:
322
+ raise ApiError(401, "a Hugging Face write token is needed to push weights/: set HF_TOKEN or "
323
+ "run `hf auth login` (or push yourself and use --skip-upload)")
324
+ try:
325
+ current = hfrepo.resolve(hf_repo, "main", token)
326
+ except hfrepo.HfRepoError:
327
+ current = None
328
+ if current is not None and not hfrepo.weights_problems(current.files, declared, report_.local):
329
+ console.print(f"{hf_repo} main already holds these weights; not pushing.")
330
+ sha = current.commit_sha
331
+ else:
332
+ size = sum(w.size for w in report_.local.values()) / 1e9
333
+ console.print(f"Pushing {len(report_.local)} weights files ({size:.2f} GB) to {hf_repo} "
334
+ f"({'private' if private else 'public'})...")
335
+ files = {p: path / agent.WEIGHTS_DIR / p for p in report_.local}
336
+ sha = hfrepo.push_weights(hf_repo, files, token, private=private, on_delete=_announce_deletes)
337
+ snap = hfrepo.resolve(hf_repo, sha, token)
338
+ console.print(f"Pinned {hf_repo} -> commit [bold]{snap.commit_sha}[/bold] "
339
+ f"({len(snap.files)} files, {snap.total_bytes / 1e9:.2f} GB)")
340
+ report(hfrepo.weights_problems(snap.files, declared, report_.local)
341
+ + checks.check([(f.path, f.size) for f in snap.files], checks.challenge_rules(ch)))
342
+
343
+ agent_yaml = (path / agent.AGENT_YAML).read_bytes()
344
+ archive = _uploads_dir() / f"bundle-{int(time.time())}-{secrets.token_hex(4)}.zip"
345
+ bundle_sha, bundle_size = agent.write_bundle(report_.bundle, archive)
346
+ console.print(f"Bundle: {len(report_.bundle)} files, {bundle_size / 1e6:.2f} MB, sha256 {bundle_sha}")
347
+
348
+ payload = protocol.bundle_submission_payload(
349
+ challenge_id=ch["id"], hotkey=signer.ss58, hf_repo=hf_repo, commit_sha=snap.commit_sha,
350
+ manifest_hash=snap.manifest_hash, bundle_sha256=bundle_sha, bundle_size=bundle_size,
351
+ agent_yaml_sha256=hashlib.sha256(agent_yaml).hexdigest(), weights_digest=protocol.weights_digest(declared),
352
+ nonce=secrets.token_hex(16), timestamp=int(time.time()))
353
+ body = {"payload": payload, "signature": signer.sign(protocol.canonical_json(payload)),
354
+ "agent_yaml": agent_yaml.decode("utf-8")}
355
+ info = client.post("/v1/submissions/bundles", json=body)
356
+ state_file = _uploads_dir() / f"{info['upload_id']}.json"
357
+ state_file.write_text(json.dumps({"upload_id": info["upload_id"], "archive": str(archive),
358
+ "challenge_id": ch["id"]}), encoding="utf-8")
359
+ console.print(f"Upload {info['upload_id']} started; checks passed. Uploading the bundle...")
360
+ _finish_upload(client, info, archive, state_file)
361
+
362
+
363
+ @app.command("resume")
364
+ def submission_resume(upload_id: str) -> None:
365
+ """Finish an interrupted `udcli submission submit` upload (within 24 hours of starting it)."""
366
+ client = api()
367
+ state_file = _uploads_dir() / f"{upload_id}.json"
368
+ if not state_file.is_file():
369
+ raise ApiError(404, f"no local state for {upload_id} in {_uploads_dir()}")
370
+ state = json.loads(state_file.read_text(encoding="utf-8"))
371
+ archive = Path(state["archive"])
372
+ info = client.get(f"/v1/submissions/bundles/{upload_id}")
373
+ if info["status"] != "uploading":
374
+ raise ApiError(409, f"upload {upload_id} is {info['status']}" + (f": {info['detail']}" if info["detail"] else ""))
375
+ if not archive.is_file() or agent.hash_file(archive)[1] != info["bundle_sha256"]:
376
+ raise ApiError(400, f"{archive} is missing or changed; start a new submission")
377
+ _finish_upload(client, info, archive, state_file)
378
+
379
+
380
+ # ------------------------------------------------------------------ list / show
381
+
382
+ def _summary_text(e: Optional[dict]) -> str:
383
+ if not e:
384
+ return "-"
385
+ if e["state"] != "counted":
386
+ return e["state"]
387
+ text = f"#{e['rank']} {e['score']:.4f} ({e['phase']})" if e["score"] is not None else f"waiting ({e['phase']})"
388
+ return text + (f", {e['problems']} failed" if e["problems"] else "")
389
+
390
+
391
+ @app.command("list")
392
+ def submission_list(as_json: bool = typer.Option(False, "--json")) -> None:
393
+ """Your submissions. `udcli submission show <id>` shows every validator's result."""
394
+ rows = api().get("/v1/account/submissions")
395
+ if as_json:
396
+ return emit_json(rows)
397
+ published = any(s["published_repo"] for s in rows)
398
+ table = Table("id", "challenge", "repo", "hotkey", "status", "evaluation", "fee", "submitted",
399
+ *(["published"] if published else []))
400
+ for s in rows:
401
+ table.add_row(str(s["id"]), s["challenge_id"], f"{s['hf_repo']}@{s['commit_sha'][:8]}", s["hotkey"][:10] + "...",
402
+ s["status"] + (f" ({s['status_reason']})" if s["status_reason"] else ""),
403
+ _summary_text(s.get("evaluation")), tao(s["fee_rao"]),
404
+ local_time(s["submitted_at"]), *([s["published_repo"] or ""] if published else []))
405
+ console.print(table)
406
+
407
+
408
+ def _validator_name(r: dict) -> str:
409
+ return r.get("name") or (f"uid {r['uid']}" if r.get("uid") is not None else r["hotkey"][:10] + "...")
410
+
411
+
412
+ def _pct(x: Optional[float]) -> str:
413
+ return "-" if x is None else f"{x * 100:.0f}%"
414
+
415
+
416
+ def _print_comparison(c: compare.Comparison) -> None:
417
+ console.print("\n[bold]Local run vs validators[/bold]")
418
+ if c.config == "different":
419
+ console.print("[yellow]Your run used a different eval config than validators; scores are not comparable.[/yellow]")
420
+ elif c.config == "unknown":
421
+ console.print("Config: no <run>/config/eval.yaml next to the results; cannot check it matched.")
422
+ else:
423
+ console.print("Config: [green]same public eval config as validators[/green]")
424
+ local = "-" if c.local_score is None else f"{c.local_score:.4f}"
425
+ platform = "-" if c.platform_score is None else f"{c.platform_score:.4f}"
426
+ gap = "" if c.gap is None else f" (local {c.gap:+.4f})"
427
+ console.print(f"Score: local {local}, platform score {platform}{gap}", markup=False)
428
+ if c.local_success_rate is not None or c.local_episodes is not None:
429
+ episodes = "-" if c.local_episodes is None else str(c.local_episodes)
430
+ console.print(f"Local run: success rate {_pct(c.local_success_rate)}, {episodes} episodes", markup=False)
431
+ for v in c.validators:
432
+ diff = "" if c.local_score is None else f" ({c.local_score - v.score:+.4f})"
433
+ episodes = "-" if v.episodes is None else str(v.episodes)
434
+ console.print(f" {v.name}: {v.score:.4f}{diff}, success rate {_pct(v.success_rate)}, {episodes} episodes",
435
+ markup=False)
436
+
437
+
438
+ @app.command("show")
439
+ def submission_show(
440
+ submission_id: int,
441
+ compare_path: Optional[Path] = typer.Option(
442
+ None, "--compare", help="results.json from `udcli submission eval` (<run>/out/results.json) to compare."),
443
+ as_json: bool = typer.Option(False, "--json"),
444
+ ) -> None:
445
+ """Every validator's status, score and error on a submission, per phase, with its success rate and episode count."""
446
+ data = api().get(f"/v1/submissions/{submission_id}")
447
+ if as_json:
448
+ return emit_json(data)
449
+ ev = data["evaluation"]
450
+ console.print(f"[bold]#{data['id']}[/bold] {data['hf_repo']}@{data['commit_sha'][:10]} on {ev['challenge_id']} "
451
+ f"[{ev['counted']['state']}] {ev['counted']['detail']}", markup=False)
452
+ if ev.get("note"):
453
+ console.print(f"[yellow]{ev['note']}[/yellow]")
454
+ for p in ev["phases"]:
455
+ score = "-" if p["score"] is None else f"{p['score']:.4f} (rank {p['rank']})"
456
+ running = ", running" if p["current"] else ""
457
+ console.print(f"\n[bold]{p['phase']} evaluation[/bold]{running}: score {score}, {p['validators']} validator(s), "
458
+ f"waiting on {p['pending_validators']}")
459
+ table = Table("validator", "status", "score", "success rate", "episodes", "evaluator", "config", "reported", "error")
460
+ for r in p["results"]:
461
+ evaluator = r.get("evaluator") or {}
462
+ metrics = r.get("metrics") or {}
463
+ cfg = "-" if not r.get("config_sha256") else "current" if r["config_sha256"] == p["config_sha256"] else "earlier"
464
+ table.add_row(_validator_name(r), r["status"], "-" if r["score"] is None else f"{r['score']:.4f}",
465
+ _pct(metrics.get("success_rate")),
466
+ "-" if metrics.get("episodes") is None else str(int(metrics["episodes"])),
467
+ f"{evaluator.get('name', '')} {evaluator.get('version', '')}".strip() or "-", cfg,
468
+ local_time(r["created_at"]), r.get("error") or "")
469
+ if p["results"]:
470
+ console.print(table)
471
+ if not p["detail_visible"] and p["results"]:
472
+ console.print(" Success rate and episodes appear once the private config is revealed.")
473
+ if compare_path:
474
+ public = next((p for p in ev["phases"] if p["phase"] == "public"), None)
475
+ if public is None:
476
+ raise ApiError(400, f"{ev['challenge_id']} has no public evaluation to compare with")
477
+ local = json.loads(compare_path.read_text(encoding="utf-8"))
478
+ _print_comparison(compare.compare(local, public, compare.local_config_sha256(compare_path)))
@@ -0,0 +1,124 @@
1
+ """udcli validator: pending submissions and fetching one with every hash checked."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import shutil
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ import typer
11
+ from rich.table import Table
12
+
13
+ from .. import agent, hfrepo, protocol
14
+ from .._common import api, console, emit_json, group, hash_progress, local_time, report
15
+ from ..http import ApiError
16
+
17
+ app = group("Validator accounts: pending submissions and fetching one with every hash checked.")
18
+
19
+
20
+ @app.command("pending")
21
+ def validator_pending(challenge: Optional[str] = typer.Option(None, "--challenge"),
22
+ as_json: bool = typer.Option(False, "--json")) -> None:
23
+ """Submissions your linked hotkeys have not scored yet in each live challenge's current phase."""
24
+ data = api().get("/v1/validator/submissions", params={"challenge_id": challenge} if challenge else None)
25
+ if as_json:
26
+ return emit_json(data)
27
+ if not data["hotkeys"]:
28
+ console.print("No linked validator hotkey: you can fetch bundles but not miners' read tokens.")
29
+ table = Table("#", "challenge", "phase", "hotkey", "weights", "bundle", "deadline")
30
+ for s in data["items"]:
31
+ table.add_row(str(s["submission_id"]), s["challenge_id"], s["phase"], s["hotkey"][:10],
32
+ f"{s['hf_repo']}@{s['commit_sha'][:8]}",
33
+ f"{s['bundle']['size_bytes'] / 1e6:.1f} MB" if s["bundle"] else "-", local_time(s["deadline"]))
34
+ console.print(table)
35
+
36
+
37
+ def _download_zip(url: str, target: Path) -> str:
38
+ from ..http import raw_client
39
+
40
+ digest = hashlib.sha256()
41
+ with raw_client() as client, client.stream("GET", url) as resp, target.open("wb") as out:
42
+ if resp.status_code >= 400:
43
+ raise ApiError(resp.status_code, "the bundle link was refused or has expired; fetch again")
44
+ for chunk in resp.iter_bytes(1 << 20):
45
+ out.write(chunk)
46
+ digest.update(chunk)
47
+ return digest.hexdigest()
48
+
49
+
50
+ def _unpack(archive: Path, dest: Path, expected: list) -> list[str]:
51
+ """Extract the ZIP and compare every entry with the hashes stored at submission. Returns the problems."""
52
+ import zipfile
53
+
54
+ want = {path: sha for path, sha, _ in expected}
55
+ problems = []
56
+ with zipfile.ZipFile(archive) as zf:
57
+ names = [i.filename for i in zf.infolist()]
58
+ for name in names:
59
+ problem = agent.bundle_entry_problem(name)
60
+ if problem:
61
+ problems.append(problem)
62
+ if problems:
63
+ return problems
64
+ for name in names:
65
+ target = dest / name
66
+ target.parent.mkdir(parents=True, exist_ok=True)
67
+ with zf.open(name) as src, target.open("wb") as out:
68
+ shutil.copyfileobj(src, out)
69
+ if agent.hash_file(target)[1] != want.get(name):
70
+ problems.append(f"{name}: sha256 differs from the stored bundle")
71
+ problems += [f"{p}: missing from the ZIP" for p in sorted(set(want) - set(names))]
72
+ return problems
73
+
74
+
75
+ @app.command("fetch")
76
+ def validator_fetch(
77
+ submission_id: int,
78
+ out: Path = typer.Option(..., "--out", help="Empty folder to assemble the submission in (the brain's /submission)."),
79
+ token: bool = typer.Option(True, "--token/--no-token", help="Ask for the miner's read token (audited)."),
80
+ skip_weights: bool = typer.Option(False, "--skip-weights", help="Fetch and check the ZIP only."),
81
+ ) -> None:
82
+ """Download one submission, check the ZIP, agent.yaml and every weights file against the signed hashes."""
83
+ client = api()
84
+ if out.exists() and any(out.iterdir()):
85
+ raise ApiError(400, f"{out} is not empty")
86
+ out.mkdir(parents=True, exist_ok=True)
87
+ try:
88
+ grant = client.post(f"/v1/validator/submissions/{submission_id}/access", json={"bundle": True, "token": token})
89
+ except ApiError as exc:
90
+ if not token or exc.status != 403:
91
+ raise
92
+ console.print("No linked validator hotkey; fetching without the miner's read token.")
93
+ grant = client.post(f"/v1/validator/submissions/{submission_id}/access", json={"bundle": True, "token": False})
94
+ problems: list[str] = []
95
+ if grant["mode"] == "bundle":
96
+ b = grant["bundle"]
97
+ archive = out.parent / f".submission-{submission_id}.zip"
98
+ try:
99
+ sha = _download_zip(b["url"], archive)
100
+ if sha != b["sha256"] or sha != grant["payload"].get("bundle_sha256"):
101
+ report([f"ZIP sha256 {sha} differs from the signed {b['sha256']}"])
102
+ problems += _unpack(archive, out, b["files"])
103
+ finally:
104
+ archive.unlink(missing_ok=True)
105
+ yaml_sha = agent.hash_file(out / agent.AGENT_YAML)[1]
106
+ if yaml_sha != b["agent_yaml_sha256"]:
107
+ problems.append("agent.yaml: sha256 differs from the signed hash")
108
+ _, cfg = agent.read_agent(out)
109
+ declared = agent.declared_weights(cfg)
110
+ if protocol.weights_digest(declared) != b["weights_digest"]:
111
+ problems.append("agent.yaml weights map does not match the signed weights_digest")
112
+ console.print(f"Bundle: {len(b['files'])} files, sha256 {sha} [green]ok[/green]" if not problems
113
+ else "Bundle: problems found")
114
+ weights_dir = out / agent.WEIGHTS_DIR
115
+ else:
116
+ declared, weights_dir = {}, out
117
+ if not skip_weights:
118
+ console.print(f"Downloading {grant['hf_repo']}@{grant['commit_sha'][:10]}...")
119
+ hfrepo.download(grant["hf_repo"], grant["commit_sha"], grant["hf_token"], weights_dir)
120
+ if grant["mode"] == "bundle":
121
+ problems += agent.weights_diff(declared, agent.hash_weights(out, hash_progress))
122
+ report(problems)
123
+ console.print(f"[green]Ready[/green]: {out} matches what the miner signed. Run `brain-comm check {out}` in the "
124
+ "brain image next.")