unidroid-cli 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unidroid_cli/__init__.py +1 -0
- unidroid_cli/__main__.py +3 -0
- unidroid_cli/_common.py +74 -0
- unidroid_cli/agent.py +392 -0
- unidroid_cli/bench.py +143 -0
- unidroid_cli/checks.py +42 -0
- unidroid_cli/cli.py +105 -0
- unidroid_cli/commands/__init__.py +1 -0
- unidroid_cli/commands/account.py +118 -0
- unidroid_cli/commands/auth.py +155 -0
- unidroid_cli/commands/bench.py +132 -0
- unidroid_cli/commands/challenge.py +97 -0
- unidroid_cli/commands/dataset.py +546 -0
- unidroid_cli/commands/submission.py +478 -0
- unidroid_cli/commands/validator.py +124 -0
- unidroid_cli/compare.py +64 -0
- unidroid_cli/config.py +85 -0
- unidroid_cli/dataset.py +414 -0
- unidroid_cli/dataset_sample.py +594 -0
- unidroid_cli/hfrepo.py +202 -0
- unidroid_cli/http.py +66 -0
- unidroid_cli/protocol.py +80 -0
- unidroid_cli/safe_extract.py +58 -0
- unidroid_cli/uploads.py +76 -0
- unidroid_cli/wallet.py +62 -0
- unidroid_cli-0.3.0.dist-info/METADATA +196 -0
- unidroid_cli-0.3.0.dist-info/RECORD +29 -0
- unidroid_cli-0.3.0.dist-info/WHEEL +4 -0
- unidroid_cli-0.3.0.dist-info/entry_points.txt +2 -0
unidroid_cli/cli.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""udcli - the Unidroid command line."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
import typer
|
|
9
|
+
from typer.core import TyperGroup
|
|
10
|
+
|
|
11
|
+
from . import __version__, agent, bench, dataset, hfrepo, wallet
|
|
12
|
+
from ._common import CONTEXT, STATE, console, err
|
|
13
|
+
from .commands import account, auth, challenge, submission, validator
|
|
14
|
+
from .commands import bench as bench_cmd
|
|
15
|
+
from .commands import dataset as dataset_cmd
|
|
16
|
+
from .commands.submission import docker_command
|
|
17
|
+
from .http import ApiError
|
|
18
|
+
|
|
19
|
+
__all__ = ["app", "console", "docker_command", "err", "main"]
|
|
20
|
+
|
|
21
|
+
_AUTH_FLAGS = {"--api-key": "api_key", "--api-url": "api_url"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class _AuthGroup(TyperGroup):
|
|
25
|
+
"""Accept --api-key and --api-url anywhere, including after a subcommand."""
|
|
26
|
+
|
|
27
|
+
def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
|
|
28
|
+
if ctx.parent is None:
|
|
29
|
+
STATE["api_url"] = None
|
|
30
|
+
STATE["api_key"] = None
|
|
31
|
+
args = _take_auth_flags(args)
|
|
32
|
+
return super().parse_args(ctx, args)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _take_auth_flags(args: list[str]) -> list[str]:
|
|
36
|
+
cleaned: list[str] = []
|
|
37
|
+
i = 0
|
|
38
|
+
while i < len(args):
|
|
39
|
+
arg = args[i]
|
|
40
|
+
name = _AUTH_FLAGS.get(arg)
|
|
41
|
+
if name is None:
|
|
42
|
+
for flag, flag_name in _AUTH_FLAGS.items():
|
|
43
|
+
prefix = flag + "="
|
|
44
|
+
if arg.startswith(prefix):
|
|
45
|
+
STATE[flag_name] = arg[len(prefix):]
|
|
46
|
+
break
|
|
47
|
+
else:
|
|
48
|
+
cleaned.append(arg)
|
|
49
|
+
i += 1
|
|
50
|
+
continue
|
|
51
|
+
if i + 1 >= len(args):
|
|
52
|
+
raise click.UsageError(f"Option {arg!r} requires a value.")
|
|
53
|
+
STATE[name] = args[i + 1]
|
|
54
|
+
i += 2
|
|
55
|
+
return cleaned
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
app = typer.Typer(cls=_AuthGroup, help="Unidroid: robotics model challenges on Bittensor.", no_args_is_help=True,
|
|
59
|
+
pretty_exceptions_enable=False, context_settings=CONTEXT)
|
|
60
|
+
app.add_typer(auth.app, name="auth")
|
|
61
|
+
app.add_typer(account.app, name="account")
|
|
62
|
+
app.add_typer(challenge.app, name="challenge")
|
|
63
|
+
app.add_typer(submission.app, name="submission")
|
|
64
|
+
app.add_typer(bench_cmd.app, name="bench")
|
|
65
|
+
app.add_typer(dataset_cmd.app, name="dataset")
|
|
66
|
+
app.add_typer(validator.app, name="validator")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@app.callback()
|
|
70
|
+
def _global(
|
|
71
|
+
api_url: Optional[str] = typer.Option(
|
|
72
|
+
None, "--api-url", envvar="UNIDROID_API_URL",
|
|
73
|
+
help="API URL for this run. Can be placed on any command."),
|
|
74
|
+
api_key: Optional[str] = typer.Option(
|
|
75
|
+
None, "--api-key", envvar="UNIDROID_API_KEY",
|
|
76
|
+
help="API key for this run (unidroid_...). Use another account, or skip `udcli auth login`. "
|
|
77
|
+
"Can be placed on any command."),
|
|
78
|
+
) -> None:
|
|
79
|
+
# Explicit flags are applied in _AuthGroup before parsing. These values are the env vars
|
|
80
|
+
# when the flags were omitted, and must not replace a flag already taken from the command.
|
|
81
|
+
if api_url and not STATE["api_url"]:
|
|
82
|
+
STATE["api_url"] = api_url
|
|
83
|
+
if api_key and not STATE["api_key"]:
|
|
84
|
+
STATE["api_key"] = api_key
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@app.command()
|
|
88
|
+
def version() -> None:
|
|
89
|
+
"""Show the udcli version."""
|
|
90
|
+
console.print(f"udcli {__version__}")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def main() -> None:
|
|
94
|
+
try:
|
|
95
|
+
app()
|
|
96
|
+
except ApiError as exc:
|
|
97
|
+
err.print(f"[red]Error:[/red] {exc.detail}")
|
|
98
|
+
raise SystemExit(1) from None
|
|
99
|
+
except (wallet.WalletError, bench.BenchError, dataset.DatasetError, hfrepo.HfRepoError) as exc:
|
|
100
|
+
err.print(f"[red]Error:[/red] {exc}")
|
|
101
|
+
raise SystemExit(1) from None
|
|
102
|
+
except agent.AgentError as exc:
|
|
103
|
+
for problem in exc.problems:
|
|
104
|
+
err.print(f"[red]x[/red] {problem}", markup=True, highlight=False)
|
|
105
|
+
raise SystemExit(1) from None
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""udcli command groups, one module per `udcli <group>`."""
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""udcli account: platform balance, deposits and coldkeys."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
from urllib.parse import quote
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
from .. import protocol, wallet
|
|
12
|
+
from .._common import YES, api, confirm, console, emit_json, group, local_time, tao
|
|
13
|
+
|
|
14
|
+
app = group("Balance, ledger, TAO deposits and registered coldkeys.")
|
|
15
|
+
coldkey_app = group("Coldkeys whose transfers credit your balance.")
|
|
16
|
+
app.add_typer(coldkey_app, name="coldkey")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@app.command("balance")
|
|
20
|
+
def account_balance() -> None:
|
|
21
|
+
"""Platform balance used for submission fees, plus how to deposit TAO."""
|
|
22
|
+
data = api().get("/v1/account/balance")
|
|
23
|
+
console.print(f"Balance: [bold]{tao(data['balance_rao'])}[/bold]")
|
|
24
|
+
if not data.get("deposit_available"):
|
|
25
|
+
console.print("Deposits are currently unavailable.")
|
|
26
|
+
return
|
|
27
|
+
console.print(f"Network: {data.get('network')}")
|
|
28
|
+
console.print(f"Send TAO to: [bold]{data['platform_address']}[/bold]")
|
|
29
|
+
if data.get("coldkeys"):
|
|
30
|
+
console.print("From (registered coldkeys):")
|
|
31
|
+
for ck in data["coldkeys"]:
|
|
32
|
+
console.print(f" {ck}")
|
|
33
|
+
else:
|
|
34
|
+
console.print("Register a coldkey first: [bold]udcli account coldkey register --wallet <name>[/bold]")
|
|
35
|
+
for tier in data.get("bonus_tiers") or []:
|
|
36
|
+
console.print(f"Bonus: deposit ≥ {tier['min_tao']} TAO → {tier['multiplier']}x credit")
|
|
37
|
+
console.print("Then claim with the extrinsic id:")
|
|
38
|
+
console.print(" udcli account claim <block-index> # e.g. 9092713-0006")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@app.command("ledger")
|
|
42
|
+
def account_ledger(as_json: bool = typer.Option(False, "--json")) -> None:
|
|
43
|
+
"""Deposits, bonuses, fees and refunds."""
|
|
44
|
+
rows = api().get("/v1/account/ledger")
|
|
45
|
+
if as_json:
|
|
46
|
+
return emit_json(rows)
|
|
47
|
+
table = Table("time", "kind", "amount", "balance after", "note")
|
|
48
|
+
for r in rows:
|
|
49
|
+
table.add_row(local_time(r["created_at"]), r["kind"], tao(r["amount_rao"]),
|
|
50
|
+
tao(r.get("balance_after_rao", 0)), r["note"])
|
|
51
|
+
console.print(table)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@app.command("claim")
|
|
55
|
+
def account_claim(reference: str = typer.Argument(..., help="Extrinsic id, e.g. 9092713-0006.")) -> None:
|
|
56
|
+
"""Credit one transfer from your coldkey to the platform address by extrinsic id."""
|
|
57
|
+
data = api().post("/v1/account/deposits/claim", json={"reference": reference})
|
|
58
|
+
_print_deposit_result(data)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _print_deposit_result(data: dict) -> None:
|
|
62
|
+
if data.get("confirmed"):
|
|
63
|
+
msg = f"Credited {tao(data['credited_rao'])} from {data['confirmed']} transfer(s)"
|
|
64
|
+
if data.get("bonus_rao"):
|
|
65
|
+
msg += f" (incl. {tao(data['bonus_rao'])} bonus)"
|
|
66
|
+
console.print(msg)
|
|
67
|
+
elif data.get("message"):
|
|
68
|
+
console.print(data["message"])
|
|
69
|
+
else:
|
|
70
|
+
console.print("No new deposits found")
|
|
71
|
+
console.print(f"Balance: [bold]{tao(data['balance_rao'])}[/bold]")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# ------------------------------------------------------------------ coldkeys
|
|
75
|
+
|
|
76
|
+
@coldkey_app.command("list")
|
|
77
|
+
def coldkey_list(as_json: bool = typer.Option(False, "--json")) -> None:
|
|
78
|
+
"""Coldkeys registered to this account."""
|
|
79
|
+
rows = api().get("/v1/account/coldkeys")
|
|
80
|
+
if as_json:
|
|
81
|
+
return emit_json(rows)
|
|
82
|
+
table = Table("coldkey", "registered")
|
|
83
|
+
for r in rows:
|
|
84
|
+
table.add_row(r["coldkey"], local_time(r.get("created_at")))
|
|
85
|
+
console.print(table)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@coldkey_app.command("register")
|
|
89
|
+
def coldkey_register(
|
|
90
|
+
wallet_name: str = typer.Option(..., "--wallet", help="Bittensor wallet name."),
|
|
91
|
+
wallet_path: Optional[str] = typer.Option(None, "--wallet-path"),
|
|
92
|
+
) -> None:
|
|
93
|
+
"""Prove ownership of a coldkey (signs a one-time nonce) so its transfers credit your balance."""
|
|
94
|
+
client = api()
|
|
95
|
+
signer = wallet.coldkey_signer(wallet_name, wallet_path)
|
|
96
|
+
nonce = client.get("/v1/account/coldkey/nonce")
|
|
97
|
+
msg = protocol.coldkey_message(user_id=nonce["user_id"], coldkey=signer.ss58, nonce=nonce["nonce"])
|
|
98
|
+
client.post("/v1/account/coldkeys", json={"coldkey": signer.ss58, "nonce": nonce["nonce"],
|
|
99
|
+
"signature": signer.sign(msg)})
|
|
100
|
+
console.print(f"[green]Coldkey {signer.ss58} registered.[/green]")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@coldkey_app.command("remove")
|
|
104
|
+
def coldkey_remove(
|
|
105
|
+
coldkey: Optional[str] = typer.Argument(None, help="SS58 address to unregister."),
|
|
106
|
+
wallet_name: Optional[str] = typer.Option(None, "--wallet", help="Bittensor wallet; uses its coldkey address."),
|
|
107
|
+
wallet_path: Optional[str] = typer.Option(None, "--wallet-path"),
|
|
108
|
+
yes: bool = YES,
|
|
109
|
+
) -> None:
|
|
110
|
+
"""Unregister a coldkey. Later transfers from it no longer credit this account."""
|
|
111
|
+
if coldkey and wallet_name:
|
|
112
|
+
raise typer.BadParameter("Pass the SS58 address or --wallet, not both.")
|
|
113
|
+
ss58 = wallet.coldkey_address(wallet_name, wallet_path) if wallet_name else coldkey
|
|
114
|
+
if not ss58:
|
|
115
|
+
raise typer.BadParameter("Pass the coldkey SS58 address, or --wallet.")
|
|
116
|
+
confirm(f"Unregister coldkey {ss58}? Its later transfers will not credit your balance.", yes)
|
|
117
|
+
api().delete(f"/v1/account/coldkeys/{quote(ss58, safe='')}")
|
|
118
|
+
console.print(f"[green]Coldkey {ss58} removed.[/green]")
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""udcli auth: log in, API keys and the Hugging Face read token."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from rich.table import Table
|
|
9
|
+
|
|
10
|
+
from .. import config
|
|
11
|
+
from .._common import STATE, YES, api, confirm, console, group, local_time
|
|
12
|
+
from ..http import Api, ApiError
|
|
13
|
+
|
|
14
|
+
app = group("Log in and out, see who you are, manage API keys and your Hugging Face read token.")
|
|
15
|
+
keys_app = group("API keys for udcli and scripts.")
|
|
16
|
+
hf_app = group("Hugging Face read token the platform and validators use to download your model.")
|
|
17
|
+
app.add_typer(keys_app, name="keys")
|
|
18
|
+
app.add_typer(hf_app, name="hf")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@app.command()
|
|
22
|
+
def login(key: Optional[str] = typer.Option(None, "--key", help="API key (unidroid_...); prompted if omitted.")) -> None:
|
|
23
|
+
"""Store an API key created at unidroid.ai/account/api-keys."""
|
|
24
|
+
key = key or STATE["api_key"] or typer.prompt("API key", hide_input=True)
|
|
25
|
+
if not key.startswith("unidroid_"):
|
|
26
|
+
raise ApiError(400, "API keys start with 'unidroid_'")
|
|
27
|
+
me = Api(STATE["api_url"], key).get("/v1/auth/me")
|
|
28
|
+
data = {"api_key": key}
|
|
29
|
+
if STATE["api_url"]:
|
|
30
|
+
data["api_url"] = STATE["api_url"]
|
|
31
|
+
path = config.save_credentials(data)
|
|
32
|
+
console.print(f"[green]Logged in[/green] as {me['email']} ({me['role']}). Saved to {path}")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@app.command()
|
|
36
|
+
def logout() -> None:
|
|
37
|
+
"""Forget stored credentials."""
|
|
38
|
+
config.clear_credentials()
|
|
39
|
+
console.print("Logged out.")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _print_account(me: dict, coldkeys: list | None = None) -> None:
|
|
43
|
+
"""Labeled account summary, in the same shape as npcli account whoami."""
|
|
44
|
+
console.print("[bold]Current user[/bold]")
|
|
45
|
+
lines: list[tuple[str, str | None]] = [
|
|
46
|
+
("Name", me.get("fullname")),
|
|
47
|
+
("Email", me.get("email")),
|
|
48
|
+
("Role", me.get("role")),
|
|
49
|
+
("Status", me.get("status")),
|
|
50
|
+
]
|
|
51
|
+
if me.get("auth"):
|
|
52
|
+
lines.append(("Auth", me["auth"]))
|
|
53
|
+
if me.get("email_verified") is not None:
|
|
54
|
+
lines.append(("Email verified", "yes" if me["email_verified"] else "no"))
|
|
55
|
+
if me.get("created_at"):
|
|
56
|
+
lines.append(("Joined", local_time(me["created_at"])))
|
|
57
|
+
for label, key in (("GitHub", "github_url"), ("Discord", "discord_handle"), ("Website", "website_url")):
|
|
58
|
+
if me.get(key):
|
|
59
|
+
lines.append((label, me[key]))
|
|
60
|
+
for label, value in lines:
|
|
61
|
+
if value:
|
|
62
|
+
console.print(f" {label}: {value}")
|
|
63
|
+
hotkeys = me.get("hotkeys") or []
|
|
64
|
+
if hotkeys:
|
|
65
|
+
console.print(" Validator hotkeys:")
|
|
66
|
+
for hk in hotkeys:
|
|
67
|
+
console.print(f" {hk}")
|
|
68
|
+
if coldkeys is not None:
|
|
69
|
+
if coldkeys:
|
|
70
|
+
console.print(" Coldkeys:")
|
|
71
|
+
for row in coldkeys:
|
|
72
|
+
console.print(f" {row['coldkey'] if isinstance(row, dict) else row}")
|
|
73
|
+
else:
|
|
74
|
+
console.print(" Coldkeys: none")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@app.command()
|
|
78
|
+
def whoami() -> None:
|
|
79
|
+
"""Show the authenticated account."""
|
|
80
|
+
client = api()
|
|
81
|
+
me = client.get("/v1/auth/me")
|
|
82
|
+
coldkeys = None
|
|
83
|
+
try:
|
|
84
|
+
listed = client.get("/v1/account/coldkeys")
|
|
85
|
+
except ApiError as exc:
|
|
86
|
+
if exc.status not in (401, 403):
|
|
87
|
+
raise
|
|
88
|
+
else:
|
|
89
|
+
if isinstance(listed, list):
|
|
90
|
+
coldkeys = listed
|
|
91
|
+
_print_account(me, coldkeys)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# ------------------------------------------------------------------ API keys
|
|
95
|
+
|
|
96
|
+
@keys_app.command("list")
|
|
97
|
+
def keys_list() -> None:
|
|
98
|
+
"""Your API keys and whether they are revoked."""
|
|
99
|
+
table = Table("id", "name", "prefix", "created", "expires", "revoked")
|
|
100
|
+
for k in api().get("/v1/account/api-keys"):
|
|
101
|
+
table.add_row(str(k["id"]), k["name"], k["prefix"] + "...",
|
|
102
|
+
local_time(k["created_at"]), local_time(k["expires_at"]), "yes" if k["revoked"] else "")
|
|
103
|
+
console.print(table)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@keys_app.command("create")
|
|
107
|
+
def keys_create(name: str = typer.Option(...),
|
|
108
|
+
expires_in_days: Optional[int] = typer.Option(None)) -> None:
|
|
109
|
+
"""Create a key. It is shown once."""
|
|
110
|
+
key = api().post("/v1/account/api-keys", json={"name": name, "expires_in_days": expires_in_days})
|
|
111
|
+
console.print(f"[green]Created[/green] {key['name']}:\n{key['key']}\n"
|
|
112
|
+
"[yellow]Store it now; it cannot be shown again.[/yellow]")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@keys_app.command("revoke")
|
|
116
|
+
def keys_revoke(key_id: int) -> None:
|
|
117
|
+
"""Stop a key from authenticating. It stays in the list until deleted."""
|
|
118
|
+
api().post(f"/v1/account/api-keys/{key_id}/revoke")
|
|
119
|
+
console.print(f"Revoked API key {key_id}.")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@keys_app.command("delete")
|
|
123
|
+
def keys_delete(key_id: int, yes: bool = YES) -> None:
|
|
124
|
+
"""Revoke a key if it is still active, then remove it."""
|
|
125
|
+
confirm(f"Delete API key {key_id}? Anything using it stops working.", yes)
|
|
126
|
+
api().delete(f"/v1/account/api-keys/{key_id}")
|
|
127
|
+
console.print(f"Deleted API key {key_id}.")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# ------------------------------------------------------------------ Hugging Face read token
|
|
131
|
+
|
|
132
|
+
@hf_app.command("set")
|
|
133
|
+
def hf_set(token: Optional[str] = typer.Option(None, "--token", help="Prompted (hidden) if omitted.")) -> None:
|
|
134
|
+
"""Save a Hugging Face read token. Use a fine-grained, read-only token scoped to your submitted repos:
|
|
135
|
+
it is passed to validators so they can download your model at the pinned commit."""
|
|
136
|
+
token = token or typer.prompt("Hugging Face read token", hide_input=True)
|
|
137
|
+
data = api().put("/v1/account/hf-token", json={"token": token})
|
|
138
|
+
console.print(f"[green]Saved[/green] token for Hugging Face user {data['hf_username']}.")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@hf_app.command("status")
|
|
142
|
+
def hf_status() -> None:
|
|
143
|
+
"""Whether a token is saved, and for which Hugging Face user."""
|
|
144
|
+
data = api().get("/v1/account/hf-token")
|
|
145
|
+
if data["set"]:
|
|
146
|
+
console.print(f"Token set for {data['hf_username']} (updated {local_time(data['updated_at'])}).")
|
|
147
|
+
else:
|
|
148
|
+
console.print("No token saved (only public repos can be submitted).")
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@hf_app.command("remove")
|
|
152
|
+
def hf_remove() -> None:
|
|
153
|
+
"""Delete the saved token."""
|
|
154
|
+
api().delete("/v1/account/hf-token")
|
|
155
|
+
console.print("Hugging Face token removed.")
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""udcli bench: BenchHub benchmarks and the local bundle cache."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
from .. import bench, config
|
|
12
|
+
from .._common import api, console, emit_json, group, local_time
|
|
13
|
+
from ..http import ApiError
|
|
14
|
+
|
|
15
|
+
app = group("BenchHub: download, upload, link EnvHub environments and cache benchmark bundles.")
|
|
16
|
+
cache_app = group("Local BenchHub cache.")
|
|
17
|
+
app.add_typer(cache_app, name="cache")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _access_line(v: dict) -> str:
|
|
21
|
+
a = v["access"]
|
|
22
|
+
colour = "green" if a["can_download"] else "red"
|
|
23
|
+
return f"[{colour}]{'can download now' if a['can_download'] else 'cannot download: ' + a['reason']}[/{colour}]"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@app.command("list")
|
|
27
|
+
def bench_list(category: Optional[str] = None,
|
|
28
|
+
robot_type: Optional[str] = typer.Option(None, help="robot type"),
|
|
29
|
+
visibility: Optional[str] = typer.Option(None, help="public|private"),
|
|
30
|
+
search: Optional[str] = None, as_json: bool = typer.Option(False, "--json")) -> None:
|
|
31
|
+
"""List BenchHub benchmarks."""
|
|
32
|
+
params = {k: v for k, v in {"category": category, "robot_type": robot_type,
|
|
33
|
+
"visibility": visibility, "search": search}.items() if v}
|
|
34
|
+
rows = api(require_key=False).get("/v1/benchmarks", params=params)
|
|
35
|
+
if as_json:
|
|
36
|
+
return emit_json(rows)
|
|
37
|
+
table = Table("id", "category", "robot", "source", "visibility", "state", "you")
|
|
38
|
+
for b in rows:
|
|
39
|
+
robot = " ".join(x for x in (b.get("robot"), b.get("robot_type")) if x) or "-"
|
|
40
|
+
table.add_row(b["id"], b["category"], robot,
|
|
41
|
+
bench.source_label(b), b["visibility"], b["state"],
|
|
42
|
+
"yes" if b["access"]["can_download"] else "no")
|
|
43
|
+
console.print(table)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@app.command("info")
|
|
47
|
+
def bench_info(bench_id: str) -> None:
|
|
48
|
+
"""Visibility, active window and whether you can download it now."""
|
|
49
|
+
client = api(require_key=False)
|
|
50
|
+
v = client.get(f"/v1/benchmarks/{bench_id}")
|
|
51
|
+
console.print(f"[bold]{v['id']}[/bold] {v['visibility']} state={v['state']}")
|
|
52
|
+
if v["visibility"] == "private":
|
|
53
|
+
console.print(f" active window: {local_time(v['active_from'])} -> {local_time(v['active_until'])}")
|
|
54
|
+
console.print(f" rule: {v['access']['rule']}")
|
|
55
|
+
console.print(f" {_access_line(v)}")
|
|
56
|
+
console.print(f" source: {bench.source_label(v)}", markup=False)
|
|
57
|
+
console.print(f" sha256 {v['sha256']} size {v['size_bytes'] / 1e6:.1f} MB")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@app.command("download")
|
|
61
|
+
def bench_download(refs: list[str] = typer.Argument(..., help="benchmark id"),
|
|
62
|
+
force: bool = typer.Option(False, "--force"),
|
|
63
|
+
cache_dir: Optional[Path] = typer.Option(None, "--cache-dir")) -> None:
|
|
64
|
+
"""Download (and cache) bundles. Private bundles only during their active window."""
|
|
65
|
+
client = api()
|
|
66
|
+
root = config.bench_cache_dir(str(cache_dir) if cache_dir else None)
|
|
67
|
+
for ref in refs:
|
|
68
|
+
path = bench.download(client, ref, root, force=force)
|
|
69
|
+
console.print(f"[green]{ref}[/green] -> {path}")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@app.command("upload")
|
|
73
|
+
def bench_upload(folder: Path, visibility: str = typer.Option(..., help="public|private"),
|
|
74
|
+
thumbnail: Optional[Path] = typer.Option(None, "--thumbnail")) -> None:
|
|
75
|
+
"""Upload a bundle folder containing manifest.yaml (team/admin only). An admin reviews it."""
|
|
76
|
+
if visibility not in ("public", "private"):
|
|
77
|
+
raise ApiError(400, "--visibility must be public or private")
|
|
78
|
+
v = bench.upload(api(), folder, visibility, thumbnail,
|
|
79
|
+
progress=lambda done, total: console.print(f" uploaded part {done}/{total}"))
|
|
80
|
+
console.print(f"[green]Uploaded[/green] {v['id']}: {v['state']}"
|
|
81
|
+
+ (f" ({v['review_note']})" if v.get("review_note") else ""))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@app.command("link")
|
|
85
|
+
def bench_link(envhub_env_id: str = typer.Argument(..., help="Nepher EnvHub environment id"),
|
|
86
|
+
visibility: str = typer.Option(..., help="public|private")) -> None:
|
|
87
|
+
"""Register a Nepher EnvHub environment as a benchmark (admin only).
|
|
88
|
+
|
|
89
|
+
The bundle stays on EnvHub. Its manifest.yaml becomes the BenchHub record, and BenchHub pins
|
|
90
|
+
the current sha256. You need no Nepher API key.
|
|
91
|
+
"""
|
|
92
|
+
if visibility not in ("public", "private"):
|
|
93
|
+
raise ApiError(400, "--visibility must be public or private")
|
|
94
|
+
v = bench.link(api(), envhub_env_id, visibility)
|
|
95
|
+
console.print(f"[green]Linked[/green] {v['id']} -> EnvHub {envhub_env_id}: "
|
|
96
|
+
f"{v['state']}, sha256 {v['sha256'][:12]}")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@app.command("status")
|
|
100
|
+
def bench_status(bench_id: Optional[str] = typer.Argument(None)) -> None:
|
|
101
|
+
"""Review status of your uploads."""
|
|
102
|
+
rows = [r for r in api().get("/v1/account/benchmark-uploads") if not bench_id or r["id"] == bench_id]
|
|
103
|
+
table = Table("benchmark", "source", "requested", "state", "window", "note")
|
|
104
|
+
for r in rows:
|
|
105
|
+
window = f"{local_time(r['active_from'])} -> {local_time(r['active_until'])}" if r["active_from"] else ""
|
|
106
|
+
table.add_row(r["id"], bench.source_label(r), r["visibility"], r["state"], window, r["review_note"])
|
|
107
|
+
console.print(table)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@cache_app.command("list")
|
|
111
|
+
def cache_list(cache_dir: Optional[Path] = typer.Option(None, "--cache-dir")) -> None:
|
|
112
|
+
"""Cached bundles with their size and path."""
|
|
113
|
+
table = Table("benchmark", "size", "path")
|
|
114
|
+
for e in bench.cache_entries(config.bench_cache_dir(str(cache_dir) if cache_dir else None)):
|
|
115
|
+
table.add_row(e["benchmark_id"], f"{e['disk_bytes'] / 1e6:.1f} MB", e["path"])
|
|
116
|
+
console.print(table)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@cache_app.command("info")
|
|
120
|
+
def cache_info(cache_dir: Optional[Path] = typer.Option(None, "--cache-dir")) -> None:
|
|
121
|
+
"""Cache location, bundle count and total size."""
|
|
122
|
+
root = config.bench_cache_dir(str(cache_dir) if cache_dir else None)
|
|
123
|
+
entries = bench.cache_entries(root)
|
|
124
|
+
console.print(f"{root}: {len(entries)} bundles, {sum(e['disk_bytes'] for e in entries) / 1e9:.2f} GB")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@cache_app.command("clear")
|
|
128
|
+
def cache_clear(bench_id: Optional[str] = typer.Argument(None),
|
|
129
|
+
cache_dir: Optional[Path] = typer.Option(None, "--cache-dir")) -> None:
|
|
130
|
+
"""Remove one cached benchmark, or all of them."""
|
|
131
|
+
removed = bench.cache_clear(config.bench_cache_dir(str(cache_dir) if cache_dir else None), bench_id)
|
|
132
|
+
console.print(f"Removed {removed} cached benchmark(s).")
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""udcli challenge: browse challenges and leaderboards."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from rich.table import Table
|
|
9
|
+
|
|
10
|
+
from .._common import api, console, emit_json, group, local_time, tao
|
|
11
|
+
|
|
12
|
+
app = group("Browse challenges and leaderboards.")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@app.command("list")
|
|
16
|
+
def challenge_list(all_: bool = typer.Option(False, "--all", help="Include upcoming and finished."),
|
|
17
|
+
as_json: bool = typer.Option(False, "--json")) -> None:
|
|
18
|
+
"""Challenges that are live now (or all of them with --all)."""
|
|
19
|
+
rows = api(require_key=False).get("/v1/challenges", params=None if all_ else {"status": "active"})
|
|
20
|
+
if as_json:
|
|
21
|
+
return emit_json(rows)
|
|
22
|
+
table = Table("id", "name", "status", "brain", "robot", "period", "uploads close", "fee")
|
|
23
|
+
for c in rows:
|
|
24
|
+
table.add_row(c["id"], c["name"], c.get("status", ""), brain_label(c.get("brain")),
|
|
25
|
+
robots_label(c.get("robots")), c["current_period"], local_time(c["eval_start"]), tao(c["fee_rao"]))
|
|
26
|
+
console.print(table)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def brain_label(brain: Optional[dict]) -> str:
|
|
30
|
+
if not brain:
|
|
31
|
+
return "-"
|
|
32
|
+
kind = brain.get("model_type_name") or str(brain.get("model_type") or "").upper()
|
|
33
|
+
return f"{brain['name']} ({kind})"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def robots_label(robots: Optional[list]) -> str:
|
|
37
|
+
return ", ".join(" ".join(x for x in (r.get("robot"), r.get("robot_type")) if x) for r in robots or []) or "-"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def eval_summary(text: Optional[str], revealed: bool = True) -> str:
|
|
41
|
+
if not revealed:
|
|
42
|
+
return "hidden until Reward"
|
|
43
|
+
if not text:
|
|
44
|
+
return "none"
|
|
45
|
+
lines = [line for line in text.splitlines() if line.strip()]
|
|
46
|
+
return f"YAML ({len(lines)} lines)"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@app.command("show")
|
|
50
|
+
def challenge_show(challenge_id: str, as_json: bool = typer.Option(False, "--json")) -> None:
|
|
51
|
+
"""Schedule, brain, fee, eval configs and benchmarks of one challenge."""
|
|
52
|
+
c = api(require_key=False).get(f"/v1/challenges/{challenge_id}")
|
|
53
|
+
if as_json:
|
|
54
|
+
return emit_json(c)
|
|
55
|
+
console.print(f"[bold]{c['name']}[/bold] ({c['id']}) - period: {c['current_period']}")
|
|
56
|
+
brain = c.get("brain")
|
|
57
|
+
console.print(f" brain {brain_label(brain)}" + (f" - {brain['id']}" if brain else ""))
|
|
58
|
+
if brain:
|
|
59
|
+
dim = f", {brain['action_dim']}-D" if brain.get("action_dim") else ""
|
|
60
|
+
console.print(f" checkpoint {brain.get('checkpoint_format') or '-'}; base {brain.get('base_checkpoint') or '-'}; "
|
|
61
|
+
f"actions {brain.get('action_space') or '-'}{dim}")
|
|
62
|
+
console.print(" submit with a submission folder: udcli submission submit --path ... --hf-repo ...")
|
|
63
|
+
console.print(f" robot {robots_label(c.get('robots'))}")
|
|
64
|
+
console.print(f" simulation {c.get('simulation') or '-'}")
|
|
65
|
+
for f in ("contest_start", "eval_start", "review_start", "reward_start", "reward_end"):
|
|
66
|
+
console.print(f" {f:<14} {local_time(c[f])}")
|
|
67
|
+
fee = c.get("fee_schedule") or {}
|
|
68
|
+
fee_line = f" fee {tao(c['fee_rao'])} for your next upload"
|
|
69
|
+
if fee:
|
|
70
|
+
fee_line += (f" (base {tao(fee['base_rao'])}, max {tao(fee['max_rao'])}, "
|
|
71
|
+
f"back to base {fee['decay_hours']:g}h after your last upload to this challenge)")
|
|
72
|
+
console.print(f"{fee_line}; podium split {c['podium_split']}")
|
|
73
|
+
if c.get("test_mode"):
|
|
74
|
+
console.print(" test mode on; only whitelisted accounts can submit")
|
|
75
|
+
console.print(f" task repo {c.get('task_repo', '-')}")
|
|
76
|
+
console.print(f" eval repo {c.get('eval_repo', '-')}")
|
|
77
|
+
console.print(f" public eval {eval_summary(c.get('public_eval'))}")
|
|
78
|
+
console.print(f" private eval {eval_summary(c.get('private_eval'), c.get('private_eval_revealed', False))}")
|
|
79
|
+
console.print(f" private sha {c.get('private_eval_sha256', '-')}")
|
|
80
|
+
for b in c.get("benchmarks", []):
|
|
81
|
+
console.print(f" benchmark {b.get('visibility', '-')}: {b['id']} - {b['access']['rule']}",
|
|
82
|
+
markup=False)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@app.command("leaderboard")
|
|
86
|
+
def challenge_leaderboard(challenge_id: str, phase: Optional[str] = typer.Option(None, help="public|private"),
|
|
87
|
+
as_json: bool = typer.Option(False, "--json")) -> None:
|
|
88
|
+
"""Ranked scores for the public or private phase."""
|
|
89
|
+
data = api(require_key=False).get(f"/v1/challenges/{challenge_id}/leaderboard",
|
|
90
|
+
params={"phase": phase} if phase else None)
|
|
91
|
+
if as_json:
|
|
92
|
+
return emit_json(data)
|
|
93
|
+
table = Table("rank", "hotkey", "repo", "score", "validators", "submitted", title=f"{data['phase']} leaderboard")
|
|
94
|
+
for r in data["rows"]:
|
|
95
|
+
table.add_row(str(r["rank"]), r["hotkey"][:12] + "...", f"{r['hf_repo']}@{r['commit_sha'][:8]}",
|
|
96
|
+
f"{r['score']:.4f}", str(r["validators"]), local_time(r["submitted_at"]))
|
|
97
|
+
console.print(table)
|