unidroid-cli 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unidroid_cli/__init__.py +1 -0
- unidroid_cli/__main__.py +3 -0
- unidroid_cli/_common.py +74 -0
- unidroid_cli/agent.py +392 -0
- unidroid_cli/bench.py +143 -0
- unidroid_cli/checks.py +42 -0
- unidroid_cli/cli.py +105 -0
- unidroid_cli/commands/__init__.py +1 -0
- unidroid_cli/commands/account.py +118 -0
- unidroid_cli/commands/auth.py +155 -0
- unidroid_cli/commands/bench.py +132 -0
- unidroid_cli/commands/challenge.py +97 -0
- unidroid_cli/commands/dataset.py +546 -0
- unidroid_cli/commands/submission.py +478 -0
- unidroid_cli/commands/validator.py +124 -0
- unidroid_cli/compare.py +64 -0
- unidroid_cli/config.py +85 -0
- unidroid_cli/dataset.py +414 -0
- unidroid_cli/dataset_sample.py +594 -0
- unidroid_cli/hfrepo.py +202 -0
- unidroid_cli/http.py +66 -0
- unidroid_cli/protocol.py +80 -0
- unidroid_cli/safe_extract.py +58 -0
- unidroid_cli/uploads.py +76 -0
- unidroid_cli/wallet.py +62 -0
- unidroid_cli-0.3.0.dist-info/METADATA +196 -0
- unidroid_cli-0.3.0.dist-info/RECORD +29 -0
- unidroid_cli-0.3.0.dist-info/WHEEL +4 -0
- unidroid_cli-0.3.0.dist-info/entry_points.txt +2 -0
unidroid_cli/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.0"
|
unidroid_cli/__main__.py
ADDED
unidroid_cli/_common.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""State and helpers shared by every udcli command group."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from rich.console import Console
|
|
10
|
+
|
|
11
|
+
from .http import Api, ApiError
|
|
12
|
+
|
|
13
|
+
console = Console()
|
|
14
|
+
err = Console(stderr=True)
|
|
15
|
+
|
|
16
|
+
STATE: dict = {"api_url": None, "api_key": None}
|
|
17
|
+
|
|
18
|
+
CONTEXT = {"help_option_names": ["-h", "--help"]}
|
|
19
|
+
|
|
20
|
+
TAO = 1_000_000_000
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def group(help: str) -> typer.Typer:
|
|
24
|
+
"""A command group: `udcli <group>` with no subcommand prints its help."""
|
|
25
|
+
return typer.Typer(help=help, no_args_is_help=True, context_settings=CONTEXT)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def api(require_key: bool = True) -> Api:
|
|
29
|
+
return Api(STATE["api_url"], STATE["api_key"], require_key=require_key)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
YES = typer.Option(False, "--yes", "-y", help="Do not ask to confirm.")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def confirm(question: str, yes: bool) -> None:
|
|
36
|
+
"""Ask before something that cannot be undone; exits when the answer is no."""
|
|
37
|
+
if not yes:
|
|
38
|
+
typer.confirm(question, abort=True)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def tao(rao: int) -> str:
|
|
42
|
+
return f"{rao / TAO:.4f} TAO"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def local_time(iso: str | None) -> str:
|
|
46
|
+
if not iso:
|
|
47
|
+
return "-"
|
|
48
|
+
return datetime.fromisoformat(iso.replace("Z", "+00:00")).astimezone().strftime("%Y-%m-%d %H:%M %Z")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def emit_json(data) -> None:
|
|
52
|
+
console.print_json(json.dumps(data))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def pick_challenge(client: Api, challenge_id: str | None) -> dict:
|
|
56
|
+
if challenge_id:
|
|
57
|
+
return client.get(f"/v1/challenges/{challenge_id}")
|
|
58
|
+
open_ = [c for c in client.get("/v1/challenges", params={"status": "active"}) if c["accepting_submissions"]]
|
|
59
|
+
if len(open_) == 1:
|
|
60
|
+
return client.get(f"/v1/challenges/{open_[0]['id']}")
|
|
61
|
+
ids = ", ".join(c["id"] for c in open_) or "none"
|
|
62
|
+
raise ApiError(400, f"pass --challenge (challenges accepting submissions: {ids})")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def report(problems: list[str]) -> None:
|
|
66
|
+
if problems:
|
|
67
|
+
for p in problems:
|
|
68
|
+
err.print(f"[red]x[/red] {p}")
|
|
69
|
+
raise typer.Exit(1)
|
|
70
|
+
console.print("[green]All checks passed.[/green]")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def hash_progress(path: str, done: int, total: int) -> None:
|
|
74
|
+
console.print(f" hashing {done}/{total} weights/{path}", markup=False, highlight=False)
|
unidroid_cli/agent.py
ADDED
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
"""brain-comm submissions (agent.yaml, brain/, train/, weights/): weight hashes, checks and the bundle ZIP.
|
|
2
|
+
|
|
3
|
+
The rules mirror PROTOCOL.md section 4 and the backend's ``services/bundle.py``.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import re
|
|
10
|
+
import unicodedata
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from fnmatch import fnmatch
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from . import protocol
|
|
18
|
+
|
|
19
|
+
AGENT_YAML = "agent.yaml"
|
|
20
|
+
WEIGHTS_DIR = "weights"
|
|
21
|
+
# Never hashed, never uploaded.
|
|
22
|
+
IGNORED_NAMES = (".DS_Store", "Thumbs.db", "*.swp", "*.swo", "*~")
|
|
23
|
+
IGNORED_DIRS = (".cache", ".git", "__pycache__")
|
|
24
|
+
ROOT_FILE_PREFIXES = ("README", "LICENSE", "NOTICE")
|
|
25
|
+
ROOT_DIRS = ("brain", "train", "tests", "docs")
|
|
26
|
+
REQUIRED_DIRS = ("brain", "train")
|
|
27
|
+
REFUSED_FILES = (
|
|
28
|
+
"*.pyc", "*.pyo", "*.pkl", "*.pickle", "*.joblib", "*.dill",
|
|
29
|
+
"*.so", "*.pyd", "*.dll", "*.dylib", "*.exe",
|
|
30
|
+
"*.zip", "*.tar", "*.gz", "*.tgz", "*.bz2", "*.xz", "*.7z", "*.rar", "*.whl", "*.egg", "*.jar",
|
|
31
|
+
)
|
|
32
|
+
# Left out of the bundle without a warning.
|
|
33
|
+
BUNDLE_JUNK = ("*.pyc", "*.pyo", ".DS_Store", "Thumbs.db", "*.swp", "*.swo", "*~")
|
|
34
|
+
BUNDLE_MAX_BYTES = 256 * 1024 * 1024
|
|
35
|
+
BUNDLE_MAX_FILES = 2000
|
|
36
|
+
BUNDLE_MAX_FILE_BYTES = 32 * 1024 * 1024
|
|
37
|
+
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
|
|
38
|
+
_UNSAFE_KEY = re.compile(r"[:#\"'\x00-\x1f\x7f]")
|
|
39
|
+
_WEIGHTS_LINE = re.compile(r"^weights\s*:(.*)$")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class AgentError(Exception):
|
|
43
|
+
def __init__(self, problems: list[str]):
|
|
44
|
+
super().__init__("; ".join(problems))
|
|
45
|
+
self.problems = problems
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class LocalWeight:
|
|
50
|
+
path: str # relative to weights/
|
|
51
|
+
size: int
|
|
52
|
+
sha256: str
|
|
53
|
+
blob_id: str # git blob SHA-1, what Hugging Face lists for files without LFS
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# ---------------------------------------------------------------- paths
|
|
57
|
+
|
|
58
|
+
def path_problem(path: str) -> str | None:
|
|
59
|
+
if not path or len(path) > 400:
|
|
60
|
+
return "empty or too long path"
|
|
61
|
+
if "\\" in path or path.startswith("/"):
|
|
62
|
+
return f"{path}: use relative paths with '/' separators"
|
|
63
|
+
if unicodedata.normalize("NFC", path) != path:
|
|
64
|
+
return f"{path}: path is not NFC-normalized"
|
|
65
|
+
if any(ord(ch) < 32 or ord(ch) == 127 for ch in path):
|
|
66
|
+
return f"{path}: control characters in path"
|
|
67
|
+
if any(part in ("", ".", "..") for part in path.split("/")):
|
|
68
|
+
return f"{path}: empty, '.' or '..' path segment"
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def weights_path_problem(path: str) -> str | None:
|
|
73
|
+
problem = path_problem(path)
|
|
74
|
+
if problem:
|
|
75
|
+
return problem
|
|
76
|
+
if _UNSAFE_KEY.search(path) or path != path.strip():
|
|
77
|
+
return f"{path}: weights paths cannot hold ':', '#', quotes or surrounding spaces"
|
|
78
|
+
if any(part.startswith(".") for part in path.split("/")):
|
|
79
|
+
return f"{path}: dot files are not allowed in weights/ (brain-comm check refuses them)"
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def bundle_entry_problem(path: str) -> str | None:
|
|
84
|
+
problem = path_problem(path)
|
|
85
|
+
if problem:
|
|
86
|
+
return problem
|
|
87
|
+
parts = path.split("/")
|
|
88
|
+
if any(part.startswith(".") for part in parts):
|
|
89
|
+
return f"{path}: dot files are not allowed in the bundle"
|
|
90
|
+
if parts[0] == WEIGHTS_DIR:
|
|
91
|
+
return f"{path}: weights/ goes to Hugging Face, not into the bundle"
|
|
92
|
+
if len(parts) == 1:
|
|
93
|
+
if path != AGENT_YAML and not path.upper().startswith(ROOT_FILE_PREFIXES):
|
|
94
|
+
return f"{path}: only agent.yaml, README, LICENSE and NOTICE files may sit at the root"
|
|
95
|
+
elif parts[0] not in ROOT_DIRS:
|
|
96
|
+
return f"{path}: top-level directory must be one of {', '.join(d + '/' for d in ROOT_DIRS)}"
|
|
97
|
+
if any(fnmatch(parts[-1].lower(), pattern) for pattern in REFUSED_FILES):
|
|
98
|
+
return f"{path}: compiled code, pickles and archives are not allowed in the bundle"
|
|
99
|
+
return None
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _ignored(rel_parts: tuple[str, ...]) -> bool:
|
|
103
|
+
return any(p in IGNORED_DIRS for p in rel_parts[:-1]) or any(fnmatch(rel_parts[-1], pat) for pat in IGNORED_NAMES)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# ---------------------------------------------------------------- weights
|
|
107
|
+
|
|
108
|
+
def weight_files(root: Path) -> tuple[list[tuple[str, Path]], list[str]]:
|
|
109
|
+
"""(relative path, file) for every file under weights/ that is hashed, and the problems found."""
|
|
110
|
+
weights = root / WEIGHTS_DIR
|
|
111
|
+
files, problems = [], []
|
|
112
|
+
if not weights.is_dir():
|
|
113
|
+
return [], ["missing directory: weights/"]
|
|
114
|
+
for path in sorted(weights.rglob("*")):
|
|
115
|
+
rel = path.relative_to(weights)
|
|
116
|
+
if _ignored(rel.parts) or path.is_dir():
|
|
117
|
+
continue
|
|
118
|
+
name = rel.as_posix()
|
|
119
|
+
if path.is_symlink():
|
|
120
|
+
problems.append(f"weights/{name}: symlinks are not allowed")
|
|
121
|
+
continue
|
|
122
|
+
problem = weights_path_problem(name)
|
|
123
|
+
if problem:
|
|
124
|
+
problems.append(f"weights/{problem}")
|
|
125
|
+
continue
|
|
126
|
+
files.append((name, path))
|
|
127
|
+
lowered = [n.lower() for n, _ in files]
|
|
128
|
+
if len(set(lowered)) != len(lowered):
|
|
129
|
+
problems.append("weights/: two paths differ only by case")
|
|
130
|
+
return files, problems
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def hash_file(path: Path) -> tuple[int, str, str]:
|
|
134
|
+
size = path.stat().st_size
|
|
135
|
+
sha256 = hashlib.sha256()
|
|
136
|
+
sha1 = hashlib.sha1(b"blob %d\0" % size) # noqa: S324 - git's blob id
|
|
137
|
+
with path.open("rb") as fh:
|
|
138
|
+
for chunk in iter(lambda: fh.read(8 << 20), b""):
|
|
139
|
+
sha256.update(chunk)
|
|
140
|
+
sha1.update(chunk)
|
|
141
|
+
return size, sha256.hexdigest(), sha1.hexdigest()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def hash_weights(root: Path, progress: Callable[[str, int, int], None] = lambda path, done, total: None,
|
|
145
|
+
) -> dict[str, LocalWeight]:
|
|
146
|
+
"""Stream-hash every weights file. Raises AgentError when weights/ holds files that cannot be submitted."""
|
|
147
|
+
files, problems = weight_files(root)
|
|
148
|
+
if problems:
|
|
149
|
+
raise AgentError(problems)
|
|
150
|
+
out: dict[str, LocalWeight] = {}
|
|
151
|
+
for i, (name, path) in enumerate(files, start=1):
|
|
152
|
+
progress(name, i, len(files))
|
|
153
|
+
size, sha256, blob = hash_file(path)
|
|
154
|
+
out[name] = LocalWeight(name, size, sha256, blob)
|
|
155
|
+
return out
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# ---------------------------------------------------------------- agent.yaml
|
|
159
|
+
|
|
160
|
+
def load_yaml(text: str) -> dict[str, Any]:
|
|
161
|
+
"""brain-comm's ``yaml_min.load_yaml``: flat and one-level maps of scalars."""
|
|
162
|
+
lines = []
|
|
163
|
+
for number, raw in enumerate(text.splitlines(), start=1):
|
|
164
|
+
stripped = raw.split("#", 1)[0].rstrip()
|
|
165
|
+
if not stripped.strip():
|
|
166
|
+
continue
|
|
167
|
+
indent = len(stripped) - len(stripped.lstrip(" "))
|
|
168
|
+
if "\t" in raw[:indent]:
|
|
169
|
+
raise ValueError(f"tabs are not allowed in indentation (line {number})")
|
|
170
|
+
lines.append((number, indent, stripped.strip()))
|
|
171
|
+
result, index = _parse_map(lines, 0, 0)
|
|
172
|
+
if index != len(lines):
|
|
173
|
+
raise ValueError(f"unexpected content at line {lines[index][0]}")
|
|
174
|
+
return result
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _parse_map(lines, index: int, indent: int):
|
|
178
|
+
mapping: dict[str, Any] = {}
|
|
179
|
+
while index < len(lines):
|
|
180
|
+
number, level, content = lines[index]
|
|
181
|
+
if level < indent:
|
|
182
|
+
break
|
|
183
|
+
if level > indent:
|
|
184
|
+
raise ValueError(f"unexpected indent on line {number}")
|
|
185
|
+
if content == "{}":
|
|
186
|
+
raise ValueError(f"a bare mapping is not a document (line {number})")
|
|
187
|
+
key, _, remainder = content.partition(":")
|
|
188
|
+
key, remainder = key.strip(), remainder.strip()
|
|
189
|
+
if not key:
|
|
190
|
+
raise ValueError(f"missing key on line {number}")
|
|
191
|
+
index += 1
|
|
192
|
+
if remainder == "":
|
|
193
|
+
if index >= len(lines) or lines[index][1] <= level:
|
|
194
|
+
mapping[key] = {}
|
|
195
|
+
continue
|
|
196
|
+
mapping[key], index = _parse_map(lines, index, lines[index][1])
|
|
197
|
+
continue
|
|
198
|
+
mapping[key] = {} if remainder == "{}" else _scalar(remainder, number)
|
|
199
|
+
return mapping, index
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _scalar(text: str, line: int):
|
|
203
|
+
if text in ("null", "~", "Null", "NULL"):
|
|
204
|
+
return None
|
|
205
|
+
if text in ("true", "True"):
|
|
206
|
+
return True
|
|
207
|
+
if text in ("false", "False"):
|
|
208
|
+
return False
|
|
209
|
+
if len(text) >= 2 and text[0] == text[-1] and text[0] in ("'", '"'):
|
|
210
|
+
return text[1:-1]
|
|
211
|
+
if text[:1] in "{[":
|
|
212
|
+
raise ValueError(f"unsupported value on line {line}: {text}")
|
|
213
|
+
if re.fullmatch(r"-?\d+", text):
|
|
214
|
+
return int(text)
|
|
215
|
+
if re.fullmatch(r"-?\d+\.\d+", text):
|
|
216
|
+
return float(text)
|
|
217
|
+
return text
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def read_agent(root: Path) -> tuple[str, dict]:
|
|
221
|
+
path = root / AGENT_YAML
|
|
222
|
+
if not path.is_file():
|
|
223
|
+
raise AgentError([f"missing {AGENT_YAML}"])
|
|
224
|
+
text = path.read_bytes().decode("utf-8")
|
|
225
|
+
try:
|
|
226
|
+
return text, load_yaml(text)
|
|
227
|
+
except ValueError as exc:
|
|
228
|
+
raise AgentError([f"{AGENT_YAML}: {exc}"]) from exc
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def schema_problems(config: dict) -> list[str]:
|
|
232
|
+
"""brain-comm check's schema rules, plus the bundle's rules for weights paths."""
|
|
233
|
+
problems = []
|
|
234
|
+
entry = config.get("entry")
|
|
235
|
+
if not isinstance(entry, str) or entry.count(":") != 1 or entry.startswith(":") or entry.endswith(":"):
|
|
236
|
+
problems.append("entry must be 'module:Class'")
|
|
237
|
+
for key in ("horizon", "action_dim"):
|
|
238
|
+
value = config.get(key)
|
|
239
|
+
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
|
240
|
+
problems.append(f"{key} must be an integer >= 1")
|
|
241
|
+
memory = config.get("max_gpu_mem_gb")
|
|
242
|
+
if memory is not None and (isinstance(memory, bool) or not isinstance(memory, (int, float)) or memory <= 0):
|
|
243
|
+
problems.append("max_gpu_mem_gb must be a positive number when set")
|
|
244
|
+
weights = config.get("weights")
|
|
245
|
+
if not isinstance(weights, dict):
|
|
246
|
+
problems.append("weights must be a mapping of relative path to sha256 (run `udcli submission hash`)")
|
|
247
|
+
return problems
|
|
248
|
+
for rel, digest in weights.items():
|
|
249
|
+
problem = weights_path_problem(str(rel))
|
|
250
|
+
if problem:
|
|
251
|
+
problems.append(f"weights: {problem}")
|
|
252
|
+
if not _HEX64.match(str(digest).lower()):
|
|
253
|
+
problems.append(f"weights digest for {rel} is not 64 hex characters")
|
|
254
|
+
return problems
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def declared_weights(config: dict) -> dict[str, str]:
|
|
258
|
+
weights = config.get("weights")
|
|
259
|
+
return {str(k): str(v).lower() for k, v in weights.items()} if isinstance(weights, dict) else {}
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _digest_text(digest: str) -> str:
|
|
263
|
+
# The minimal parser reads an all-digit value as an integer; quote it so it stays a string.
|
|
264
|
+
return f'"{digest}"' if digest.isdigit() else digest
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def rewrite_weights_block(text: str, weights: dict[str, str]) -> str:
|
|
268
|
+
"""Replace only the top-level ``weights:`` block with sorted ``path: sha256`` lines. Everything else stays."""
|
|
269
|
+
newline = "\r\n" if "\r\n" in text else "\n"
|
|
270
|
+
lines = text.splitlines()
|
|
271
|
+
block = ["weights:"] + [f" {path}: {_digest_text(digest)}" for path, digest in sorted(weights.items())]
|
|
272
|
+
start = next((i for i, line in enumerate(lines) if _WEIGHTS_LINE.match(line)), None)
|
|
273
|
+
if start is None:
|
|
274
|
+
out = lines + block
|
|
275
|
+
else:
|
|
276
|
+
end = start + 1
|
|
277
|
+
last = start
|
|
278
|
+
while end < len(lines):
|
|
279
|
+
line = lines[end]
|
|
280
|
+
stripped = line.split("#", 1)[0].strip()
|
|
281
|
+
if stripped and not line.startswith((" ", "\t")):
|
|
282
|
+
break
|
|
283
|
+
if stripped:
|
|
284
|
+
last = end
|
|
285
|
+
end += 1
|
|
286
|
+
# Comments or blank lines right after the block belong to what follows.
|
|
287
|
+
out = lines[:start] + block + lines[last + 1:]
|
|
288
|
+
return newline.join(out) + newline
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def weights_diff(declared: dict[str, str], local: dict[str, LocalWeight]) -> list[str]:
|
|
292
|
+
problems = []
|
|
293
|
+
for path in sorted(set(local) - set(declared)):
|
|
294
|
+
problems.append(f"weights/{path} has no hash in agent.yaml")
|
|
295
|
+
for path in sorted(set(declared) - set(local)):
|
|
296
|
+
problems.append(f"weights/{path} is listed in agent.yaml but missing")
|
|
297
|
+
for path in sorted(set(declared) & set(local)):
|
|
298
|
+
if declared[path] != local[path].sha256:
|
|
299
|
+
problems.append(f"sha256 mismatch for weights/{path}")
|
|
300
|
+
return problems
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
# ---------------------------------------------------------------- the bundle
|
|
304
|
+
|
|
305
|
+
def bundle_files(root: Path) -> tuple[list[tuple[str, Path]], list[str], list[str]]:
|
|
306
|
+
"""(archive path, file) for the bundle, the problems that stop a submission, and what was left out."""
|
|
307
|
+
files, problems, skipped = [], [], []
|
|
308
|
+
for path in sorted(root.rglob("*")):
|
|
309
|
+
rel = path.relative_to(root)
|
|
310
|
+
parts = rel.parts
|
|
311
|
+
if parts[0] == WEIGHTS_DIR or path.is_dir():
|
|
312
|
+
continue
|
|
313
|
+
name = rel.as_posix()
|
|
314
|
+
if len(parts) == 1 and (name.startswith(".") or (name != AGENT_YAML and not name.upper().startswith(
|
|
315
|
+
ROOT_FILE_PREFIXES))):
|
|
316
|
+
skipped.append(name)
|
|
317
|
+
continue
|
|
318
|
+
if len(parts) > 1 and (parts[0] not in ROOT_DIRS):
|
|
319
|
+
if parts[0] not in IGNORED_DIRS and not parts[0].startswith("."):
|
|
320
|
+
skipped.append(name)
|
|
321
|
+
continue
|
|
322
|
+
if any(p in IGNORED_DIRS for p in parts) or any(fnmatch(parts[-1], pat) for pat in BUNDLE_JUNK):
|
|
323
|
+
continue
|
|
324
|
+
if path.is_symlink():
|
|
325
|
+
problems.append(f"{name}: symlinks are not allowed in the bundle")
|
|
326
|
+
continue
|
|
327
|
+
problem = bundle_entry_problem(name)
|
|
328
|
+
if problem:
|
|
329
|
+
problems.append(problem)
|
|
330
|
+
continue
|
|
331
|
+
size = path.stat().st_size
|
|
332
|
+
if size > BUNDLE_MAX_FILE_BYTES:
|
|
333
|
+
problems.append(f"{name}: {size} bytes exceeds the per-file limit of {BUNDLE_MAX_FILE_BYTES}")
|
|
334
|
+
files.append((name, path))
|
|
335
|
+
names = [n for n, _ in files]
|
|
336
|
+
lowered = [n.lower() for n in names]
|
|
337
|
+
if len(set(lowered)) != len(lowered):
|
|
338
|
+
problems.append("two bundle files differ only by case")
|
|
339
|
+
if AGENT_YAML not in names:
|
|
340
|
+
problems.append(f"missing {AGENT_YAML}")
|
|
341
|
+
for required in REQUIRED_DIRS:
|
|
342
|
+
if not any(n.startswith(required + "/") for n in names):
|
|
343
|
+
problems.append(f"{required}/ must hold at least one file")
|
|
344
|
+
if len(files) > BUNDLE_MAX_FILES:
|
|
345
|
+
problems.append(f"{len(files)} bundle files exceeds the limit of {BUNDLE_MAX_FILES}")
|
|
346
|
+
total = sum(p.stat().st_size for _, p in files)
|
|
347
|
+
if total > BUNDLE_MAX_BYTES:
|
|
348
|
+
problems.append(f"the bundle holds {total} bytes; the limit is {BUNDLE_MAX_BYTES}")
|
|
349
|
+
return files, problems, skipped
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def write_bundle(files: list[tuple[str, Path]], out: Path) -> tuple[str, int]:
|
|
353
|
+
"""Write the deterministic ZIP. Returns its sha256 and size."""
|
|
354
|
+
with out.open("wb") as fh:
|
|
355
|
+
protocol.write_bundle_zip(fh, files)
|
|
356
|
+
digest = hashlib.sha256()
|
|
357
|
+
with out.open("rb") as fh:
|
|
358
|
+
for chunk in iter(lambda: fh.read(8 << 20), b""):
|
|
359
|
+
digest.update(chunk)
|
|
360
|
+
return digest.hexdigest(), out.stat().st_size
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
@dataclass
|
|
364
|
+
class Report:
|
|
365
|
+
config: dict
|
|
366
|
+
agent_text: str
|
|
367
|
+
local: dict[str, LocalWeight]
|
|
368
|
+
bundle: list[tuple[str, Path]]
|
|
369
|
+
problems: list[str]
|
|
370
|
+
skipped: list[str]
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def check(root: Path, *, hash_weights_too: bool = True,
|
|
374
|
+
progress: Callable[[str, int, int], None] = lambda path, done, total: None) -> Report:
|
|
375
|
+
"""Every offline check of a submission folder. Problems are collected, not raised."""
|
|
376
|
+
problems: list[str] = []
|
|
377
|
+
try:
|
|
378
|
+
text, config = read_agent(root)
|
|
379
|
+
except AgentError as exc:
|
|
380
|
+
text, config = "", {}
|
|
381
|
+
problems += exc.problems
|
|
382
|
+
if config:
|
|
383
|
+
problems += schema_problems(config)
|
|
384
|
+
local: dict[str, LocalWeight] = {}
|
|
385
|
+
files, weight_problems = weight_files(root)
|
|
386
|
+
problems += weight_problems
|
|
387
|
+
if hash_weights_too and not weight_problems:
|
|
388
|
+
local = hash_weights(root, progress)
|
|
389
|
+
if config and isinstance(config.get("weights"), dict):
|
|
390
|
+
problems += weights_diff(declared_weights(config), local)
|
|
391
|
+
bundle, bundle_problems, skipped = bundle_files(root)
|
|
392
|
+
return Report(config, text, local, bundle, problems + bundle_problems, skipped)
|
unidroid_cli/bench.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""BenchHub client: cached downloads (sha256-verified), multipart uploads and Nepher EnvHub links."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import shutil
|
|
8
|
+
import tempfile
|
|
9
|
+
import zipfile
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import yaml
|
|
13
|
+
|
|
14
|
+
from . import uploads
|
|
15
|
+
from .safe_extract import UnsafeArchive, safe_extract
|
|
16
|
+
from .http import Api, raw_client
|
|
17
|
+
|
|
18
|
+
MANIFEST = "manifest.yaml"
|
|
19
|
+
REQUIRED_KEYS = ("id", "name", "category", "format", "description")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class BenchError(Exception):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def source_label(v: dict) -> str:
|
|
27
|
+
"""R2, EnvHub <env id>, or an R2 copy of an EnvHub environment. Hidden EnvHub ids are only sent to insiders."""
|
|
28
|
+
source = v.get("source") or {"kind": "hosted"}
|
|
29
|
+
env_id = source.get("envhub_env_id")
|
|
30
|
+
if source.get("kind") == "envhub":
|
|
31
|
+
return f"EnvHub {env_id}" if env_id else "EnvHub"
|
|
32
|
+
return f"R2 (copy of {env_id})" if env_id else "R2"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _sha256_file(path: Path) -> str:
|
|
36
|
+
digest = hashlib.sha256()
|
|
37
|
+
with path.open("rb") as fh:
|
|
38
|
+
for chunk in iter(lambda: fh.read(8 << 20), b""):
|
|
39
|
+
digest.update(chunk)
|
|
40
|
+
return digest.hexdigest()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def download(api: Api, bench_id: str, cache_dir: Path, force: bool = False) -> Path:
|
|
44
|
+
"""Download and extract a bundle; returns the extracted directory."""
|
|
45
|
+
info = api.get(f"/v1/benchmarks/{bench_id}/download")
|
|
46
|
+
target = cache_dir / bench_id
|
|
47
|
+
meta_path = target / "meta.json"
|
|
48
|
+
if not force and meta_path.is_file():
|
|
49
|
+
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
|
50
|
+
if meta.get("sha256") == info["sha256"] and (target / "bundle").is_dir():
|
|
51
|
+
return target / "bundle"
|
|
52
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
53
|
+
tmp = target / "bundle.zip.part"
|
|
54
|
+
digest = hashlib.sha256()
|
|
55
|
+
with raw_client() as client, client.stream("GET", info["url"]) as resp:
|
|
56
|
+
if resp.status_code >= 400:
|
|
57
|
+
raise BenchError(f"download failed with HTTP {resp.status_code}")
|
|
58
|
+
with tmp.open("wb") as fh:
|
|
59
|
+
for chunk in resp.iter_bytes(8 << 20):
|
|
60
|
+
digest.update(chunk)
|
|
61
|
+
fh.write(chunk)
|
|
62
|
+
if digest.hexdigest() != info["sha256"]:
|
|
63
|
+
tmp.unlink(missing_ok=True)
|
|
64
|
+
raise BenchError("sha256 mismatch: the downloaded bundle is corrupted")
|
|
65
|
+
archive = target / "bundle.zip"
|
|
66
|
+
tmp.replace(archive)
|
|
67
|
+
shutil.rmtree(target / "bundle", ignore_errors=True)
|
|
68
|
+
try:
|
|
69
|
+
safe_extract(archive, target / "bundle")
|
|
70
|
+
except UnsafeArchive as exc:
|
|
71
|
+
raise BenchError(str(exc)) from None
|
|
72
|
+
meta = {k: info[k] for k in ("benchmark_id", "sha256", "size_bytes")}
|
|
73
|
+
meta["source"] = info.get("source", "hosted")
|
|
74
|
+
meta_path.write_text(json.dumps(meta, indent=2), encoding="utf-8")
|
|
75
|
+
return target / "bundle"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def read_manifest(folder: Path) -> dict:
|
|
79
|
+
return read_manifest_file(folder / MANIFEST)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def read_manifest_file(path: Path) -> dict:
|
|
83
|
+
if path.is_dir():
|
|
84
|
+
path = path / MANIFEST
|
|
85
|
+
if not path.is_file():
|
|
86
|
+
raise BenchError(f"{path} not found")
|
|
87
|
+
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
88
|
+
missing = [k for k in REQUIRED_KEYS if k not in data]
|
|
89
|
+
if missing:
|
|
90
|
+
raise BenchError(f"{path.name} is missing: {', '.join(missing)}")
|
|
91
|
+
return json.loads(json.dumps(data, default=str))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def link(api: Api, env_id: str, visibility: str) -> dict:
|
|
95
|
+
"""Register a Nepher EnvHub environment as a benchmark. The server reads its manifest.yaml and pins the sha256."""
|
|
96
|
+
return api.post("/v1/benchmarks/links", json={"envhub_env_id": env_id, "visibility": visibility})
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def make_zip(folder: Path, out: Path, thumbnail: Path | None = None) -> None:
|
|
100
|
+
with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
|
101
|
+
for path in sorted(folder.rglob("*")):
|
|
102
|
+
if path.is_file():
|
|
103
|
+
zf.write(path, path.relative_to(folder).as_posix())
|
|
104
|
+
if thumbnail:
|
|
105
|
+
zf.write(thumbnail, f"thumbnail{thumbnail.suffix.lower()}")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def upload(api: Api, folder: Path, visibility: str, thumbnail: Path | None = None,
|
|
109
|
+
progress=lambda done, total: None) -> dict:
|
|
110
|
+
manifest = read_manifest(folder)
|
|
111
|
+
if thumbnail:
|
|
112
|
+
manifest["thumbnail"] = f"thumbnail{thumbnail.suffix.lower()}"
|
|
113
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
114
|
+
archive = Path(tmpdir) / "bundle.zip"
|
|
115
|
+
make_zip(folder, archive, thumbnail)
|
|
116
|
+
size = archive.stat().st_size
|
|
117
|
+
info = api.post("/v1/benchmarks/uploads", json={
|
|
118
|
+
"manifest": manifest, "sha256": _sha256_file(archive), "size_bytes": size, "visibility": visibility})
|
|
119
|
+
try:
|
|
120
|
+
parts = uploads.upload_parts(archive, info["part_size"], {p["part_number"]: p["url"] for p in info["parts"]},
|
|
121
|
+
progress=progress)
|
|
122
|
+
except uploads.PartUploadError as exc:
|
|
123
|
+
raise BenchError(str(exc)) from None
|
|
124
|
+
return api.post(f"/v1/benchmarks/uploads/{info['benchmark_id']}/complete", json={"parts": parts})
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def cache_entries(cache_dir: Path) -> list[dict]:
|
|
128
|
+
out = []
|
|
129
|
+
for meta in sorted(cache_dir.glob("*/meta.json")):
|
|
130
|
+
data = json.loads(meta.read_text(encoding="utf-8"))
|
|
131
|
+
size = sum(p.stat().st_size for p in meta.parent.rglob("*") if p.is_file())
|
|
132
|
+
out.append({**data, "path": str(meta.parent / "bundle"), "disk_bytes": size})
|
|
133
|
+
return out
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def cache_clear(cache_dir: Path, bench_id: str | None = None) -> int:
|
|
137
|
+
targets = [cache_dir / bench_id] if bench_id else [p for p in cache_dir.glob("*") if p.is_dir()]
|
|
138
|
+
removed = 0
|
|
139
|
+
for target in targets:
|
|
140
|
+
if target.exists():
|
|
141
|
+
shutil.rmtree(target)
|
|
142
|
+
removed += 1
|
|
143
|
+
return removed
|
unidroid_cli/checks.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Pre-submission checks.
|
|
2
|
+
|
|
3
|
+
The defaults reject pickle files. Callers can pass extra rules::
|
|
4
|
+
|
|
5
|
+
{"required_files": ["*.safetensors"],
|
|
6
|
+
"forbidden_files": ["*.pkl", "*.pt"],
|
|
7
|
+
"max_total_bytes": 20000000000}
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from fnmatch import fnmatch
|
|
13
|
+
|
|
14
|
+
PICKLE_FILES = ["*.pkl", "*.pickle", "*.joblib", "*.dill"]
|
|
15
|
+
# Refused by the backend unless the challenge's brain has submission_code.
|
|
16
|
+
CODE_FILES = ["*.py", "*.pyc", "*.pyo", "*.pyd", "*.so", "*.dll", "*.dylib", "*.exe", "*.sh", "*.bash",
|
|
17
|
+
"*.bat", "*.cmd", "*.ps1", "*.whl", "*.egg", "*.jar", "*.ipynb"]
|
|
18
|
+
DEFAULT_RULES = {"forbidden_files": PICKLE_FILES}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def challenge_rules(challenge: dict) -> dict:
|
|
22
|
+
"""The file rules the backend applies to this challenge's brain."""
|
|
23
|
+
brain = challenge.get("brain") or {}
|
|
24
|
+
return {} if brain.get("submission_code") else {"forbidden_files": PICKLE_FILES + CODE_FILES}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def check(files: list[tuple[str, int]], rules: dict | None) -> list[str]:
|
|
28
|
+
rules = {**DEFAULT_RULES, **(rules or {})}
|
|
29
|
+
paths = [p for p, _ in files]
|
|
30
|
+
problems = []
|
|
31
|
+
for pattern in rules.get("required_files", []):
|
|
32
|
+
if not any(fnmatch(p, pattern) for p in paths):
|
|
33
|
+
problems.append(f"missing required file matching '{pattern}'")
|
|
34
|
+
for pattern in rules.get("forbidden_files", []):
|
|
35
|
+
bad = [p for p in paths if fnmatch(p, pattern) or fnmatch(p.rsplit("/", 1)[-1].lower(), pattern)]
|
|
36
|
+
if bad:
|
|
37
|
+
problems.append(f"forbidden file(s) {', '.join(bad[:5])} (pattern '{pattern}')")
|
|
38
|
+
limit = rules.get("max_total_bytes")
|
|
39
|
+
total = sum(size for _, size in files)
|
|
40
|
+
if limit and total > limit:
|
|
41
|
+
problems.append(f"total size {total / 1e9:.2f} GB exceeds the limit of {limit / 1e9:.2f} GB")
|
|
42
|
+
return problems
|