unidroid-cli 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unidroid_cli/__init__.py +1 -0
- unidroid_cli/__main__.py +3 -0
- unidroid_cli/_common.py +74 -0
- unidroid_cli/agent.py +392 -0
- unidroid_cli/bench.py +143 -0
- unidroid_cli/checks.py +42 -0
- unidroid_cli/cli.py +105 -0
- unidroid_cli/commands/__init__.py +1 -0
- unidroid_cli/commands/account.py +118 -0
- unidroid_cli/commands/auth.py +155 -0
- unidroid_cli/commands/bench.py +132 -0
- unidroid_cli/commands/challenge.py +97 -0
- unidroid_cli/commands/dataset.py +546 -0
- unidroid_cli/commands/submission.py +478 -0
- unidroid_cli/commands/validator.py +124 -0
- unidroid_cli/compare.py +64 -0
- unidroid_cli/config.py +85 -0
- unidroid_cli/dataset.py +414 -0
- unidroid_cli/dataset_sample.py +594 -0
- unidroid_cli/hfrepo.py +202 -0
- unidroid_cli/http.py +66 -0
- unidroid_cli/protocol.py +80 -0
- unidroid_cli/safe_extract.py +58 -0
- unidroid_cli/uploads.py +76 -0
- unidroid_cli/wallet.py +62 -0
- unidroid_cli-0.3.0.dist-info/METADATA +196 -0
- unidroid_cli-0.3.0.dist-info/RECORD +29 -0
- unidroid_cli-0.3.0.dist-info/WHEEL +4 -0
- unidroid_cli-0.3.0.dist-info/entry_points.txt +2 -0
unidroid_cli/hfrepo.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Hugging Face access from the miner's machine (HF_TOKEN or `hf auth login`; never sent to Unidroid)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
from . import protocol
|
|
10
|
+
|
|
11
|
+
DATASET_REPO_RE = re.compile(r"^[A-Za-z0-9][\w.-]{0,95}/[\w.-]{1,96}$")
|
|
12
|
+
_HF_DATASET_PREFIXES = (
|
|
13
|
+
"https://huggingface.co/datasets/",
|
|
14
|
+
"https://www.huggingface.co/datasets/",
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class RepoFile:
|
|
20
|
+
path: str
|
|
21
|
+
hash: str
|
|
22
|
+
size: int
|
|
23
|
+
lfs: bool
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class RepoSnapshot:
|
|
28
|
+
repo: str
|
|
29
|
+
commit_sha: str
|
|
30
|
+
files: list[RepoFile]
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def manifest_hash(self) -> str:
|
|
34
|
+
return protocol.manifest_hash((f.path, f.hash) for f in self.files)
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def total_bytes(self) -> int:
|
|
38
|
+
return sum(f.size for f in self.files)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class HfRepoError(Exception):
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def local_token(explicit: str | None = None) -> str | None:
|
|
46
|
+
if explicit:
|
|
47
|
+
return explicit
|
|
48
|
+
from huggingface_hub import get_token
|
|
49
|
+
|
|
50
|
+
return get_token()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve(repo: str, revision: str, token: str | None) -> RepoSnapshot:
|
|
54
|
+
"""Resolve a branch/tag/sha to the exact commit and its file listing (no download)."""
|
|
55
|
+
from huggingface_hub import HfApi
|
|
56
|
+
|
|
57
|
+
try:
|
|
58
|
+
info = HfApi().model_info(repo, revision=revision, files_metadata=True, token=token or False)
|
|
59
|
+
except Exception as exc: # noqa: BLE001
|
|
60
|
+
raise HfRepoError(f"cannot read {repo}@{revision} on Hugging Face: {exc}") from exc
|
|
61
|
+
files = []
|
|
62
|
+
for sib in info.siblings or []:
|
|
63
|
+
if sib.lfs is not None:
|
|
64
|
+
sha256 = sib.lfs.sha256 if hasattr(sib.lfs, "sha256") else sib.lfs["sha256"]
|
|
65
|
+
files.append(RepoFile(sib.rfilename, sha256, int(sib.size or 0), True))
|
|
66
|
+
else:
|
|
67
|
+
files.append(RepoFile(sib.rfilename, sib.blob_id or "", int(sib.size or 0), False))
|
|
68
|
+
return RepoSnapshot(repo, info.sha, files)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def download(repo: str, sha: str, token: str | None, dest) -> None:
|
|
72
|
+
"""Every file of one commit into ``dest``. Hugging Face's ``.cache/`` and ``.gitattributes`` are removed."""
|
|
73
|
+
import shutil
|
|
74
|
+
from pathlib import Path
|
|
75
|
+
|
|
76
|
+
from huggingface_hub import snapshot_download
|
|
77
|
+
|
|
78
|
+
dest = Path(dest)
|
|
79
|
+
try:
|
|
80
|
+
snapshot_download(repo, revision=sha, token=token or False, local_dir=dest)
|
|
81
|
+
except Exception as exc: # noqa: BLE001
|
|
82
|
+
raise HfRepoError(f"cannot download {repo}@{sha}: {exc}") from exc
|
|
83
|
+
shutil.rmtree(dest / ".cache", ignore_errors=True)
|
|
84
|
+
(dest / ".gitattributes").unlink(missing_ok=True)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
HF_GENERATED = (".gitattributes",)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def comparable(files: list[RepoFile], declared: dict[str, str]) -> dict[str, RepoFile]:
|
|
91
|
+
"""The files that must equal agent.yaml: .gitattributes and an undeclared README.md are left out."""
|
|
92
|
+
return {f.path: f for f in files
|
|
93
|
+
if f.path not in HF_GENERATED and not (f.path == "README.md" and "README.md" not in declared)}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def weights_problems(files: list[RepoFile], declared: dict[str, str], local: dict | None = None) -> list[str]:
|
|
97
|
+
"""Differences between a commit and agent.yaml. Files without LFS have only a git blob id on Hugging Face,
|
|
98
|
+
so they are compared with the local file's blob id; pass ``local`` (path -> LocalWeight) to check them."""
|
|
99
|
+
remote = comparable(files, declared)
|
|
100
|
+
problems = []
|
|
101
|
+
for path in sorted(set(remote) | set(declared)):
|
|
102
|
+
f = remote.get(path)
|
|
103
|
+
if f is None:
|
|
104
|
+
problems.append(f"{path}: in agent.yaml, not at the commit")
|
|
105
|
+
elif path not in declared:
|
|
106
|
+
problems.append(f"{path}: at the commit, not in agent.yaml")
|
|
107
|
+
elif f.lfs and f.hash.lower() != declared[path]:
|
|
108
|
+
problems.append(f"{path}: sha256 at the commit differs from agent.yaml")
|
|
109
|
+
elif not f.lfs and local is not None:
|
|
110
|
+
mine = local.get(path)
|
|
111
|
+
if mine is None or mine.sha256 != declared[path] or mine.blob_id != f.hash:
|
|
112
|
+
problems.append(f"{path}: file at the commit differs from the local file hashed in agent.yaml")
|
|
113
|
+
return problems
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def push_weights(repo: str, files: dict[str, object], token: str, *, private: bool = True,
|
|
117
|
+
message: str = "Upload weights with udcli", on_delete=None) -> str:
|
|
118
|
+
"""Make the repo's main branch hold exactly ``files`` (path in repo -> local file) and return the commit SHA.
|
|
119
|
+
|
|
120
|
+
Files are sent with the Hub's LFS/Xet upload, which skips content the Hub already has. Files that are no
|
|
121
|
+
longer declared are deleted in the same commit.
|
|
122
|
+
"""
|
|
123
|
+
from huggingface_hub import CommitOperationAdd, CommitOperationDelete, HfApi
|
|
124
|
+
|
|
125
|
+
api = HfApi(token=token)
|
|
126
|
+
try:
|
|
127
|
+
api.create_repo(repo, private=private, exist_ok=True, repo_type="model")
|
|
128
|
+
try:
|
|
129
|
+
existing = [f.path for f in resolve(repo, "main", token).files]
|
|
130
|
+
except HfRepoError:
|
|
131
|
+
existing = []
|
|
132
|
+
operations: list = [CommitOperationAdd(path_in_repo=p, path_or_fileobj=str(path))
|
|
133
|
+
for p, path in sorted(files.items())]
|
|
134
|
+
stale = [p for p in existing if p not in files and p not in HF_GENERATED and p != "README.md"]
|
|
135
|
+
if stale and on_delete:
|
|
136
|
+
on_delete(sorted(stale))
|
|
137
|
+
operations += [CommitOperationDelete(path_in_repo=p) for p in stale]
|
|
138
|
+
info = api.create_commit(repo, operations=operations, commit_message=message)
|
|
139
|
+
except HfRepoError:
|
|
140
|
+
raise
|
|
141
|
+
except Exception as exc: # noqa: BLE001
|
|
142
|
+
raise HfRepoError(f"upload to {repo} failed: {exc}") from exc
|
|
143
|
+
return info.oid
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def normalize_dataset_repo(repo: str) -> str:
|
|
147
|
+
"""Accept either ``<org>/<name>`` or ``https://huggingface.co/datasets/<org>/<name>``."""
|
|
148
|
+
cleaned = repo.strip().rstrip("/")
|
|
149
|
+
for prefix in _HF_DATASET_PREFIXES:
|
|
150
|
+
if cleaned.lower().startswith(prefix):
|
|
151
|
+
parts = cleaned[len(prefix):].split("/")
|
|
152
|
+
if len(parts) >= 2:
|
|
153
|
+
cleaned = f"{parts[0]}/{parts[1]}"
|
|
154
|
+
break
|
|
155
|
+
if not DATASET_REPO_RE.match(cleaned):
|
|
156
|
+
raise HfRepoError(f"{repo!r}: expected a Hugging Face dataset repo in <org>/<name> form")
|
|
157
|
+
return cleaned
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def dataset_url(repo: str) -> str:
|
|
161
|
+
return f"https://huggingface.co/datasets/{normalize_dataset_repo(repo)}"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def verify_write_token(token: str) -> dict:
|
|
165
|
+
"""Confirm the token authenticates on Hugging Face and is not explicitly read-only."""
|
|
166
|
+
from huggingface_hub import HfApi
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
who = HfApi(token=token).whoami()
|
|
170
|
+
except Exception as exc: # noqa: BLE001
|
|
171
|
+
raise HfRepoError(f"invalid Hugging Face token: {exc}") from exc
|
|
172
|
+
role = ((who.get("auth") or {}).get("accessToken") or {}).get("role")
|
|
173
|
+
if role == "read":
|
|
174
|
+
raise HfRepoError("this Hugging Face token is read-only; pass a token with write permission")
|
|
175
|
+
return who
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def push_dataset_sample(repo: str, files: dict[str, Path], token: str, *, private: bool = False,
|
|
179
|
+
message: str = "Upload dataset sample preview with udcli") -> tuple[str, str]:
|
|
180
|
+
"""Create or update a Hugging Face dataset repo so ``main`` holds ``files``. Returns (commit_sha, url)."""
|
|
181
|
+
from huggingface_hub import CommitOperationAdd, CommitOperationDelete, HfApi
|
|
182
|
+
|
|
183
|
+
repo_id = normalize_dataset_repo(repo)
|
|
184
|
+
api = HfApi(token=token)
|
|
185
|
+
try:
|
|
186
|
+
api.create_repo(repo_id, private=private, exist_ok=True, repo_type="dataset")
|
|
187
|
+
try:
|
|
188
|
+
existing_info = api.dataset_info(repo_id, revision="main", token=token)
|
|
189
|
+
existing = [sib.rfilename for sib in (existing_info.siblings or [])]
|
|
190
|
+
except Exception: # noqa: BLE001 - empty or brand-new repo
|
|
191
|
+
existing = []
|
|
192
|
+
operations: list = [CommitOperationAdd(path_in_repo=p, path_or_fileobj=str(path))
|
|
193
|
+
for p, path in sorted(files.items())]
|
|
194
|
+
operations += [CommitOperationDelete(path_in_repo=p) for p in existing
|
|
195
|
+
if p not in files and p not in HF_GENERATED]
|
|
196
|
+
info = api.create_commit(repo_id, operations=operations, commit_message=message, repo_type="dataset")
|
|
197
|
+
except HfRepoError:
|
|
198
|
+
raise
|
|
199
|
+
except Exception as exc: # noqa: BLE001
|
|
200
|
+
raise HfRepoError(f"sample upload to Hugging Face dataset {repo_id} failed: {exc}") from exc
|
|
201
|
+
return info.oid, dataset_url(repo_id)
|
|
202
|
+
|
unidroid_cli/http.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
from . import __version__, config
|
|
8
|
+
|
|
9
|
+
# Tests replace this to route requests to an in-process app.
|
|
10
|
+
TRANSPORT: httpx.BaseTransport | None = None
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ApiError(Exception):
|
|
14
|
+
def __init__(self, status: int, detail: str):
|
|
15
|
+
super().__init__(detail)
|
|
16
|
+
self.status = status
|
|
17
|
+
self.detail = detail
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _detail(resp: httpx.Response) -> str:
|
|
21
|
+
try:
|
|
22
|
+
body = resp.json()
|
|
23
|
+
except ValueError:
|
|
24
|
+
return resp.text or resp.reason_phrase
|
|
25
|
+
detail = body.get("detail", body) if isinstance(body, dict) else body
|
|
26
|
+
if isinstance(detail, list): # pydantic validation errors
|
|
27
|
+
return "; ".join(f"{'.'.join(str(x) for x in d.get('loc', []))}: {d.get('msg')}" for d in detail)
|
|
28
|
+
return str(detail)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Api:
|
|
32
|
+
def __init__(self, api_url: str | None = None, api_key: str | None = None, require_key: bool = True):
|
|
33
|
+
self.base = config.api_url(api_url)
|
|
34
|
+
self.key = config.api_key(api_key)
|
|
35
|
+
if require_key and not self.key:
|
|
36
|
+
raise ApiError(401, "not logged in: pass --api-key, or run `udcli auth login`")
|
|
37
|
+
headers = {"User-Agent": f"udcli/{__version__}"}
|
|
38
|
+
if self.key:
|
|
39
|
+
headers["Authorization"] = f"Bearer {self.key}"
|
|
40
|
+
self.client = httpx.Client(base_url=self.base, headers=headers, timeout=60, transport=TRANSPORT)
|
|
41
|
+
|
|
42
|
+
def request(self, method: str, path: str, **kwargs: Any) -> Any:
|
|
43
|
+
try:
|
|
44
|
+
resp = self.client.request(method, path, **kwargs)
|
|
45
|
+
except httpx.HTTPError as exc:
|
|
46
|
+
raise ApiError(0, f"cannot reach {self.base}: {exc}") from exc
|
|
47
|
+
if resp.status_code >= 400:
|
|
48
|
+
raise ApiError(resp.status_code, _detail(resp))
|
|
49
|
+
return resp.json() if resp.content else None
|
|
50
|
+
|
|
51
|
+
def get(self, path: str, **kwargs: Any) -> Any:
|
|
52
|
+
return self.request("GET", path, **kwargs)
|
|
53
|
+
|
|
54
|
+
def post(self, path: str, **kwargs: Any) -> Any:
|
|
55
|
+
return self.request("POST", path, **kwargs)
|
|
56
|
+
|
|
57
|
+
def put(self, path: str, **kwargs: Any) -> Any:
|
|
58
|
+
return self.request("PUT", path, **kwargs)
|
|
59
|
+
|
|
60
|
+
def delete(self, path: str, **kwargs: Any) -> Any:
|
|
61
|
+
return self.request("DELETE", path, **kwargs)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def raw_client(timeout: float = 600) -> httpx.Client:
|
|
65
|
+
"""Client for presigned storage URLs (no Unidroid auth headers)."""
|
|
66
|
+
return httpx.Client(timeout=timeout, transport=TRANSPORT, follow_redirects=True)
|
unidroid_cli/protocol.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Client side of the Unidroid wire protocol (see unidroid-backend/PROTOCOL.md).
|
|
2
|
+
|
|
3
|
+
Implemented independently; tests/protocol_vectors.json is the shared reference.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import io
|
|
10
|
+
import json
|
|
11
|
+
import zipfile
|
|
12
|
+
from collections.abc import Iterable, Mapping
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
BUNDLE_SUBMISSION_TYPE = "unidroid.submission"
|
|
16
|
+
BUNDLE_DATE_TIME = (1980, 1, 1, 0, 0, 0)
|
|
17
|
+
BUNDLE_FILE_MODE = 0o100644
|
|
18
|
+
COLDKEY_TYPE = "unidroid.coldkey"
|
|
19
|
+
PUBLICATION_TERMS = "unidroid.publish-winners"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def canonical_json(obj: Any) -> bytes:
|
|
23
|
+
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def manifest_hash(entries: Iterable[tuple[str, str]]) -> str:
|
|
27
|
+
lines = "".join(f"{path}:{digest}\n" for path, digest in sorted(entries))
|
|
28
|
+
return hashlib.sha256(lines.encode("utf-8")).hexdigest()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def git_blob_id(data: bytes) -> str:
|
|
32
|
+
return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def weights_digest(weights: Mapping[str, str]) -> str:
|
|
36
|
+
"""manifest_hash over agent.yaml's ``weights`` map (path under weights/ -> sha256)."""
|
|
37
|
+
return manifest_hash((path, str(digest).lower()) for path, digest in weights.items())
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def write_bundle_zip(out, files: Iterable[tuple[str, Any]]) -> None:
|
|
41
|
+
"""Write the deterministic bundle archive. ``files`` yields (archive path, bytes or a local Path)."""
|
|
42
|
+
entries = sorted(files, key=lambda item: item[0])
|
|
43
|
+
with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_STORED) as zf:
|
|
44
|
+
for path, source in entries:
|
|
45
|
+
info = zipfile.ZipInfo(path, date_time=BUNDLE_DATE_TIME)
|
|
46
|
+
info.compress_type = zipfile.ZIP_STORED
|
|
47
|
+
info.create_system = 3
|
|
48
|
+
info.external_attr = BUNDLE_FILE_MODE << 16
|
|
49
|
+
data = source if isinstance(source, bytes) else source.read_bytes()
|
|
50
|
+
zf.writestr(info, data)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def bundle_zip(files: Mapping[str, bytes]) -> bytes:
|
|
54
|
+
buf = io.BytesIO()
|
|
55
|
+
write_bundle_zip(buf, files.items())
|
|
56
|
+
return buf.getvalue()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def bundle_submission_payload(*, challenge_id: str, hotkey: str, hf_repo: str, commit_sha: str, manifest_hash: str,
|
|
60
|
+
bundle_sha256: str, bundle_size: int, agent_yaml_sha256: str, weights_digest: str,
|
|
61
|
+
nonce: str, timestamp: int) -> dict[str, Any]:
|
|
62
|
+
return {
|
|
63
|
+
"type": BUNDLE_SUBMISSION_TYPE,
|
|
64
|
+
"challenge_id": challenge_id,
|
|
65
|
+
"hotkey": hotkey,
|
|
66
|
+
"hf_repo": hf_repo,
|
|
67
|
+
"commit_sha": commit_sha,
|
|
68
|
+
"manifest_hash": manifest_hash,
|
|
69
|
+
"bundle_sha256": bundle_sha256,
|
|
70
|
+
"bundle_size": bundle_size,
|
|
71
|
+
"agent_yaml_sha256": agent_yaml_sha256,
|
|
72
|
+
"weights_digest": weights_digest,
|
|
73
|
+
"terms": PUBLICATION_TERMS,
|
|
74
|
+
"nonce": nonce,
|
|
75
|
+
"timestamp": timestamp,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def coldkey_message(*, user_id: int, coldkey: str, nonce: str) -> bytes:
|
|
80
|
+
return canonical_json({"type": COLDKEY_TYPE, "user_id": user_id, "coldkey": coldkey, "nonce": nonce})
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Unpack zip and tar archives into a folder, refusing anything that would land outside it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import stat
|
|
6
|
+
import tarfile
|
|
7
|
+
import zipfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class UnsafeArchive(Exception):
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _inside(root: Path, name: str) -> Path:
|
|
16
|
+
dest = (root / name).resolve()
|
|
17
|
+
if dest != root and root not in dest.parents:
|
|
18
|
+
raise UnsafeArchive(f"refusing to extract {name!r}: it points outside the target folder")
|
|
19
|
+
return dest
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _zip_members(zf: zipfile.ZipFile, root: Path) -> list[zipfile.ZipInfo]:
|
|
23
|
+
members = []
|
|
24
|
+
for info in zf.infolist():
|
|
25
|
+
name = info.filename
|
|
26
|
+
if name.startswith(("/", "\\")) or (len(name) > 1 and name[1] == ":") or "\\" in name:
|
|
27
|
+
raise UnsafeArchive(f"refusing to extract {name!r}: use relative paths with '/' separators")
|
|
28
|
+
mode = info.external_attr >> 16
|
|
29
|
+
if mode and (stat.S_ISLNK(mode) or stat.S_ISCHR(mode) or stat.S_ISBLK(mode) or stat.S_ISFIFO(mode)):
|
|
30
|
+
raise UnsafeArchive(f"refusing to extract {name!r}: only files and folders are allowed")
|
|
31
|
+
_inside(root, name)
|
|
32
|
+
members.append(info)
|
|
33
|
+
return members
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _tar_members(tar: tarfile.TarFile, root: Path):
|
|
37
|
+
for member in tar.getmembers():
|
|
38
|
+
if not (member.isfile() or member.isdir()): # links and devices can point outside the folder
|
|
39
|
+
raise UnsafeArchive(f"refusing to extract {member.name!r}: only files and folders are allowed")
|
|
40
|
+
_inside(root, member.name)
|
|
41
|
+
yield member
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def safe_extract(archive: Path, target: Path) -> Path:
|
|
45
|
+
"""Unpack ``archive`` (.zip, .tar, .tar.gz, .tgz) into ``target``, created if needed. Every member is checked
|
|
46
|
+
before anything is written."""
|
|
47
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
root = target.resolve()
|
|
49
|
+
if zipfile.is_zipfile(archive):
|
|
50
|
+
with zipfile.ZipFile(archive) as zf:
|
|
51
|
+
zf.extractall(target, members=_zip_members(zf, root))
|
|
52
|
+
return target
|
|
53
|
+
with tarfile.open(archive) as tar:
|
|
54
|
+
members = list(_tar_members(tar, root))
|
|
55
|
+
# the "data" filter (where available) also drops special files and permission bits
|
|
56
|
+
extra = {"filter": "data"} if hasattr(tarfile, "data_filter") else {}
|
|
57
|
+
tar.extractall(target, members=members, **extra) # noqa: S202 - members are checked above
|
|
58
|
+
return target
|
unidroid_cli/uploads.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Multipart upload of one file to presigned URLs: parallel parts, retries, and parts that are already stored."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Callable, Mapping
|
|
7
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
from .http import raw_client
|
|
13
|
+
|
|
14
|
+
Progress = Callable[[int, int], None]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class PartUploadError(Exception):
|
|
18
|
+
def __init__(self, part: int, failure: str):
|
|
19
|
+
super().__init__(f"upload of part {part} failed ({failure})")
|
|
20
|
+
self.part = part
|
|
21
|
+
self.failure = failure
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _put_part(client: httpx.Client, path: Path, number: int, part_size: int, url: str, attempts: int,
|
|
25
|
+
refresh: Callable[[int], str] | None) -> dict:
|
|
26
|
+
with path.open("rb") as fh:
|
|
27
|
+
fh.seek((number - 1) * part_size)
|
|
28
|
+
body = fh.read(part_size)
|
|
29
|
+
for attempt in range(1, attempts + 1):
|
|
30
|
+
try:
|
|
31
|
+
resp = client.put(url, content=body)
|
|
32
|
+
if resp.status_code < 400:
|
|
33
|
+
return {"part_number": number, "etag": resp.headers.get("ETag", "").strip('"')}
|
|
34
|
+
failure = f"HTTP {resp.status_code}"
|
|
35
|
+
if resp.status_code in (400, 401, 403) and refresh: # an expired URL: ask for a new one
|
|
36
|
+
url = refresh(number)
|
|
37
|
+
except httpx.HTTPError as exc:
|
|
38
|
+
failure = type(exc).__name__
|
|
39
|
+
if attempt == attempts:
|
|
40
|
+
raise PartUploadError(number, failure)
|
|
41
|
+
time.sleep(min(2 ** attempt, 20))
|
|
42
|
+
raise AssertionError("unreachable")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def upload_parts(path: Path, part_size: int, urls: Mapping[int, str], *, stored: Mapping[int, str] | None = None,
|
|
46
|
+
jobs: int = 4, attempts: int = 5, refresh: Callable[[int], str] | None = None,
|
|
47
|
+
progress: Progress | None = None, on_part: Callable[[dict], None] | None = None) -> list[dict]:
|
|
48
|
+
"""PUT every part in ``urls`` that is not in ``stored`` and return all parts, sorted, for the complete call.
|
|
49
|
+
|
|
50
|
+
``stored`` maps part number to ETag for parts the server already holds. ``refresh(number)`` returns a new
|
|
51
|
+
URL when one has expired. ``on_part`` is called with each finished part, so a caller can save it.
|
|
52
|
+
Memory use is about ``jobs`` times ``part_size``.
|
|
53
|
+
"""
|
|
54
|
+
stored = dict(stored or {})
|
|
55
|
+
numbers = sorted(set(urls) | set(stored))
|
|
56
|
+
parts = {n: {"part_number": n, "etag": etag} for n, etag in stored.items()}
|
|
57
|
+
todo = [n for n in numbers if n not in stored]
|
|
58
|
+
done, total = len(parts), len(numbers)
|
|
59
|
+
if progress:
|
|
60
|
+
progress(done, total)
|
|
61
|
+
if todo:
|
|
62
|
+
with raw_client() as client, ThreadPoolExecutor(max_workers=max(1, jobs)) as pool:
|
|
63
|
+
futures = [pool.submit(_put_part, client, path, n, part_size, urls[n], attempts, refresh) for n in todo]
|
|
64
|
+
try:
|
|
65
|
+
for future in futures:
|
|
66
|
+
part = future.result()
|
|
67
|
+
parts[part["part_number"]] = part
|
|
68
|
+
done += 1
|
|
69
|
+
if on_part:
|
|
70
|
+
on_part(part)
|
|
71
|
+
if progress:
|
|
72
|
+
progress(done, total)
|
|
73
|
+
except BaseException:
|
|
74
|
+
pool.shutdown(wait=True, cancel_futures=True)
|
|
75
|
+
raise
|
|
76
|
+
return [parts[n] for n in numbers]
|
unidroid_cli/wallet.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Bittensor wallet signing. ``bittensor-wallet`` is imported lazily (``pip install "unidroid-cli[wallet]"``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Callable
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class WalletError(Exception):
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class Signer:
|
|
15
|
+
ss58: str
|
|
16
|
+
sign: Callable[[bytes], str] # returns hex signature
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _wallet(name: str, hotkey: str, path: str | None):
|
|
20
|
+
try:
|
|
21
|
+
from bittensor_wallet import Wallet
|
|
22
|
+
except ImportError as exc:
|
|
23
|
+
raise WalletError(
|
|
24
|
+
"bittensor-wallet is not installed. Install it with `pip install bittensor-wallet` "
|
|
25
|
+
"or `pip install \"unidroid-cli[wallet]\"`."
|
|
26
|
+
) from exc
|
|
27
|
+
kwargs = {"name": name, "hotkey": hotkey}
|
|
28
|
+
if path:
|
|
29
|
+
kwargs["path"] = path
|
|
30
|
+
return Wallet(**kwargs)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def hotkey_signer(name: str, hotkey: str, path: str | None = None) -> Signer:
|
|
34
|
+
wallet = _wallet(name, hotkey, path)
|
|
35
|
+
if not wallet.hotkey_file.exists_on_device():
|
|
36
|
+
raise WalletError(
|
|
37
|
+
f"hotkey '{hotkey}' not found for wallet '{name}'. "
|
|
38
|
+
f"Pass --wallet and --hotkey (check `btcli wallet list`)."
|
|
39
|
+
)
|
|
40
|
+
kp = wallet.hotkey
|
|
41
|
+
return Signer(kp.ss58_address, lambda msg: kp.sign(msg).hex())
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def coldkey_signer(name: str, path: str | None = None) -> Signer:
|
|
45
|
+
wallet = _wallet(name, "default", path)
|
|
46
|
+
if not wallet.coldkey_file.exists_on_device():
|
|
47
|
+
raise WalletError(
|
|
48
|
+
f"coldkey for wallet '{name}' not found. Pass --wallet (check `btcli wallet list`)."
|
|
49
|
+
)
|
|
50
|
+
kp = wallet.coldkey # prompts for the coldkey password
|
|
51
|
+
return Signer(kp.ss58_address, lambda msg: kp.sign(msg).hex())
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def coldkey_address(name: str, path: str | None = None) -> str:
|
|
55
|
+
"""Public SS58 of a wallet coldkey. Does not unlock the key."""
|
|
56
|
+
wallet = _wallet(name, "default", path)
|
|
57
|
+
pub = wallet.coldkeypub_file
|
|
58
|
+
if not pub.exists_on_device():
|
|
59
|
+
raise WalletError(
|
|
60
|
+
f"coldkey for wallet '{name}' not found. Pass --wallet (check `btcli wallet list`)."
|
|
61
|
+
)
|
|
62
|
+
return wallet.coldkeypub.ss58_address
|