llmt-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmt/__init__.py +7 -0
- llmt/api.py +138 -0
- llmt/cli.py +562 -0
- llmt/clients.py +1316 -0
- llmt/data/manifest-pubkey.json +7 -0
- llmt/doctor.py +970 -0
- llmt/doctorcmd.py +142 -0
- llmt/download.py +321 -0
- llmt/errors.py +132 -0
- llmt/layout.py +676 -0
- llmt/manifest.py +314 -0
- llmt/matcher.py +358 -0
- llmt/models.py +130 -0
- llmt/output.py +51 -0
- llmt/permissions.py +154 -0
- llmt/scancmd.py +556 -0
- llmt/scanner.py +328 -0
- llmt/telemetry/__init__.py +15 -0
- llmt/telemetry/artifacts.py +217 -0
- llmt/telemetry/beat.py +221 -0
- llmt/telemetry/cmd.py +358 -0
- llmt/telemetry/collect.py +252 -0
- llmt/telemetry/config.py +237 -0
- llmt/telemetry/consent.py +57 -0
- llmt/telemetry/errors.py +41 -0
- llmt/telemetry/wire.py +194 -0
- llmt/verify.py +129 -0
- llmt_cli-0.2.0.dist-info/METADATA +307 -0
- llmt_cli-0.2.0.dist-info/RECORD +33 -0
- llmt_cli-0.2.0.dist-info/WHEEL +5 -0
- llmt_cli-0.2.0.dist-info/entry_points.txt +2 -0
- llmt_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
- llmt_cli-0.2.0.dist-info/top_level.txt +1 -0
llmt/manifest.py
ADDED
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""Ed25519 manifest verification — llmt's authenticity layer (Fix M2, Option A).
|
|
2
|
+
|
|
3
|
+
For every published torrent the catalog serves a manifest document signed
|
|
4
|
+
with the site's Ed25519 manifest key (FROZEN contract: llmt-manifest-spec
|
|
5
|
+
v0.1.0, WP-3.3 signing, WP-8.1 endpoint):
|
|
6
|
+
|
|
7
|
+
GET /api/manifests/{info_hash} ->
|
|
8
|
+
{"data": <manifest doc, incl. its `signature` block>,
|
|
9
|
+
"meta": {"schema_version", "signature", "signed_at"}}
|
|
10
|
+
|
|
11
|
+
SIGNING INPUT (must match the server byte-for-byte):
|
|
12
|
+
canonical JSON of the manifest WITHOUT its `signature` key — object keys
|
|
13
|
+
sorted recursively by UTF-8 byte order, array order preserved,
|
|
14
|
+
separators "," and ":", unicode NOT escaped, slashes NOT escaped, UTF-8,
|
|
15
|
+
no trailing newline. In Python that is exactly
|
|
16
|
+
``json.dumps(body, sort_keys=True, separators=(",", ":"),
|
|
17
|
+
ensure_ascii=False).encode("utf-8")``.
|
|
18
|
+
The signature is a DETACHED Ed25519 signature (libsodium
|
|
19
|
+
``crypto_sign_detached``), base64 in ``signature.sig`` alongside
|
|
20
|
+
``signature.algo`` ("ed25519") and ``signature.key_id``.
|
|
21
|
+
|
|
22
|
+
TRUST ROOT: the public key PINNED IN THIS PACKAGE
|
|
23
|
+
(``llmt/data/manifest-pubkey.json``) — decided 2026-07-13 (M2, Option A).
|
|
24
|
+
The key published at https://llmtorrents.com/manifest-pubkey is NEVER
|
|
25
|
+
fetched at runtime for trust decisions; a manifest signed by any other key
|
|
26
|
+
does not verify here. Key rotation therefore ships as a new llmt release —
|
|
27
|
+
the wrong-key failure message says so.
|
|
28
|
+
|
|
29
|
+
Every failure raises :class:`~llmt.errors.ManifestVerificationError` with a
|
|
30
|
+
distinct, honest reason. There is no silent fallback to the old TLS-only
|
|
31
|
+
"file matches the catalog sha256" claim: unverifiable means NOT verified.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import base64
|
|
37
|
+
import binascii
|
|
38
|
+
import hashlib
|
|
39
|
+
import json
|
|
40
|
+
from dataclasses import dataclass
|
|
41
|
+
from typing import Any, Optional
|
|
42
|
+
|
|
43
|
+
from nacl.exceptions import BadSignatureError
|
|
44
|
+
from nacl.signing import VerifyKey
|
|
45
|
+
|
|
46
|
+
from .errors import LlmtError, ManifestVerificationError
|
|
47
|
+
|
|
48
|
+
_PINNED_RESOURCE = "manifest-pubkey.json"
|
|
49
|
+
_ED25519_KEY_BYTES = 32
|
|
50
|
+
_ED25519_SIG_BYTES = 64
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class PinnedKey:
|
|
55
|
+
"""The packaged trust root: algo + key id + raw 32-byte public key."""
|
|
56
|
+
|
|
57
|
+
algo: str
|
|
58
|
+
key_id: str
|
|
59
|
+
public_key: bytes
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def fingerprint(self) -> str:
|
|
63
|
+
"""SHA-256 hex of the raw public-key bytes (for display/audit)."""
|
|
64
|
+
return hashlib.sha256(self.public_key).hexdigest()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def load_pinned_key() -> PinnedKey:
|
|
68
|
+
"""Load the Ed25519 public key shipped inside the llmt package.
|
|
69
|
+
|
|
70
|
+
This file is part of the installed package — it is the trust anchor, so
|
|
71
|
+
a damaged/missing/inconsistent pin is a hard failure, never a downgrade.
|
|
72
|
+
"""
|
|
73
|
+
from importlib import resources
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
text = (resources.files("llmt") / "data" / _PINNED_RESOURCE
|
|
77
|
+
).read_text("utf-8")
|
|
78
|
+
data = json.loads(text)
|
|
79
|
+
algo = data["algo"]
|
|
80
|
+
key_id = data["key_id"]
|
|
81
|
+
public_key = bytes.fromhex(data["public_key_hex"])
|
|
82
|
+
expected_fp = data.get("public_key_sha256_fingerprint")
|
|
83
|
+
except Exception as exc:
|
|
84
|
+
raise ManifestVerificationError(
|
|
85
|
+
"cannot load the manifest public key pinned in this llmt build "
|
|
86
|
+
f"({exc.__class__.__name__}: {exc}); the installation is damaged "
|
|
87
|
+
"— reinstall llmt. Nothing was verified."
|
|
88
|
+
) from exc
|
|
89
|
+
|
|
90
|
+
if (algo != "ed25519" or len(public_key) != _ED25519_KEY_BYTES
|
|
91
|
+
or not key_id):
|
|
92
|
+
raise ManifestVerificationError(
|
|
93
|
+
"the manifest public key pinned in this llmt build is invalid "
|
|
94
|
+
"(wrong algo, key id, or key length); the installation is "
|
|
95
|
+
"damaged — reinstall llmt. Nothing was verified."
|
|
96
|
+
)
|
|
97
|
+
pinned = PinnedKey(algo=algo, key_id=key_id, public_key=public_key)
|
|
98
|
+
if expected_fp and pinned.fingerprint != expected_fp.lower():
|
|
99
|
+
raise ManifestVerificationError(
|
|
100
|
+
"the manifest public key pinned in this llmt build does not "
|
|
101
|
+
"match its recorded fingerprint; the installation is damaged — "
|
|
102
|
+
"reinstall llmt. Nothing was verified."
|
|
103
|
+
)
|
|
104
|
+
return pinned
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def canonical_json(value: Any) -> bytes:
|
|
108
|
+
"""Serialize per the frozen canonicalization (llmt-manifest-spec).
|
|
109
|
+
|
|
110
|
+
``json.dumps(sort_keys=True)`` sorts object keys recursively by code
|
|
111
|
+
point, which equals UTF-8 byte order; list order is preserved;
|
|
112
|
+
``separators=(",", ":")`` removes whitespace; ``ensure_ascii=False``
|
|
113
|
+
leaves unicode unescaped (Python never escapes slashes).
|
|
114
|
+
"""
|
|
115
|
+
_reject_floats(value)
|
|
116
|
+
return json.dumps(value, sort_keys=True, separators=(",", ":"),
|
|
117
|
+
ensure_ascii=False).encode("utf-8")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _reject_floats(value: Any) -> None:
|
|
121
|
+
# The frozen spec defines canonical encoding only for objects, arrays,
|
|
122
|
+
# strings, booleans, null and INTEGERS. Float formatting differs across
|
|
123
|
+
# languages, so a float in a manifest is unverifiable — refuse rather
|
|
124
|
+
# than guess (fail closed).
|
|
125
|
+
if isinstance(value, float):
|
|
126
|
+
raise ManifestVerificationError(
|
|
127
|
+
"manifest contains a non-integer number; the frozen "
|
|
128
|
+
"canonicalization does not define float encoding — refusing to "
|
|
129
|
+
"verify (not verified)."
|
|
130
|
+
)
|
|
131
|
+
if isinstance(value, dict):
|
|
132
|
+
for v in value.values():
|
|
133
|
+
_reject_floats(v)
|
|
134
|
+
elif isinstance(value, list):
|
|
135
|
+
for v in value:
|
|
136
|
+
_reject_floats(v)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def signing_input(doc: dict) -> bytes:
|
|
140
|
+
"""Canonical bytes of the manifest with its ``signature`` block removed."""
|
|
141
|
+
body = {k: v for k, v in doc.items() if k != "signature"}
|
|
142
|
+
return canonical_json(body)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def verify_manifest_document(doc: Any, *, info_hash: Optional[str] = None,
|
|
146
|
+
pinned: Optional[PinnedKey] = None) -> dict:
|
|
147
|
+
"""Verify a manifest document's Ed25519 signature against the pinned key.
|
|
148
|
+
|
|
149
|
+
Returns the document iff EVERYTHING checks out; otherwise raises
|
|
150
|
+
:class:`ManifestVerificationError` with a distinct reason. When
|
|
151
|
+
``info_hash`` is given, additionally requires the SIGNED document to
|
|
152
|
+
name that info hash (v1 or v2) — a validly-signed manifest for a
|
|
153
|
+
*different* torrent is a substitution, not a verification.
|
|
154
|
+
"""
|
|
155
|
+
if pinned is None:
|
|
156
|
+
pinned = load_pinned_key()
|
|
157
|
+
|
|
158
|
+
if not isinstance(doc, dict) or not isinstance(doc.get("signature"), dict):
|
|
159
|
+
raise ManifestVerificationError(
|
|
160
|
+
"the catalog's manifest is malformed (no signature block) — "
|
|
161
|
+
"not verified."
|
|
162
|
+
)
|
|
163
|
+
sig_block = doc["signature"]
|
|
164
|
+
algo = sig_block.get("algo")
|
|
165
|
+
key_id = sig_block.get("key_id")
|
|
166
|
+
sig_b64 = sig_block.get("sig")
|
|
167
|
+
|
|
168
|
+
if algo != pinned.algo:
|
|
169
|
+
raise ManifestVerificationError(
|
|
170
|
+
f"manifest signature algorithm {algo!r} is not the pinned "
|
|
171
|
+
f"{pinned.algo!r} — refusing to verify (not verified)."
|
|
172
|
+
)
|
|
173
|
+
if key_id != pinned.key_id:
|
|
174
|
+
raise ManifestVerificationError(
|
|
175
|
+
f"manifest is signed with key id {key_id!r} but this llmt build "
|
|
176
|
+
f"pins {pinned.key_id!r}. If the catalog has rotated its signing "
|
|
177
|
+
"key, a newer llmt release carries the new pinned key — upgrade "
|
|
178
|
+
"llmt via pip/your package manager. Not verified."
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
sig: Optional[bytes] = None
|
|
182
|
+
if isinstance(sig_b64, str):
|
|
183
|
+
try:
|
|
184
|
+
sig = base64.b64decode(sig_b64, validate=True)
|
|
185
|
+
except (binascii.Error, ValueError):
|
|
186
|
+
sig = None
|
|
187
|
+
if sig is None or len(sig) != _ED25519_SIG_BYTES:
|
|
188
|
+
raise ManifestVerificationError(
|
|
189
|
+
"manifest signature is not a valid base64-encoded Ed25519 "
|
|
190
|
+
"signature — not verified."
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
message = signing_input(doc)
|
|
194
|
+
try:
|
|
195
|
+
VerifyKey(pinned.public_key).verify(message, sig)
|
|
196
|
+
except BadSignatureError:
|
|
197
|
+
raise ManifestVerificationError(
|
|
198
|
+
"manifest signature is INVALID: the document does not verify "
|
|
199
|
+
"against the catalog public key pinned in this llmt build. The "
|
|
200
|
+
"manifest may be forged, tampered with, or corrupted — not "
|
|
201
|
+
"verified."
|
|
202
|
+
) from None
|
|
203
|
+
|
|
204
|
+
if info_hash:
|
|
205
|
+
torrent = doc.get("torrent")
|
|
206
|
+
torrent = torrent if isinstance(torrent, dict) else {}
|
|
207
|
+
needle = info_hash.strip().lower()
|
|
208
|
+
named = {str(torrent.get(k) or "").lower()
|
|
209
|
+
for k in ("info_hash_v1", "info_hash_v2")}
|
|
210
|
+
if needle not in named:
|
|
211
|
+
raise ManifestVerificationError(
|
|
212
|
+
"the signed manifest describes a DIFFERENT torrent (info "
|
|
213
|
+
f"hash {needle} is not named by the manifest) — possible "
|
|
214
|
+
"manifest substitution; not verified."
|
|
215
|
+
)
|
|
216
|
+
return doc
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def fetch_verified_manifest(client: Any, info_hash: Optional[str], *,
|
|
220
|
+
pinned: Optional[PinnedKey] = None) -> dict:
|
|
221
|
+
"""Fetch ``/api/manifests/{info_hash}`` and verify it. Fail closed.
|
|
222
|
+
|
|
223
|
+
``client`` is an :class:`llmt.api.ApiClient`. ANY failure — no info
|
|
224
|
+
hash, network/API error (including offline), malformed envelope, bad
|
|
225
|
+
signature — raises :class:`ManifestVerificationError`; there is no
|
|
226
|
+
quiet degradation to unauthenticated hashes.
|
|
227
|
+
"""
|
|
228
|
+
if not info_hash:
|
|
229
|
+
raise ManifestVerificationError(
|
|
230
|
+
"the catalog supplied no info hash for this download, so its "
|
|
231
|
+
"signed manifest cannot be fetched — not verified."
|
|
232
|
+
)
|
|
233
|
+
try:
|
|
234
|
+
payload = client.get_manifest(info_hash)
|
|
235
|
+
except LlmtError as exc:
|
|
236
|
+
raise ManifestVerificationError(
|
|
237
|
+
f"could not fetch the signed manifest for {info_hash} ({exc}) — "
|
|
238
|
+
"without the Ed25519-signed manifest llmt cannot authenticate "
|
|
239
|
+
"the catalog's hashes, so nothing is reported as verified. "
|
|
240
|
+
"(TLS alone is not treated as verification; if you are offline, "
|
|
241
|
+
"re-run when the catalog is reachable.)"
|
|
242
|
+
) from exc
|
|
243
|
+
doc = payload.get("data") if isinstance(payload, dict) else None
|
|
244
|
+
if not isinstance(doc, dict):
|
|
245
|
+
raise ManifestVerificationError(
|
|
246
|
+
"the catalog's manifest response is malformed (no manifest "
|
|
247
|
+
"document) — not verified."
|
|
248
|
+
)
|
|
249
|
+
return verify_manifest_document(doc, info_hash=info_hash, pinned=pinned)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def manifest_files(doc: dict) -> list[dict[str, Any]]:
|
|
253
|
+
"""Normalise a VERIFIED manifest's ``files[]`` to {path, sha256, size}.
|
|
254
|
+
|
|
255
|
+
The manifest schema requires ``path``/``size_bytes``/``sha256`` on every
|
|
256
|
+
entry; anything else is malformed and refused (fail closed).
|
|
257
|
+
"""
|
|
258
|
+
files = doc.get("files")
|
|
259
|
+
if not isinstance(files, list) or not files:
|
|
260
|
+
raise ManifestVerificationError(
|
|
261
|
+
"the signed manifest lists no files — nothing can be verified "
|
|
262
|
+
"against it (not verified)."
|
|
263
|
+
)
|
|
264
|
+
out: list[dict[str, Any]] = []
|
|
265
|
+
for f in files:
|
|
266
|
+
if (not isinstance(f, dict) or not f.get("path")
|
|
267
|
+
or not f.get("sha256")):
|
|
268
|
+
raise ManifestVerificationError(
|
|
269
|
+
"the signed manifest has a malformed files[] entry (missing "
|
|
270
|
+
"path or sha256) — not verified."
|
|
271
|
+
)
|
|
272
|
+
size = f.get("size_bytes")
|
|
273
|
+
out.append({
|
|
274
|
+
"path": f["path"],
|
|
275
|
+
"sha256": str(f["sha256"]).lower(),
|
|
276
|
+
"size": int(size)
|
|
277
|
+
if isinstance(size, int) and not isinstance(size, bool) else None,
|
|
278
|
+
})
|
|
279
|
+
return out
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def check_catalog_consistency(catalog_files: list[dict[str, Any]],
|
|
283
|
+
verified_files: list[dict[str, Any]]) -> None:
|
|
284
|
+
"""The catalog's ``download.files[]`` must agree with the SIGNED manifest.
|
|
285
|
+
|
|
286
|
+
The manifest is the authenticated source of truth; the catalog's file
|
|
287
|
+
block is unauthenticated convenience data. If they disagree, something
|
|
288
|
+
between us and the signer is lying — refuse loudly rather than attest
|
|
289
|
+
against either version.
|
|
290
|
+
"""
|
|
291
|
+
cat = {f["path"]: f for f in catalog_files if f.get("path")}
|
|
292
|
+
man = {f["path"]: f for f in verified_files}
|
|
293
|
+
problems = []
|
|
294
|
+
for path in sorted(set(cat) | set(man)):
|
|
295
|
+
c, m = cat.get(path), man.get(path)
|
|
296
|
+
if c is None:
|
|
297
|
+
problems.append(f"{path!r} is in the signed manifest but not "
|
|
298
|
+
"the catalog")
|
|
299
|
+
elif m is None:
|
|
300
|
+
problems.append(f"{path!r} is in the catalog but not the signed "
|
|
301
|
+
"manifest")
|
|
302
|
+
elif (c.get("sha256") or "").lower() != m["sha256"]:
|
|
303
|
+
problems.append(f"{path!r}: catalog sha256 differs from the "
|
|
304
|
+
"signed manifest")
|
|
305
|
+
elif (c.get("size") is not None and m["size"] is not None
|
|
306
|
+
and c["size"] != m["size"]):
|
|
307
|
+
problems.append(f"{path!r}: catalog size differs from the "
|
|
308
|
+
"signed manifest")
|
|
309
|
+
if problems:
|
|
310
|
+
raise ManifestVerificationError(
|
|
311
|
+
"the catalog's file list does not match the Ed25519-signed "
|
|
312
|
+
"manifest — refusing to verify (not verified): "
|
|
313
|
+
+ "; ".join(problems)
|
|
314
|
+
)
|
llmt/matcher.py
ADDED
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
"""Match locally-hashed files against the registry via the frozen WP-14.2
|
|
2
|
+
artifact-lookup API, and assemble seedable matches.
|
|
3
|
+
|
|
4
|
+
``POST /api/artifacts/lookup`` body ``{"sha256":[... <=100 ...]}`` returns
|
|
5
|
+
``{"data":{"results":[...]},"meta":{...}}`` with results in INPUT ORDER.
|
|
6
|
+
|
|
7
|
+
Trust model (red-team axes):
|
|
8
|
+
* **HTTPS enforced** — we refuse to send file hashes over a non-HTTPS API
|
|
9
|
+
base (except explicit loopback for local dev). A downgraded/plaintext base
|
|
10
|
+
is a MITM vector.
|
|
11
|
+
* **Local hash is the only seedability proof** — a file is only ever offered
|
|
12
|
+
for seeding when the registry reports ``known:true`` for the EXACT digest we
|
|
13
|
+
computed locally from the full file. A right-size / wrong-hash / truncated
|
|
14
|
+
file yields a different digest and therefore can never match. We never match
|
|
15
|
+
on size alone.
|
|
16
|
+
* **Response is untrusted** — shapes are validated defensively; results are
|
|
17
|
+
re-keyed by their own ``sha256`` field (not blindly by position), so a
|
|
18
|
+
reordered / duplicated / spoofed response cannot cross-wire one file's
|
|
19
|
+
digest onto another's containers. A declared ``size_bytes`` that disagrees
|
|
20
|
+
with the local file size is treated as a corruption/spoof signal and the
|
|
21
|
+
file is NOT offered.
|
|
22
|
+
* **Distribution fields are untrusted** — ``magnet`` is only carried for an
|
|
23
|
+
``available`` non-revoked container; webseed URLs are never stored or
|
|
24
|
+
surfaced (they can carry credentials). ``info_hash`` values are hex-checked.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import re
|
|
30
|
+
from dataclasses import dataclass, field
|
|
31
|
+
from typing import Any, Optional
|
|
32
|
+
from urllib.parse import parse_qsl, urlsplit
|
|
33
|
+
|
|
34
|
+
import requests
|
|
35
|
+
|
|
36
|
+
from .api import ApiClient
|
|
37
|
+
from .errors import ApiError, RateLimitedError, TimeoutErrorLlmt
|
|
38
|
+
from .scanner import HashedFile
|
|
39
|
+
|
|
40
|
+
MAX_BATCH = 100
|
|
41
|
+
_HEX64_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
42
|
+
_HEX40_RE = re.compile(r"^[0-9a-f]{40}$")
|
|
43
|
+
_VALID_STATUS = {"active", "superseded", "revoked"}
|
|
44
|
+
_LOOPBACK = {"localhost", "127.0.0.1", "::1"}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# --------------------------------------------------------------------------- #
|
|
48
|
+
# Validated response objects
|
|
49
|
+
# --------------------------------------------------------------------------- #
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class Container:
|
|
53
|
+
torrent_id: Any
|
|
54
|
+
torrent_name: str
|
|
55
|
+
path: str
|
|
56
|
+
file_name: str
|
|
57
|
+
status: str
|
|
58
|
+
available: bool
|
|
59
|
+
quants: list = field(default_factory=list)
|
|
60
|
+
magnet: Optional[str] = None
|
|
61
|
+
info_hash_v1: Optional[str] = None
|
|
62
|
+
info_hash_v2: Optional[str] = None
|
|
63
|
+
seeders: Optional[int] = None
|
|
64
|
+
leechers: Optional[int] = None
|
|
65
|
+
|
|
66
|
+
def container_dict(self) -> dict[str, Any]:
|
|
67
|
+
"""Minimal dict the frozen layout mapper consumes (path/file_name)."""
|
|
68
|
+
return {"path": self.path, "file_name": self.file_name}
|
|
69
|
+
|
|
70
|
+
def describe(self) -> str:
|
|
71
|
+
for q in self.quants:
|
|
72
|
+
if isinstance(q, dict):
|
|
73
|
+
name = q.get("model_name") or q.get("model_slug") or "?"
|
|
74
|
+
ql = q.get("quant_label") or "?"
|
|
75
|
+
fmt = q.get("format") or "?"
|
|
76
|
+
return f"{name} {ql} ({fmt})"
|
|
77
|
+
return self.torrent_name or self.file_name or "?"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass
|
|
81
|
+
class LookupEntry:
|
|
82
|
+
sha256: str
|
|
83
|
+
known: bool
|
|
84
|
+
size_bytes: Optional[int] = None
|
|
85
|
+
kind: Optional[str] = None
|
|
86
|
+
containers: list = field(default_factory=list)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass
|
|
90
|
+
class Match:
|
|
91
|
+
file: HashedFile
|
|
92
|
+
entry: LookupEntry
|
|
93
|
+
container: Container
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# --------------------------------------------------------------------------- #
|
|
97
|
+
# Defensive field cleaners
|
|
98
|
+
# --------------------------------------------------------------------------- #
|
|
99
|
+
|
|
100
|
+
def _clean_magnet(raw: Any) -> Optional[str]:
|
|
101
|
+
if not isinstance(raw, str):
|
|
102
|
+
return None
|
|
103
|
+
m = raw.strip()
|
|
104
|
+
if not m.startswith("magnet:?"):
|
|
105
|
+
return None
|
|
106
|
+
if any(c in m for c in (" ", "\n", "\r", "\t", "\x00")):
|
|
107
|
+
return None
|
|
108
|
+
if len(m) > 8192:
|
|
109
|
+
return None
|
|
110
|
+
# Untrusted server string: drop a magnet whose embedded tracker/webseed URL
|
|
111
|
+
# smuggles userinfo (user:pass@host) even though the frozen API sanitizes.
|
|
112
|
+
try:
|
|
113
|
+
for key, val in parse_qsl(urlsplit(m).query, keep_blank_values=True):
|
|
114
|
+
if key.lower() in ("tr", "ws", "xs", "as", "mt") and "://" in val:
|
|
115
|
+
if "@" in urlsplit(val).netloc:
|
|
116
|
+
return None
|
|
117
|
+
except ValueError:
|
|
118
|
+
return None
|
|
119
|
+
return m
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _clean_hash(raw: Any, *, length: int) -> Optional[str]:
|
|
123
|
+
if not isinstance(raw, str):
|
|
124
|
+
return None
|
|
125
|
+
h = raw.strip().lower()
|
|
126
|
+
if length == 40 and _HEX40_RE.match(h):
|
|
127
|
+
return h
|
|
128
|
+
if length == 64 and _HEX64_RE.match(h):
|
|
129
|
+
return h
|
|
130
|
+
return None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _int_or_none(raw: Any) -> Optional[int]:
|
|
134
|
+
if isinstance(raw, bool):
|
|
135
|
+
return None
|
|
136
|
+
if isinstance(raw, int):
|
|
137
|
+
return raw
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _parse_container(raw: Any) -> Optional[Container]:
|
|
142
|
+
if not isinstance(raw, dict):
|
|
143
|
+
return None
|
|
144
|
+
path = raw.get("path")
|
|
145
|
+
file_name = raw.get("file_name")
|
|
146
|
+
if not isinstance(path, str) and not isinstance(file_name, str):
|
|
147
|
+
return None # nothing to map a layout from
|
|
148
|
+
if not isinstance(path, str):
|
|
149
|
+
path = file_name
|
|
150
|
+
if not isinstance(file_name, str):
|
|
151
|
+
file_name = path.replace("\\", "/").rsplit("/", 1)[-1]
|
|
152
|
+
status = raw.get("status")
|
|
153
|
+
if status not in _VALID_STATUS:
|
|
154
|
+
status = "unknown"
|
|
155
|
+
available = bool(raw.get("available"))
|
|
156
|
+
quants = raw.get("quants")
|
|
157
|
+
quants = quants if isinstance(quants, list) else []
|
|
158
|
+
|
|
159
|
+
magnet = ihv1 = ihv2 = None
|
|
160
|
+
seeders = leechers = None
|
|
161
|
+
# Distribution fields exist ONLY for an available, non-revoked container.
|
|
162
|
+
if available and status != "revoked":
|
|
163
|
+
magnet = _clean_magnet(raw.get("magnet"))
|
|
164
|
+
ihv1 = _clean_hash(raw.get("info_hash_v1"), length=40)
|
|
165
|
+
ihv2 = _clean_hash(raw.get("info_hash_v2"), length=64)
|
|
166
|
+
seeders = _int_or_none(raw.get("seeders"))
|
|
167
|
+
leechers = _int_or_none(raw.get("leechers"))
|
|
168
|
+
return Container(
|
|
169
|
+
torrent_id=raw.get("torrent_id"),
|
|
170
|
+
torrent_name=str(raw.get("torrent_name") or ""),
|
|
171
|
+
path=path, file_name=file_name, status=status, available=available,
|
|
172
|
+
quants=quants, magnet=magnet, info_hash_v1=ihv1, info_hash_v2=ihv2,
|
|
173
|
+
seeders=seeders, leechers=leechers,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _parse_entry(sha: str, raw: Any) -> LookupEntry:
|
|
178
|
+
if not isinstance(raw, dict):
|
|
179
|
+
return LookupEntry(sha256=sha, known=False)
|
|
180
|
+
# Defensive: an entry must echo OUR digest. A mismatch means the response is
|
|
181
|
+
# reordered/spoofed for this slot -> treat as unknown (never cross-wire).
|
|
182
|
+
got = raw.get("sha256")
|
|
183
|
+
if not isinstance(got, str) or got.strip().lower() != sha:
|
|
184
|
+
return LookupEntry(sha256=sha, known=False)
|
|
185
|
+
if raw.get("valid") is False:
|
|
186
|
+
return LookupEntry(sha256=sha, known=False)
|
|
187
|
+
if not raw.get("known"):
|
|
188
|
+
return LookupEntry(sha256=sha, known=False)
|
|
189
|
+
size = raw.get("size_bytes")
|
|
190
|
+
size = size if isinstance(size, int) and not isinstance(size, bool) else None
|
|
191
|
+
containers = []
|
|
192
|
+
for c in raw.get("containers") or []:
|
|
193
|
+
parsed = _parse_container(c)
|
|
194
|
+
if parsed is not None:
|
|
195
|
+
containers.append(parsed)
|
|
196
|
+
return LookupEntry(
|
|
197
|
+
sha256=sha, known=True, size_bytes=size,
|
|
198
|
+
kind=(raw.get("kind") if isinstance(raw.get("kind"), str) else None),
|
|
199
|
+
containers=containers,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
# --------------------------------------------------------------------------- #
|
|
204
|
+
# HTTPS enforcement + POST
|
|
205
|
+
# --------------------------------------------------------------------------- #
|
|
206
|
+
|
|
207
|
+
def require_https(base: str) -> None:
|
|
208
|
+
parts = urlsplit(base)
|
|
209
|
+
host = (parts.hostname or "").lower()
|
|
210
|
+
if parts.scheme == "https":
|
|
211
|
+
return
|
|
212
|
+
if parts.scheme == "http" and host in _LOOPBACK:
|
|
213
|
+
return
|
|
214
|
+
raise ApiError(
|
|
215
|
+
f"refusing to send file hashes over a non-HTTPS API base: {base!r}. "
|
|
216
|
+
f"Use an https:// endpoint (MITM/spoofing defense).",
|
|
217
|
+
code="insecure_api_base",
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _post_lookup(client: ApiClient, chunk: list[str]) -> dict[str, Any]:
|
|
222
|
+
url = f"{client.base}/artifacts/lookup"
|
|
223
|
+
headers = {
|
|
224
|
+
"Accept": "application/json",
|
|
225
|
+
"Content-Type": "application/json",
|
|
226
|
+
"User-Agent": "llmt-cli",
|
|
227
|
+
}
|
|
228
|
+
if client.key:
|
|
229
|
+
headers["Authorization"] = f"Bearer {client.key}"
|
|
230
|
+
try:
|
|
231
|
+
resp = client.session.post(
|
|
232
|
+
url, json={"sha256": chunk}, headers=headers, timeout=client.timeout,
|
|
233
|
+
)
|
|
234
|
+
except requests.Timeout as exc:
|
|
235
|
+
raise TimeoutErrorLlmt(f"artifact lookup timed out: {url}") from exc
|
|
236
|
+
except requests.RequestException as exc:
|
|
237
|
+
raise ApiError(f"artifact lookup failed: {exc}") from exc
|
|
238
|
+
|
|
239
|
+
if resp.status_code == 422:
|
|
240
|
+
raise ApiError("artifact lookup rejected the batch (batch_too_large).",
|
|
241
|
+
code="batch_too_large", status=422)
|
|
242
|
+
if resp.status_code == 429:
|
|
243
|
+
# Scan F1: a 429 is a typed RateLimitedError (honors Retry-After),
|
|
244
|
+
# aligned with the telemetry write-API's 429 handling.
|
|
245
|
+
ra = resp.headers.get("Retry-After")
|
|
246
|
+
retry_after = None
|
|
247
|
+
if ra is not None:
|
|
248
|
+
try:
|
|
249
|
+
retry_after = int(ra)
|
|
250
|
+
except ValueError:
|
|
251
|
+
retry_after = None
|
|
252
|
+
raise RateLimitedError(
|
|
253
|
+
"artifact lookup rate-limited (429).",
|
|
254
|
+
retry_after=retry_after, code="rate_limited", status=429)
|
|
255
|
+
if resp.status_code != 200:
|
|
256
|
+
raise ApiError(f"artifact lookup returned HTTP {resp.status_code}.",
|
|
257
|
+
status=resp.status_code)
|
|
258
|
+
try:
|
|
259
|
+
payload = resp.json()
|
|
260
|
+
except ValueError as exc:
|
|
261
|
+
raise ApiError("artifact lookup returned non-JSON.",
|
|
262
|
+
code="bad_response") from exc
|
|
263
|
+
return payload
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _merge(results: dict[str, LookupEntry], chunk: list[str],
|
|
267
|
+
payload: dict[str, Any]) -> None:
|
|
268
|
+
data = payload.get("data") if isinstance(payload, dict) else None
|
|
269
|
+
res = data.get("results") if isinstance(data, dict) else None
|
|
270
|
+
if not isinstance(res, list):
|
|
271
|
+
raise ApiError("malformed lookup response: missing data.results[].",
|
|
272
|
+
code="bad_response")
|
|
273
|
+
# Results are contractually in input order, but we re-key by each entry's
|
|
274
|
+
# OWN sha256 so a reordered/duplicated response cannot mis-assign. Any
|
|
275
|
+
# requested sha with no self-consistent entry stays unknown.
|
|
276
|
+
by_sha: dict[str, Any] = {}
|
|
277
|
+
for item in res:
|
|
278
|
+
if isinstance(item, dict):
|
|
279
|
+
s = item.get("sha256")
|
|
280
|
+
if isinstance(s, str) and _HEX64_RE.match(s.strip().lower()):
|
|
281
|
+
by_sha.setdefault(s.strip().lower(), item)
|
|
282
|
+
for sha in chunk:
|
|
283
|
+
results[sha] = _parse_entry(sha, by_sha.get(sha))
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def lookup_shas(client: ApiClient, shas: list[str]) -> dict[str, LookupEntry]:
|
|
287
|
+
"""Look up digests in chunks of <=100. Returns ``{sha256 -> LookupEntry}``."""
|
|
288
|
+
require_https(client.base)
|
|
289
|
+
uniq: list[str] = []
|
|
290
|
+
seen: set[str] = set()
|
|
291
|
+
for s in shas:
|
|
292
|
+
if isinstance(s, str):
|
|
293
|
+
sl = s.strip().lower()
|
|
294
|
+
if _HEX64_RE.match(sl) and sl not in seen:
|
|
295
|
+
seen.add(sl)
|
|
296
|
+
uniq.append(sl)
|
|
297
|
+
results: dict[str, LookupEntry] = {}
|
|
298
|
+
for i in range(0, len(uniq), MAX_BATCH):
|
|
299
|
+
chunk = uniq[i:i + MAX_BATCH]
|
|
300
|
+
_merge(results, chunk, _post_lookup(client, chunk))
|
|
301
|
+
return results
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
# --------------------------------------------------------------------------- #
|
|
305
|
+
# Match assembly (the seed/refuse gate)
|
|
306
|
+
# --------------------------------------------------------------------------- #
|
|
307
|
+
|
|
308
|
+
#: The ONLY catalog statuses ever offered for seeding. Any other value —
|
|
309
|
+
#: ``revoked``, or an unrecognized/garbage status normalized to ``unknown`` by
|
|
310
|
+
#: ``_parse_container`` — FAILS CLOSED and is never seeded. Untrusted status
|
|
311
|
+
#: must not open a seeding path.
|
|
312
|
+
_SEEDABLE_STATUS = ("active", "superseded")
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def choose_container(entry: LookupEntry) -> Optional[Container]:
|
|
316
|
+
"""Pick the best seedable container: available, has a magnet, and a status
|
|
317
|
+
we explicitly recognize as seedable. Prefer ``active`` over ``superseded``.
|
|
318
|
+
Revoked / unavailable / unknown-status containers are never returned."""
|
|
319
|
+
seedable = [c for c in entry.containers
|
|
320
|
+
if c.available and c.status in _SEEDABLE_STATUS and c.magnet]
|
|
321
|
+
for pref in _SEEDABLE_STATUS:
|
|
322
|
+
for c in seedable:
|
|
323
|
+
if c.status == pref:
|
|
324
|
+
return c
|
|
325
|
+
return None
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def assemble_matches(
|
|
329
|
+
hashed: list[HashedFile], results: dict[str, LookupEntry],
|
|
330
|
+
) -> tuple[list[Match], list[HashedFile], list[tuple[HashedFile, LookupEntry]]]:
|
|
331
|
+
"""Split scanned files into (matches, unmatched, size-disagreements).
|
|
332
|
+
|
|
333
|
+
A Match requires ALL of:
|
|
334
|
+
* registry ``known:true`` keyed by the local full-file digest,
|
|
335
|
+
* the entry's echoed digest equals the local digest,
|
|
336
|
+
* declared ``size_bytes`` (if any) equals the local file size,
|
|
337
|
+
* a seedable container (available, non-revoked, valid magnet).
|
|
338
|
+
Anything else is never offered for seeding.
|
|
339
|
+
"""
|
|
340
|
+
matches: list[Match] = []
|
|
341
|
+
unmatched: list[HashedFile] = []
|
|
342
|
+
size_bad: list[tuple[HashedFile, LookupEntry]] = []
|
|
343
|
+
for hf in hashed:
|
|
344
|
+
entry = results.get(hf.sha256)
|
|
345
|
+
if entry is None or not entry.known or entry.sha256 != hf.sha256:
|
|
346
|
+
unmatched.append(hf)
|
|
347
|
+
continue
|
|
348
|
+
if entry.size_bytes is not None and entry.size_bytes != hf.size:
|
|
349
|
+
# Digest matched but the server's declared size disagrees with the
|
|
350
|
+
# bytes on disk: refuse (spoof / metadata-corruption signal).
|
|
351
|
+
size_bad.append((hf, entry))
|
|
352
|
+
continue
|
|
353
|
+
container = choose_container(entry)
|
|
354
|
+
if container is None:
|
|
355
|
+
unmatched.append(hf) # known but nothing seedable (revoked/gated)
|
|
356
|
+
continue
|
|
357
|
+
matches.append(Match(file=hf, entry=entry, container=container))
|
|
358
|
+
return matches, unmatched, size_bad
|