llmt-cli 0.2.0__py3-none-any.whl

llmt/scancmd.py

@@ -0,0 +1,556 @@
+"""``llmt scan`` — "you already have these models; seed them."
+
+Flow:
+  1. Resolve scan roots (auto-detected known layouts, or explicit --path).
+  2. Prefilter + hash local candidate files (full-file SHA-256).
+  3. Look them up against the registry (POST /api/artifacts/lookup, <=100/chunk).
+  4. Offer only catalog-matched, seedable matches (the local file's
+     full-file SHA-256 equals a SHA-256 the catalog returned over the
+     TLS lookup — this is NOT an Ed25519 signed-manifest check).
+  5. On accept: plan the layout (link farm, no copy/move) via the frozen
+     plan_layout/apply_plan, then add to the user's client (qBit/Transmission/
+     fallback) with the add-paused -> recheck -> resume recipe.
+
+No telemetry, no ledger, no account, no heartbeat (DR-27): the only network the
+MVP performs is the anonymous public lookup plus whatever a normal BitTorrent
+client does once the torrent is added.
+
+Multi-file containers (M5, size-aware Option C, Pat 2026-07-13): before a
+multi-file torrent is handed to a client, scan reads the torrent's full file
+list from its Ed25519-signed manifest and checks which files are already
+present at the layout. Missing license/README-class siblings (<= 10 MiB
+cumulatively per container) stay wanted and are fetched by the client, with a
+one-line notice naming them and the directory they land in; any larger
+missing file is marked do-not-download (qBt filePrio 0 / Transmission
+files-unwanted) BEFORE the torrent can resume, with a warning that the
+torrent will seed only what is present and will show as incomplete. Model
+weights are never re-downloaded.
+
+Trust level (honest scoping): a scan "match" means the local file's
+full-file SHA-256 equals a SHA-256 the catalog returned over the TLS-only,
+unauthenticated lookup. Unlike ``pull``/``verify``/``seed`` (Fix M2), scan
+runs NO Ed25519 signed-manifest verification, so it reports matches as
+"catalog-matched", never "verified". Signed-manifest gating for ``scan``
+is a roadmap item.
+
+Exit status (D15, Fix 11): scan processes every accepted container to
+completion ("make -k") -- a handoff that fails or cannot be verified
+(``AddResult.ok`` False, or a ``ClientError`` raised for one container) is
+recorded and does NOT abort the others. Each ``--json`` ``added[]`` record
+carries ``handoff_ok`` (True = confirmed seeding, False = failed or not
+confirmed, null = FallbackClient manual instructions) plus
+``handoff_message``; a
+top-level ``handoff_summary`` counts each state. If ANY handoff failed the
+whole run exits 14 (the client-error code); an all-success and/or
+fallback-only run exits 0.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from typing import Optional
+
+from . import scanner, matcher
+from .api import ApiClient, DEFAULT_API_BASE
+from .clients import ClientError, FileSelection, SeedClient, build_client
+from .errors import LayoutError, ManifestVerificationError
+from .layout import (
+    CLIENT_LINK_SUPPORT,
+    DEFAULT_BASE_DIR,
+    M5_AUTO_FETCH_MAX_BYTES,
+    apply_plan,
+    classify_missing_files,
+    plan_layout,
+    safe_join,
+    sanitize_relpath,
+)
+from .manifest import fetch_verified_manifest, manifest_files
+from .output import emit_json, human_size, safe_text
+
+
+# --------------------------------------------------------------------------- #
+# argparse wiring (called from cli.build_parser)
+# --------------------------------------------------------------------------- #
+
+def _size_arg(raw: str) -> int:
+    s = raw.strip().upper()
+    mult = 1
+    for suf, m in (("K", 1024), ("M", 1024**2), ("G", 1024**3), ("T", 1024**4)):
+        if s.endswith(suf) or s.endswith(suf + "B"):
+            mult = m
+            s = s.rstrip("B").rstrip(suf)
+            break
+    try:
+        return int(float(s) * mult)
+    except ValueError:
+        raise ValueError(f"invalid size: {raw!r}")
+
+
+def add_scan_args(sp) -> None:
+    sp.add_argument("--path", action="append", default=[],
+                    help="Explicit directory to scan (repeatable). If given, "
+                         "only these paths are scanned (bounds scope). "
+                         "Otherwise known layouts are auto-detected.")
+    sp.add_argument("--min-size", type=_size_arg, default=scanner.DEFAULT_MIN_SIZE,
+                    help="Prefilter: ignore files smaller than this "
+                         "(e.g. 100M, 2G). Default 1M.")
+    sp.add_argument("--all-files", action="store_true",
+                    help="Hash every file passing the size filter, not just "
+                         "known model extensions.")
+    sp.add_argument("--max-depth", type=int, default=scanner.DEFAULT_MAX_DEPTH,
+                    help="Max directory recursion depth (loop/DoS guard).")
+    sp.add_argument("--layout-base", default=None,
+                    help="Root for generated seeding link farms "
+                         "(default ~/.llmt/seeding-layouts). Must be a path "
+                         "your torrent client can see: a dockerized/remote "
+                         "client (seedbox/unRAID) only sees its mounted "
+                         "volumes, so point this at one of them or the "
+                         "handoff rechecks empty and fails closed.")
+    sp.add_argument("--client", choices=["qbittorrent", "transmission", "none"],
+                    default=None,
+                    help="Torrent client to add matches to. Default: auto "
+                         "(qBit/Transmission if a URL is configured, else "
+                         "fallback to printing the magnet).")
+    sp.add_argument("--qb-url", default=None, help="qBittorrent WebUI base URL "
+                    "(env LLMT_QB_URL).")
+    sp.add_argument("--qb-user", default=None, help="qBittorrent user (env "
+                    "LLMT_QB_USER).")
+    sp.add_argument("--qb-pass", default=None, help="qBittorrent password (env "
+                    "LLMT_QB_PASS).")
+    sp.add_argument("--tr-url", default=None, help="Transmission RPC URL (env "
+                    "LLMT_TR_URL).")
+    sp.add_argument("--tr-user", default=None, help="Transmission user (env "
+                    "LLMT_TR_USER).")
+    sp.add_argument("--tr-pass", default=None, help="Transmission password (env "
+                    "LLMT_TR_PASS).")
+    sp.add_argument("--yes", "-y", action="store_true",
+                    help="Non-interactive: accept and seed every match.")
+    sp.add_argument("--dry-run", action="store_true",
+                    help="Show matches and the plan; create no links and add "
+                         "nothing to any client.")
+
+
+# --------------------------------------------------------------------------- #
+# helpers
+# --------------------------------------------------------------------------- #
+
+def _resolve_roots(args) -> list[tuple[str, str]]:
+    if args.path:
+        return [(os.path.abspath(os.path.expanduser(p)), scanner.LAYOUT_PATH)
+                for p in args.path]
+    return scanner.default_roots()
+
+
+def _layout_client_hint(client: SeedClient) -> str:
+    return client.name if client.name in CLIENT_LINK_SUPPORT else "generic"
+
+
+def _client_visible_path(plan) -> str:
+    """Per the frozen layout contract, the path the client will present."""
+    if plan.mode == "link-farm":
+        for act in reversed(plan.actions):
+            if act.kind in ("hardlink", "symlink"):
+                return act.target
+    return plan.source_file
+
+
+def _fmt_files(pairs) -> str:
+    return ", ".join(f"{safe_text(path)} ({human_size(size)})"
+                     for path, size in pairs)
+
+
+def _plan_file_selection(api, pairs):
+    """M5 (size-aware Option C, Pat 2026-07-13): classify a container's files.
+
+    ``pairs`` is ``[(match, plan), ...]`` -- ALL accepted matches that belong
+    to ONE container (same info hash), sharing one save path.  The matcher
+    yields one Match per LOCAL FILE, so a container the user holds several
+    files of arrives here as several pairs; every one of them counts as
+    PRESENT (review fix for 6d548a2: previously each match was classified in
+    isolation, so a user-held sibling was falsely marked missing and the same
+    info-hash was added once per match).
+
+    For a multi-file torrent (the sanitised container path has a root
+    folder), fetch the torrent's full file list from its Ed25519-SIGNED
+    manifest, determine which files are already present at the layout, and
+    classify the missing ones via :func:`llmt.layout.classify_missing_files`.
+
+    Returns ``(selection, missing_report, human_lines)``:
+      * ``selection``      -- a :class:`FileSelection` for the client
+                              adapters, or None when nothing is missing
+                              (fully-present container / single-file torrent
+                              -- those flows are UNCHANGED, no manifest
+                              fetch happens for single-file torrents);
+      * ``missing_report`` -- ``{"fetched": [...], "skipped": [...]}`` with
+                              path + size_bytes for ``--json``;
+      * ``human_lines``    -- the notice / warning lines for humans.
+
+    Fail closed: if the manifest cannot be fetched/verified, lists any path
+    twice (after normalization -- ambiguous bookkeeping), or does not list
+    every matched file with its local digest and size, the container is
+    refused (ManifestVerificationError) rather than handed to a client that
+    would download unknown siblings.
+    """
+    empty = {"fetched": [], "skipped": []}
+    m0, plan0 = pairs[0]
+    if not plan0.torrent_root:
+        # Single-file torrent: no siblings can exist -- unchanged flow.
+        return None, empty, []
+
+    container = m0.container
+    doc = fetch_verified_manifest(api, container.info_hash_v1)
+    files = manifest_files(doc)
+
+    def norm(path):
+        return "/".join(sanitize_relpath(path))
+
+    by_path = {}
+    for f in files:
+        rel = norm(f["path"])
+        if rel in by_path:
+            raise ManifestVerificationError(
+                f"the Ed25519-signed manifest for torrent "
+                f"{container.info_hash_v1} lists {rel!r} more than once "
+                "(after path normalization) -- ambiguous file list; refusing "
+                "to seed this container (nothing was added).")
+        by_path[rel] = f
+
+    matched = set()
+    for m, plan in pairs:
+        entry = by_path.get(plan.in_torrent_path)
+        if (entry is None or entry["sha256"] != m.file.sha256
+                or (entry["size"] is not None
+                    and entry["size"] != m.file.size)):
+            raise ManifestVerificationError(
+                f"the Ed25519-signed manifest for torrent "
+                f"{container.info_hash_v1} does not list the matched file "
+                f"{plan.in_torrent_path!r} with the locally-computed digest "
+                "and size -- catalog/manifest disagreement; refusing to seed "
+                "this container (nothing was added).")
+        matched.add(plan.in_torrent_path)
+
+    missing = []
+    for rel, f in by_path.items():
+        if rel in matched:
+            continue  # a user-held, catalog-matched file of this container
+        expected = safe_join(plan0.save_path, sanitize_relpath(f["path"]))
+        present = (os.path.isfile(expected)
+                   and (f["size"] is None
+                        or os.path.getsize(expected) == f["size"]))
+        if not present:
+            missing.append((rel, f["size"]))
+
+    if not missing:
+        # Fully-present multi-file container: unchanged flow, no warning.
+        return None, empty, []
+
+    fetch, skip = classify_missing_files(missing)
+    selection = FileSelection(
+        unwanted=tuple(path for path, _ in skip),
+        expect_fetch=tuple(path for path, _ in fetch),
+    )
+    report = {
+        "fetched": [{"path": path, "size_bytes": size}
+                    for path, size in fetch],
+        "skipped": [{"path": path, "size_bytes": size}
+                    for path, size in skip],
+    }
+    land_dir = os.path.join(plan0.save_path, plan0.torrent_root)
+    cap = human_size(M5_AUTO_FETCH_MAX_BYTES)
+    lines = []
+    if fetch:
+        total = human_size(sum(size or 0 for _, size in fetch))
+        lines.append(
+            f"note: {len(fetch)} small companion file(s) missing locally "
+            f"will be fetched by your client into {land_dir}: "
+            f"{_fmt_files(fetch)} ({total} total, <={cap} cap) -- model "
+            f"weights are never re-downloaded.")
+    if skip:
+        lines.append(
+            f"WARNING: {len(skip)} missing file(s) exceed the {cap} "
+            f"auto-fetch cap and will NOT be downloaded: "
+            f"{_fmt_files(skip)}. They are set to do-not-download before "
+            f"the torrent can resume; it will seed only the files you have "
+            f"and will show as incomplete in your client.")
+    return selection, report, lines
+
+
+# --------------------------------------------------------------------------- #
+# command
+# --------------------------------------------------------------------------- #
+
+def cmd_scan(args) -> int:
+    return run_scan(args)
+
+
+def run_scan(args, *, client: Optional[SeedClient] = None,
+             api_client: Optional[ApiClient] = None) -> int:
+    as_json = getattr(args, "json", False)
+
+    def note(msg: str) -> None:
+        if not as_json:
+            print(msg, file=sys.stderr)
+
+    roots = _resolve_roots(args)
+    if not roots:
+        if as_json:
+            emit_json({"scanned": 0, "matches": [], "note": "no roots"})
+        else:
+            print("No model directories found. Pass --path DIR to scan a "
+                  "specific location.", file=sys.stderr)
+        return 0
+
+    note("Scanning: " + ", ".join(r for r, _ in roots))
+    candidates = list(scanner.iter_candidates(
+        roots, min_size=args.min_size, all_files=args.all_files,
+        max_depth=args.max_depth,
+    ))
+    note(f"Found {len(candidates)} candidate file(s) to hash "
+         f"({human_size(sum(c.size for c in candidates))}).")
+
+    labels: dict[str, str] = {}
+    for root, layout in roots:
+        if layout == scanner.LAYOUT_OLLAMA:
+            labels.update(scanner.ollama_labels(root))
+
+    def progress(i: int, total: int, path: str) -> None:
+        note(f"  [hash {i}/{total}] {os.path.basename(path)}")
+
+    hashed = scanner.hash_candidates(candidates, labels=labels, progress=progress)
+
+    api = api_client or ApiClient(base=args.api_base, key=args.api_key,
+                                  timeout=args.timeout)
+    results = matcher.lookup_shas(api, [h.sha256 for h in hashed])
+    matches, unmatched, size_bad = matcher.assemble_matches(hashed, results)
+
+    if as_json:
+        emit_json({
+            "scanned": len(candidates),
+            "hashed": len(hashed),
+            "matches": [_match_json(m) for m in matches],
+            "unmatched": len(unmatched),
+            "size_mismatch": [
+                {"path": hf.path, "local_size": hf.size,
+                 "declared_size": e.size_bytes} for hf, e in size_bad],
+        })
+    else:
+        _print_report(matches, hashed, unmatched, size_bad)
+
+    if not matches:
+        return 0
+
+    # --- accept / confirm -------------------------------------------------- #
+    if args.dry_run or args.yes:
+        accepted = matches
+    else:
+        # Never prompt in --json mode: input()'s prompt would corrupt the JSON
+        # stream on stdout. Require explicit --yes/--dry-run instead.
+        if as_json or not sys.stdin.isatty():
+            note("Refusing to prompt on a non-interactive/JSON invocation; "
+                 "re-run with --yes to seed or --dry-run to preview.")
+            return 0
+        try:
+            ans = input(f"Seed these {len(matches)} file(s)? [y/N] ").strip().lower()
+        except EOFError:
+            ans = ""
+        if ans not in ("y", "yes"):
+            print("Aborted; nothing was added.")
+            return 0
+        accepted = matches
+
+    client = client or build_client(args)
+    if not as_json:
+        print(f"\nUsing client: {client.name}"
+              + (" (dry-run)" if args.dry_run else ""))
+
+    # S3 pre-flight: warn early if the client looks path-blind (dockerized or
+    # remote) so a 0% recheck below is not the FIRST hint the user gets.
+    # Advisory only -- it never blocks; the per-container recheck stays the
+    # authoritative, fail-closed gate.  Skipped under --json (its advisory
+    # goes to stderr only, so the extra login+preferences round-trip would be
+    # discarded).
+    if not args.dry_run and not as_json:
+        probe_fn = getattr(client, "probe_local_visibility", None)
+        if callable(probe_fn):
+            probe = probe_fn()
+            if probe.supported and probe.visible_locally is False:
+                eff_base = os.path.abspath(os.path.expanduser(
+                    args.layout_base or DEFAULT_BASE_DIR))
+                note(
+                    "WARNING: your torrent client's own default save path "
+                    f"({probe.client_default_path}) does not exist on the "
+                    "machine llmt is running on -- llmt and the client look "
+                    "like they are on different filesystems (a dockerized "
+                    "seedbox/unRAID or a remote client). llmt's link farm at "
+                    f"{eff_base} is a path on THIS machine, so the client may "
+                    "not be able to see it and the seed handoff below would "
+                    "recheck to 0% and fail closed. If that happens, re-run "
+                    "with --layout-base pointing at a directory the client "
+                    "can see (one of its mounted volumes).")
+
+    # M5 (review fix for 6d548a2): the matcher yields one Match per LOCAL
+    # FILE, so a container the user holds several files of appears as several
+    # matches. Group by torrent identity and hand each torrent to the client
+    # exactly ONCE, with every user-held file counted as present -- adding
+    # the same info-hash once per match double-adds (qBittorrent answers
+    # HTTP 200 "Fails.", Transmission returns torrent-duplicate) and falsely
+    # classifies later-matched siblings as missing.
+    groups: "dict[str, list]" = {}
+    for m in accepted:
+        key = (m.container.info_hash_v1 or m.container.magnet
+               or f"anon-{id(m.container)}")
+        groups.setdefault(key, []).append(m)
+
+    add_reports = []
+    for group in groups.values():
+        pairs = []
+        for m in group:
+            plan = plan_layout(
+                m.file.path, m.container.container_dict(),
+                base_dir=args.layout_base, client=_layout_client_hint(client),
+            )
+            pairs.append((m, plan))
+        save_paths = sorted({plan.save_path for _, plan in pairs})
+        if len(save_paths) > 1:
+            # One torrent cannot be added at two save paths (mixed
+            # direct-use/link-farm layouts across this container's matched
+            # files). Fail closed with an honest error instead of guessing.
+            raise LayoutError(
+                f"the {len(pairs)} matched files of "
+                f"'{safe_text(pairs[0][0].container.describe())}' resolve to "
+                "different save paths (mixed direct-use/link-farm layouts): "
+                + ", ".join(save_paths) + " -- one torrent cannot be added "
+                "at two save paths, so nothing was added for this container. "
+                "Keep those files under a single layout and re-run.")
+        m0, plan0 = pairs[0]
+        # M5: classify missing siblings BEFORE the client handoff so large
+        # missing files are do-not-download before any resume is possible;
+        # every matched file of this container counts as present.
+        selection, missing_report, m5_lines = _plan_file_selection(api, pairs)
+        logs = [apply_plan(plan, dry_run=args.dry_run) for _, plan in pairs]
+        # D15 (Fix 11): the handoff is the only per-container failure
+        # point. "make -k": a failed/refused handoff (ok=False returned, or
+        # ClientError raised) is recorded and MUST NOT abort the remaining
+        # containers. handoff_ok is tri-state -- True = confirmed seeding,
+        # False = failed/unverified (forces a non-zero exit), None =
+        # FallbackClient manual instructions (told the user what to do; not
+        # a success, not a failure, no non-zero exit for a fallback alone).
+        res = None
+        handoff_message = ""
+        method = client.name
+        try:
+            res = client.add(
+                magnet=m0.container.magnet,
+                info_hash=m0.container.info_hash_v1,
+                save_path=plan0.save_path,
+                torrent_name=m0.container.torrent_name,
+                dry_run=args.dry_run, file_selection=selection,
+            )
+        except ClientError as exc:
+            handoff_ok = False
+            handoff_message = str(exc)
+        else:
+            method = res.method
+            if res.method == "fallback-magnet":
+                handoff_ok = None
+                handoff_message = res.message
+            else:
+                handoff_ok = bool(res.ok)
+                handoff_message = res.message
+        add_reports.append({
+            "pairs": pairs, "logs": logs, "res": res,
+            "missing_report": missing_report, "method": method,
+            "handoff_ok": handoff_ok, "handoff_message": handoff_message,
+        })
+        if not as_json:
+            print(f"\n  {safe_text(m0.container.describe())}")
+            for m, _plan in pairs:
+                print(f"    file : {m.file.path}")
+            print(f"    plan : {plan0.mode} -> save_path {plan0.save_path}")
+            for line in m5_lines:
+                print(f"    {line}")
+            if res is not None:
+                for line in res.steps:
+                    print(f"    - {line}")
+                if res.method == "fallback-magnet":
+                    print(f"    magnet: {res.message}")
+            if handoff_ok is False:
+                print(f"    [HANDOFF FAILED] not seeding: {handoff_message}")
+            elif handoff_ok is None:
+                print("    [MANUAL] added, but you must finish the handoff "
+                      "shown above; NOT yet seeding")
+
+    n_ok = sum(1 for r in add_reports if r["handoff_ok"] is True)
+    n_manual = sum(1 for r in add_reports if r["handoff_ok"] is None)
+    n_failed = sum(1 for r in add_reports if r["handoff_ok"] is False)
+
+    if as_json:
+        emit_json({
+            "added": [{
+                "sha256": r["pairs"][0][0].file.sha256,
+                "path": r["pairs"][0][0].file.path,
+                "matched_files": [
+                    {"sha256": m.file.sha256, "path": m.file.path}
+                    for m, _plan in r["pairs"]],
+                "mode": r["pairs"][0][1].mode,
+                "save_path": r["pairs"][0][1].save_path,
+                "client": r["method"],
+                "steps": r["res"].steps if r["res"] is not None else [],
+                "dry_run": (r["res"].dry_run if r["res"] is not None
+                             else bool(args.dry_run)),
+                "missing_files": r["missing_report"],
+                "handoff_ok": r["handoff_ok"],
+                "handoff_message": r["handoff_message"],
+            } for r in add_reports],
+            "handoff_summary": {
+                "ok": n_ok, "manual": n_manual, "failed": n_failed},
+        })
+    else:
+        parts = [f"{n_ok} confirmed seeding"]
+        if n_manual:
+            parts.append(
+                f"{n_manual} awaiting manual action (not yet seeding)")
+        if n_failed:
+            parts.append(f"{n_failed} FAILED (not seeding)")
+        print("\nHandoff summary: " + ", ".join(parts) + ".")
+
+    # D15: any failed/unverified handoff makes the whole run exit non-zero
+    # (reuse the client-error code 14). Manual (fallback) and all-ok do not.
+    return ClientError.exit_code if n_failed else 0
+
+
+def _match_json(m: matcher.Match) -> dict:
+    return {
+        "sha256": m.file.sha256,
+        "path": m.file.path,
+        "size": m.file.size,
+        "label": m.file.label,
+        "layout": m.file.layout,
+        "model": m.container.describe(),
+        "torrent_name": m.container.torrent_name,
+        "status": m.container.status,
+    }
+
+
+def _print_report(matches, hashed, unmatched, size_bad) -> None:
+    print()
+    if size_bad:
+        print("Refused (size disagreement between disk and registry — not "
+              "offered):")
+        for hf, entry in size_bad:
+            print(f"  ! {hf.path} (on disk {human_size(hf.size)}, registry "
+                  f"declared {human_size(entry.size_bytes)})")
+        print()
+    if not matches:
+        print(f"Scanned {len(hashed)} file(s); no seedable catalog matches "
+              f"(no local file's full-file SHA-256 matched the catalog).")
+        return
+    print(f"You already have {len(matches)} model file(s) whose full-file "
+          f"SHA-256 matches the catalog (TLS lookup, not an Ed25519 "
+          f"signature). Seed these?")
+    for i, m in enumerate(matches, 1):
+        lbl = f"  ({safe_text(m.file.label)})" if m.file.label else ""
+        print(f"  {i}. {safe_text(m.container.describe())}{lbl}")
+        print(f"       {m.file.path}  [{human_size(m.file.size)}]")
+    print()