llmt-cli 0.2.0__py3-none-any.whl

llmt/scanner.py

@@ -0,0 +1,328 @@
+"""Local model-file scanner for ``llmt scan`` (WP-14.4-S).
+
+Two-phase design:
+
+1. **Candidate gathering** — walk a set of scan roots, fingerprinting known
+   model layouts, applying a *size + extension prefilter* so we never hash 4 TB
+   of unrelated data. Gathering is cheap (stat only).
+2. **Hashing** — full-file streaming SHA-256 over the (much smaller) candidate
+   set, with progress. The full-file hash is the ONLY seedability signal the
+   downstream matcher trusts: a truncated / wrong-size / corrupt file simply
+   produces a different digest and can never be "matched".
+
+Hostile-filesystem hardening (red-team axis):
+  * Symlinks are never followed — neither symlinked directories (no traversal
+    out of the scan roots, no symlink loops) nor symlinked files (we skip them
+    entirely rather than hash whatever they point at).
+  * Only regular files are ever opened: device nodes, FIFOs, sockets are
+    skipped (``entry.is_file(follow_symlinks=False)``).
+  * Recursion depth is bounded (``max_depth``) and visited directories are
+    de-duplicated by ``(st_dev, st_ino)`` to survive bind-mount / hardlink loops.
+  * Every candidate's real path is re-checked to still live under the root's
+    real path (defence in depth on top of the symlink skip).
+
+Layout honesty:
+  * ``ollama``  — we genuinely parse the manifest tree to map blob digests
+    (blobs are named ``sha256-<digest>``, NOT ``*.gguf``) to a friendly
+    ``model:tag`` label. Blobs are matched by digest.
+  * ``lmstudio`` / ``text-generation-webui`` / ``generic`` — we detect the root
+    but scan it *generically* by model file-extension; we do not parse their
+    per-tool metadata. This is documented as best-effort.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import stat
+from dataclasses import dataclass
+from typing import Callable, Iterator, Optional
+
+_CHUNK = 64 * 1024  # streaming SHA-256 block size — the trust core
+
+
+def _safe_sha256(path: str) -> str:
+    """Full-file SHA-256 with an OPEN-TIME re-check that closes the TOCTOU gap
+    between the candidate walk (which vets each entry as a regular file) and the
+    hashing pass (which re-opens by path). A hostile writer in the scan root
+    could, in that window, swap a vetted regular file for a symlink (-> we would
+    hash and leak the digest of whatever it points at) or a FIFO (-> a blocking
+    read would hang the scan forever). We defeat both at open() time:
+
+      * ``O_NOFOLLOW`` — a symlink swapped in is refused (ELOOP), never followed.
+      * ``O_NONBLOCK`` — opening a FIFO with no writer returns immediately
+        instead of blocking; we then reject it by type.
+      * ``fstat`` + ``S_ISREG`` on the OPEN fd — only a genuine regular file is
+        ever read; anything else raises OSError (the caller skips it).
+    """
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
+    fd = os.open(path, flags)
+    try:
+        st = os.fstat(fd)
+        if not stat.S_ISREG(st.st_mode):
+            raise OSError(f"not a regular file at hash time: {path!r}")
+        h = hashlib.sha256()
+        while True:
+            chunk = os.read(fd, _CHUNK)
+            if not chunk:
+                break
+            h.update(chunk)
+        return h.hexdigest()
+    finally:
+        os.close(fd)
+
+DEFAULT_MIN_SIZE = 1 * 1024 * 1024          # 1 MiB: below this it is not a model shard
+DEFAULT_MAX_DEPTH = 40
+
+KNOWN_MODEL_EXTS = {
+    ".gguf", ".ggml", ".safetensors", ".bin", ".pt", ".pth",
+    ".onnx", ".gptq", ".awq", ".npz", ".mlmodel",
+}
+
+_OLLAMA_BLOB_RE = re.compile(r"^sha256[-:][0-9a-f]{64}$")
+_HEX64_RE = re.compile(r"^[0-9a-f]{64}$")
+
+LAYOUT_GENERIC = "generic"
+LAYOUT_OLLAMA = "ollama"
+LAYOUT_LMSTUDIO = "lmstudio"
+LAYOUT_TEXTGEN = "text-generation-webui"
+LAYOUT_PATH = "path"
+
+SkipFn = Optional[Callable[[str, str], None]]
+
+
+@dataclass
+class Candidate:
+    """A file that passed the cheap prefilter and is worth hashing."""
+    path: str
+    size: int
+    layout: str
+
+
+@dataclass
+class HashedFile:
+    """A candidate with its computed full-file SHA-256."""
+    path: str
+    size: int
+    sha256: str
+    layout: str
+    label: Optional[str] = None  # friendly name (e.g. Ollama model:tag), if known
+
+
+# --------------------------------------------------------------------------- #
+# Root discovery
+# --------------------------------------------------------------------------- #
+
+def default_roots() -> list[tuple[str, str]]:
+    """Known model-store roots that currently exist, tagged with their layout.
+
+    We only return directories that exist so the scan does not invent paths.
+    Order is stable but not significant (candidates are de-duplicated by real
+    path downstream via the (dev, ino) visited-set on directories).
+    """
+    home = os.path.expanduser("~")
+    candidates = [
+        (os.path.join(home, ".ollama", "models"), LAYOUT_OLLAMA),
+        ("/usr/share/ollama/.ollama/models", LAYOUT_OLLAMA),
+        ("/var/lib/ollama/.ollama/models", LAYOUT_OLLAMA),
+        (os.path.join(home, ".cache", "lm-studio", "models"), LAYOUT_LMSTUDIO),
+        (os.path.join(home, ".lmstudio", "models"), LAYOUT_LMSTUDIO),
+        (os.path.join(home, "text-generation-webui", "models"), LAYOUT_TEXTGEN),
+        (os.path.join(home, "models"), LAYOUT_GENERIC),
+        ("/models", LAYOUT_GENERIC),
+    ]
+    out: list[tuple[str, str]] = []
+    for path, layout in candidates:
+        if os.path.isdir(path):
+            out.append((path, layout))
+    return out
+
+
+# --------------------------------------------------------------------------- #
+# Ollama manifest parsing (the one layout we truly parse)
+# --------------------------------------------------------------------------- #
+
+def ollama_labels(models_root: str) -> dict[str, str]:
+    """Map ``blob-digest -> "model:tag"`` by parsing Ollama's manifest tree.
+
+    Best-effort and fully defensive: any unreadable/oversized/non-JSON manifest
+    is skipped, never fatal. The returned digests are lowercase 64-hex; they are
+    used only to attach a friendly label to a blob we still hash ourselves.
+    """
+    labels: dict[str, str] = {}
+    man_root = os.path.join(models_root, "manifests")
+    if not os.path.isdir(man_root):
+        return labels
+    for dirpath, _dirnames, filenames in os.walk(man_root, followlinks=False):
+        for fn in filenames:
+            fp = os.path.join(dirpath, fn)
+            try:
+                if os.path.islink(fp) or not os.path.isfile(fp):
+                    continue
+                if os.path.getsize(fp) > 1_000_000:  # a manifest is tiny JSON
+                    continue
+                with open(fp, "rb") as fh:
+                    data = json.loads(fh.read().decode("utf-8", "replace"))
+            except (OSError, ValueError):
+                continue
+            if not isinstance(data, dict):
+                continue
+            model = os.path.basename(dirpath)
+            label = f"{model}:{fn}"
+            layers = data.get("layers")
+            if not isinstance(layers, list):
+                continue
+            for layer in layers:
+                if not isinstance(layer, dict):
+                    continue
+                dig = layer.get("digest")
+                mtype = str(layer.get("mediaType") or "")
+                if isinstance(dig, str) and "model" in mtype:
+                    d = dig.split(":", 1)[-1].lower()
+                    if _HEX64_RE.match(d):
+                        labels[d] = label
+    return labels
+
+
+# --------------------------------------------------------------------------- #
+# Candidate walk (hostile-fs hardened)
+# --------------------------------------------------------------------------- #
+
+def _ext_ok(name: str, exts: set[str]) -> bool:
+    dot = name.rfind(".")
+    if dot < 0:
+        return False
+    return name[dot:].lower() in exts
+
+
+def _within(base_real: str, path_real: str) -> bool:
+    try:
+        common = os.path.commonpath([base_real, path_real])
+    except ValueError:
+        return False
+    return os.path.normcase(common) == os.path.normcase(base_real)
+
+
+def _skip(on_skip: SkipFn, path: str, reason: str) -> None:
+    if on_skip is not None:
+        on_skip(path, reason)
+
+
+def iter_candidates(
+    roots: list[tuple[str, str]],
+    *,
+    min_size: int = DEFAULT_MIN_SIZE,
+    exts: Optional[set[str]] = None,
+    all_files: bool = False,
+    max_depth: int = DEFAULT_MAX_DEPTH,
+    on_skip: SkipFn = None,
+) -> Iterator[Candidate]:
+    """Yield prefiltered candidate files under ``roots`` without ever following
+    a symlink out of a root or opening a non-regular file."""
+    exts = KNOWN_MODEL_EXTS if exts is None else exts
+    seen_dirs: set[tuple[int, int]] = set()
+    for root, layout in roots:
+        root_abs = os.path.abspath(os.path.expanduser(root))
+        if not os.path.isdir(root_abs):
+            _skip(on_skip, root_abs, "root-missing")
+            continue
+        # The boundary is the root's REAL path; a candidate whose realpath
+        # leaves it is rejected even though symlinks are already skipped.
+        base_real = os.path.realpath(root_abs)
+        yield from _scan_dir(
+            root_abs, layout, base_real, 0,
+            min_size, exts, all_files, max_depth, seen_dirs, on_skip,
+        )
+
+
+def _scan_dir(
+    dirpath: str, layout: str, base_real: str, depth: int,
+    min_size: int, exts: set[str], all_files: bool, max_depth: int,
+    seen: set[tuple[int, int]], on_skip: SkipFn,
+) -> Iterator[Candidate]:
+    if depth > max_depth:
+        _skip(on_skip, dirpath, "max-depth")
+        return
+    try:
+        dstat = os.stat(dirpath)  # dir we chose to descend into (never a symlink)
+    except OSError as exc:
+        _skip(on_skip, dirpath, f"stat-dir: {exc}")
+        return
+    key = (dstat.st_dev, dstat.st_ino)
+    if key in seen:
+        _skip(on_skip, dirpath, "loop")
+        return
+    seen.add(key)
+
+    try:
+        scan_it = os.scandir(dirpath)
+    except OSError as exc:
+        _skip(on_skip, dirpath, f"opendir: {exc}")
+        return
+
+    with scan_it:
+        for entry in scan_it:
+            try:
+                if entry.is_symlink():
+                    _skip(on_skip, entry.path, "symlink")
+                    continue
+                if entry.is_dir(follow_symlinks=False):
+                    yield from _scan_dir(
+                        entry.path, layout, base_real, depth + 1,
+                        min_size, exts, all_files, max_depth, seen, on_skip,
+                    )
+                    continue
+                if not entry.is_file(follow_symlinks=False):
+                    # device node / FIFO / socket / other — never open it.
+                    _skip(on_skip, entry.path, "not-regular")
+                    continue
+                est = entry.stat(follow_symlinks=False)
+                size = est.st_size
+                if size < min_size:
+                    continue
+                name = entry.name
+                if not (all_files or _ext_ok(name, exts)
+                        or _OLLAMA_BLOB_RE.match(name)):
+                    continue
+                if not _within(base_real, os.path.realpath(entry.path)):
+                    _skip(on_skip, entry.path, "escapes-root")
+                    continue
+                yield Candidate(path=entry.path, size=size, layout=layout)
+            except OSError as exc:
+                _skip(on_skip, entry.path, f"stat: {exc}")
+                continue
+
+
+# --------------------------------------------------------------------------- #
+# Hashing
+# --------------------------------------------------------------------------- #
+
+def hash_candidates(
+    candidates: list[Candidate],
+    *,
+    labels: Optional[dict[str, str]] = None,
+    progress: Optional[Callable[[int, int, str], None]] = None,
+) -> list[HashedFile]:
+    """Full-file SHA-256 every candidate; attach a label by digest when known.
+
+    Unreadable files (races, permission flips) are skipped rather than fatal.
+    """
+    labels = labels or {}
+    out: list[HashedFile] = []
+    total = len(candidates)
+    for i, cand in enumerate(candidates, 1):
+        if progress is not None:
+            progress(i, total, cand.path)
+        try:
+            digest = _safe_sha256(cand.path)
+        except OSError:
+            # symlink/FIFO/device swapped in after the walk, race, perm flip:
+            # skip rather than hash something untrusted or block forever.
+            continue
+        out.append(HashedFile(
+            path=cand.path, size=cand.size, sha256=digest,
+            layout=cand.layout, label=labels.get(digest),
+        ))
+    return out

llmt/telemetry/__init__.py

@@ -0,0 +1,15 @@
+"""llmt telemetry — the OPT-IN agent write-API client (WP-14.4-T, BUILD2 14B).
+
+Telemetry is a SEPARATE, explicit, OFF-BY-DEFAULT, revocable opt-in (DR-27).
+Nothing in this package touches the network unless the user has run
+``llmt telemetry enable`` and a token is present. Scan data and local file
+PATHS never leave the machine: only torrent infohashes/names/states/progress,
+artifact SHA-256s, and challenge-range hashes are ever sent. See
+:mod:`llmt.telemetry.consent` for the exact, user-visible statement.
+"""
+
+from __future__ import annotations
+
+AGENT_API_PATH = "/agent/v1"
+
+__all__ = ["AGENT_API_PATH"]

llmt/telemetry/artifacts.py

@@ -0,0 +1,217 @@
+"""Artifact-level possession challenges (H2) + local artifact location.
+
+A challenge references an artifact by ``sha256`` and asks for
+``sha256(salt_bytes || file_bytes[offset : offset+length])``. We locate the
+LOCAL file whose FULL sha256 equals the challenged digest (via the same
+open-time-hardened full-file hash the scanner uses), then hash the requested
+byte range. If the file is absent, or too short to cover the range, we answer
+NOTHING for it (honest failure) — we NEVER fabricate a hash.
+
+Full-file digests are cached by (realpath, size, mtime_ns) under the llmt home
+so repeated beats do not re-hash unchanged files.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import stat
+from dataclasses import dataclass
+from typing import Optional
+
+from ..errors import ApiError
+from ..scanner import _safe_sha256
+from . import config as tcfg
+
+_HEX64_RE = re.compile(r"^[0-9a-f]{64}$")
+_MAX_CACHE_ENTRIES = 8192
+
+
+# --------------------------------------------------------------------------- #
+# Range hashing (the possession proof)
+# --------------------------------------------------------------------------- #
+
+def decode_salt(salt_hex: object) -> bytes:
+    """Decode the server-provided hex salt to raw bytes (contract (b)2)."""
+    if not isinstance(salt_hex, str):
+        raise ApiError("challenge salt is not a string.")
+    s = salt_hex.strip()
+    if s == "":
+        return b""
+    try:
+        return bytes.fromhex(s)
+    except ValueError as exc:
+        raise ApiError("challenge salt is not valid hex.") from exc
+
+
+def hash_range(path: str, salt: bytes, offset: int, length: int,
+               chunk_size: int = 1024 * 1024) -> Optional[str]:
+    """Return ``sha256(salt || file[offset:offset+length])`` as hex, or None
+    if the file is missing / not a regular file / too short for the range.
+
+    Symlinks are refused at open time (O_NOFOLLOW); only a genuine regular
+    file is ever read. Reads are streamed so a large range never loads wholly
+    into memory.
+    """
+    if offset < 0 or length < 0:
+        return None
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        fd = os.open(path, flags)
+    except OSError:
+        return None
+    try:
+        st = os.fstat(fd)
+        if not stat.S_ISREG(st.st_mode):
+            return None
+        if st.st_size < offset + length:
+            return None  # range not fully present -> fail honestly
+        os.lseek(fd, offset, os.SEEK_SET)
+        h = hashlib.sha256()
+        h.update(salt)
+        remaining = length
+        while remaining > 0:
+            chunk = os.read(fd, min(chunk_size, remaining))
+            if not chunk:
+                return None  # truncated under us -> honest failure
+            h.update(chunk)
+            remaining -= len(chunk)
+        return h.hexdigest()
+    except OSError:
+        return None
+    finally:
+        os.close(fd)
+
+
+# --------------------------------------------------------------------------- #
+# Local artifact location (full-file sha256, cached)
+# --------------------------------------------------------------------------- #
+
+class ArtifactLocator:
+    """Find the local file for a given full-file SHA-256 among candidate paths,
+    caching digests by (realpath, size, mtime_ns)."""
+
+    def __init__(self, cache_path: Optional[str] = None):
+        self.cache_path = cache_path or os.path.join(
+            tcfg.llmt_home(), "hash-cache.json")
+        self._cache: dict = {}
+        self._dirty = False
+        self._load()
+
+    def _load(self) -> None:
+        try:
+            with open(self.cache_path, "r", encoding="utf-8") as fh:
+                data = json.load(fh)
+            if isinstance(data, dict):
+                self._cache = {k: v for k, v in data.items()
+                               if isinstance(k, str) and isinstance(v, str)
+                               and _HEX64_RE.match(v)}
+        except (OSError, ValueError):
+            self._cache = {}
+
+    @staticmethod
+    def _key(path: str) -> Optional[str]:
+        try:
+            st = os.stat(path)
+        except OSError:
+            return None
+        rp = os.path.realpath(path)
+        return f"{rp}\x00{st.st_size}\x00{st.st_mtime_ns}"
+
+    def sha256_of(self, path: str) -> Optional[str]:
+        key = self._key(path)
+        if key is not None:
+            cached = self._cache.get(key)
+            if cached is not None:
+                return cached
+        try:
+            digest = _safe_sha256(path)
+        except OSError:
+            return None
+        if key is not None and len(self._cache) < _MAX_CACHE_ENTRIES:
+            self._cache[key] = digest
+            self._dirty = True
+        return digest
+
+    def locate(self, target_sha256: str, candidate_paths) -> Optional[str]:
+        """Return the first candidate whose full sha256 == target, else None.
+        Short-circuits on the first match so a whole tree need not be hashed."""
+        target = (target_sha256 or "").strip().lower()
+        if not _HEX64_RE.match(target):
+            return None
+        for path in candidate_paths:
+            if self.sha256_of(path) == target:
+                return path
+        return None
+
+    def save(self) -> None:
+        if not self._dirty:
+            return
+        try:
+            os.makedirs(os.path.dirname(self.cache_path), exist_ok=True)
+            tmp = self.cache_path + ".tmp"
+            with open(tmp, "w", encoding="utf-8") as fh:
+                json.dump(self._cache, fh)
+            os.replace(tmp, self.cache_path)
+            try:
+                os.chmod(self.cache_path, 0o600)
+            except OSError:
+                pass
+            self._dirty = False
+        except OSError:
+            pass
+
+
+# --------------------------------------------------------------------------- #
+# Challenge parsing + answering
+# --------------------------------------------------------------------------- #
+
+@dataclass
+class Challenge:
+    nonce: str
+    artifact_sha256: str
+    offset: int
+    length: int
+    salt: str
+
+    @classmethod
+    def parse(cls, raw: object) -> Optional["Challenge"]:
+        if not isinstance(raw, dict):
+            return None
+        nonce = raw.get("nonce")
+        sha = raw.get("artifact_sha256")
+        offset = raw.get("offset")
+        length = raw.get("length")
+        salt = raw.get("salt")
+        if not isinstance(nonce, str) or not nonce:
+            return None
+        if not isinstance(sha, str) or not _HEX64_RE.match(sha.strip().lower()):
+            return None
+        if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
+            return None
+        if isinstance(length, bool) or not isinstance(length, int) or length < 0:
+            return None
+        if not isinstance(salt, str):
+            return None
+        return cls(nonce=nonce, artifact_sha256=sha.strip().lower(),
+                   offset=offset, length=length, salt=salt)
+
+
+def answer_challenge(challenge: Challenge, candidate_paths, locator,
+                     chunk_size: int) -> Optional[str]:
+    """Resolve the local file for ``challenge.artifact_sha256`` and return the
+    range hash, or None when we cannot honestly answer (file absent/incomplete).
+    """
+    path = locator.locate(challenge.artifact_sha256, candidate_paths)
+    if path is None:
+        return None
+    try:
+        salt = decode_salt(challenge.salt)
+    except ApiError:
+        # A hostile / malformed salt is a per-challenge honest failure, never
+        # a reason to abort the whole beat (which would retry forever --loop).
+        return None
+    return hash_range(path, salt, challenge.offset, challenge.length,
+                      chunk_size=chunk_size)