tackbox 0.1.14__py3-none-win_amd64.whl

tackbox/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.14"

tackbox/cache.py

@@ -0,0 +1,351 @@
+"""(unit, engine) cache: layout, digests, marker ops, GC.
+
+Layout: `<TACKBOX_CACHE_HOME>/v1/<engines-hash>/<unit-digest>.<engine-id>`.
+Marker is an empty file. `TACKBOX_CACHE_HOME` defaults to `~/.cache/tackbox`;
+tests point it at a tmp dir.
+
+Semantics per plan:
+- Marker written only on success. Failures are not cached.
+- Cache hit re-touches the marker: mtime is the LRU signal.
+- Corrupt / unreadable marker -> treated as miss (rerun), never fatal.
+- `mark_clean` is best-effort; cache is an optimisation, never a hard error.
+
+Unit granularity:
+- eslint / mdlint / opengrep -> unit = file, digest = sha256(content).
+- erclint -> unit = Go package, digest = sha256(import path + own .go files +
+  transitive in-module deps' .go files + go.mod + go.sum). A signature change
+  in package B invalidates every in-module package that depends on B.
+
+engines-hash:
+- Dev mode digests the engine payload sources (go/, js/, bin/, the eslint
+  preset, npm manifest and lockfile), so editing any rule invalidates prior
+  markers and stale clean-results can never mask new findings. The
+  orchestrator under py/ is deliberately outside the payload. Hermetic
+  wheels (step 6) replace this with the bundled-payload digest.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+
+CACHE_ROOT_ENV = "TACKBOX_CACHE_HOME"
+CACHE_VERSION = "v1"
+SOFT_CAP = 20000
+
+
+def default_cache_root() -> Path:
+    """Root under which the `v1/<engines-hash>/...` tree lives."""
+    override = os.environ.get(CACHE_ROOT_ENV)
+    if override:
+        return Path(override) / CACHE_VERSION
+    return Path.home() / ".cache" / "tackbox" / CACHE_VERSION
+
+
+@dataclass(frozen=True)
+class CacheKey:
+    engines_hash: str
+    unit_digest: str
+    engine_id: str
+
+    def marker(self, root: Path) -> Path:
+        return root / self.engines_hash / f"{self.unit_digest}.{self.engine_id}"
+
+
+# Engine payload in dev mode: everything that shapes findings, nothing else.
+_DEV_PAYLOAD = (
+    "go",
+    "js",
+    "bin",
+    "eslint.config.preset.js",
+    "package.json",
+    "package-lock.json",
+)
+
+
+def engines_hash_dev(tackbox_root: Path) -> str:
+    h = hashlib.sha256()
+    h.update(b"dev-payload-v1\n")
+    for top in _DEV_PAYLOAD:
+        p = tackbox_root / top
+        if p.is_file():
+            _hash_payload_file(h, top, p)
+        elif p.is_dir():
+            for f in sorted(p.rglob("*")):
+                if f.is_file() and "__pycache__" not in f.parts:
+                    _hash_payload_file(h, f.relative_to(tackbox_root).as_posix(), f)
+    return h.hexdigest()
+
+
+def _hash_payload_file(h, rel: str, path: Path) -> None:
+    h.update(rel.encode())
+    h.update(b"\t")
+    h.update(sha256_file(path).encode())
+    h.update(b"\n")
+
+
+def is_cached(key: CacheKey, root: Path) -> bool:
+    """Return True iff a valid marker file exists; re-touch for LRU on hit."""
+    p = key.marker(root)
+    try:
+        if not p.is_file():
+            return False
+        p.touch()
+        return True
+    except OSError:
+        return False
+
+
+def mark_clean(key: CacheKey, root: Path) -> None:
+    """Write empty marker; swallow any OSError so cache never blocks a run."""
+    p = key.marker(root)
+    try:
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.touch()
+    except OSError:
+        pass
+
+
+def sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(65536), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def sha256_tree(root: Path) -> str:
+    """Deterministic digest of a directory tree: sorted rel paths + content.
+
+    Shared by the wheel builder (stamping engines.json) and doctor
+    (verifying it) - one implementation so the two can never drift.
+    """
+    h = hashlib.sha256()
+    for f in sorted(root.rglob("*")):
+        if not f.is_file():
+            continue
+        h.update(f.relative_to(root).as_posix().encode())
+        h.update(b"\0")
+        h.update(sha256_file(f).encode())
+        h.update(b"\0")
+    return h.hexdigest()
+
+
+def gc_stale_engines(current: str, root: Path) -> None:
+    """Drop every `<engines-hash>/` sibling other than `current`."""
+    if not root.is_dir():
+        return
+    for entry in root.iterdir():
+        if entry.is_dir() and entry.name != current:
+            shutil.rmtree(entry, ignore_errors=True)
+
+
+def gc_soft_cap(engines_hash: str, cap: int, root: Path) -> None:
+    """Trim markers in the current dir when the count exceeds `cap`.
+
+    Sort by mtime ascending; drop from the front until at or under cap. Files
+    that vanish under us (concurrent run) are ignored - GC never blocks.
+    """
+    d = root / engines_hash
+    if not d.is_dir():
+        return
+    markers = [p for p in d.iterdir() if p.is_file()]
+    if len(markers) <= cap:
+        return
+
+    def _mtime(p: Path) -> float:
+        # A marker can vanish between iterdir and stat (concurrent run);
+        # treat it as oldest so unlink handles it, never raise.
+        try:
+            return p.stat().st_mtime
+        except OSError:
+            return 0.0
+
+    markers.sort(key=_mtime)
+    for m in markers[: len(markers) - cap]:
+        try:
+            m.unlink()
+        except OSError:
+            pass
+
+
+# -- erclint package digest -----------------------------------------------
+
+
+def erclint_package_digests(
+    repo_root: Path, package_dirs: list[str]
+) -> dict[str, str]:
+    """Compute {package_dir: unit_digest} for erclint units.
+
+    Runs `go list -deps -json` once for all requested packages, filters to
+    same-module deps, then hashes each package's own .go files together with
+    the .go files of its transitive in-module deps plus go.mod / go.sum. A
+    change to any file that erclint would see for a package flips the digest.
+
+    Missing / not-a-package entries are dropped from the returned map; the
+    caller decides what to do (usually: skip caching for that entry).
+    """
+    if not package_dirs:
+        return {}
+    args = [f"./{p}" for p in package_dirs]
+    completed = subprocess.run(
+        ["go", "list", "-deps", "-json", *args],
+        cwd=repo_root,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    pkgs = list(_iter_json_objects(completed.stdout))
+
+    our_module = _module_path_from_pkgs(pkgs)
+    if our_module is None:
+        return {}
+
+    in_module: dict[str, dict] = {}
+    for p in pkgs:
+        if p.get("Standard"):
+            continue
+        mod = p.get("Module") or {}
+        if mod.get("Path") != our_module:
+            continue
+        import_path = p["ImportPath"]
+        dir_ = Path(p["Dir"])
+        go_files = list(p.get("GoFiles") or [])
+        # Deps is the transitive closure per `go list -json`; filter later.
+        deps = set(p.get("Deps") or [])
+        in_module[import_path] = {
+            "dir": dir_,
+            "files": [dir_ / f for f in go_files],
+            "deps": deps,
+        }
+    for info in in_module.values():
+        info["deps"] = {d for d in info["deps"] if d in in_module}
+
+    file_digest: dict[Path, str] = {}
+    for info in in_module.values():
+        for f in info["files"]:
+            if f not in file_digest and f.is_file():
+                file_digest[f] = sha256_file(f)
+
+    go_mod_digest = _optional_file_digest(repo_root / "go.mod")
+    go_sum_digest = _optional_file_digest(repo_root / "go.sum")
+
+    dir_to_import: dict[str, str] = {}
+    repo_resolved = repo_root.resolve()
+    for import_path, info in in_module.items():
+        try:
+            rel = info["dir"].resolve().relative_to(repo_resolved)
+        except ValueError:
+            continue
+        key = str(rel) if str(rel) != "." else "."
+        dir_to_import[key] = import_path
+
+    result: dict[str, str] = {}
+    for pkg_dir in package_dirs:
+        import_path = dir_to_import.get(pkg_dir)
+        if import_path is None:
+            continue
+        result[pkg_dir] = _package_digest(
+            import_path, in_module, file_digest, go_mod_digest, go_sum_digest
+        )
+    return result
+
+
+def erclint_import_paths(
+    repo_root: Path, package_dirs: list[str]
+) -> dict[str, str]:
+    """Return {package_dir: import_path}. Used to interpret erclint findings."""
+    if not package_dirs:
+        return {}
+    args = [f"./{p}" for p in package_dirs]
+    completed = subprocess.run(
+        ["go", "list", "-json", *args],
+        cwd=repo_root,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    pkgs = list(_iter_json_objects(completed.stdout))
+    repo_resolved = repo_root.resolve()
+    result: dict[str, str] = {}
+    for p in pkgs:
+        try:
+            rel = Path(p["Dir"]).resolve().relative_to(repo_resolved)
+        except ValueError:
+            continue
+        key = str(rel) if str(rel) != "." else "."
+        if key in package_dirs:
+            result[key] = p["ImportPath"]
+    return result
+
+
+def _package_digest(
+    import_path: str,
+    in_module: dict[str, dict],
+    file_digest: dict[Path, str],
+    go_mod_digest: str,
+    go_sum_digest: str,
+) -> str:
+    h = hashlib.sha256()
+    h.update(import_path.encode())
+    h.update(b"\n---self---\n")
+    _hash_files(h, in_module[import_path]["files"], file_digest)
+    h.update(b"---deps---\n")
+    for dep in sorted(in_module[import_path]["deps"]):
+        h.update(dep.encode())
+        h.update(b"\n")
+        _hash_files(h, in_module[dep]["files"], file_digest)
+    h.update(b"---go.mod---\n")
+    h.update(go_mod_digest.encode())
+    h.update(b"\n---go.sum---\n")
+    h.update(go_sum_digest.encode())
+    return h.hexdigest()
+
+
+def _hash_files(h, files, file_digest: dict[Path, str]) -> None:
+    for f in sorted(files):
+        digest = file_digest.get(f)
+        if digest is None:
+            continue
+        h.update(f.name.encode())
+        h.update(b"\t")
+        h.update(digest.encode())
+        h.update(b"\n")
+
+
+def _optional_file_digest(path: Path) -> str:
+    if not path.is_file():
+        return ""
+    return sha256_file(path)
+
+
+def _module_path_from_pkgs(pkgs: list[dict]) -> str | None:
+    for p in pkgs:
+        if p.get("Standard"):
+            continue
+        mod = p.get("Module") or {}
+        path = mod.get("Path")
+        if path:
+            return path
+    return None
+
+
+def _iter_json_objects(text: str):
+    """Iterate over concatenated top-level JSON objects in `text`."""
+    decoder = json.JSONDecoder()
+    idx = 0
+    n = len(text)
+    while idx < n:
+        while idx < n and text[idx].isspace():
+            idx += 1
+        if idx >= n:
+            break
+        obj, end = decoder.raw_decode(text, idx)
+        yield obj
+        idx = end

tackbox/cli.py

@@ -0,0 +1,349 @@
+"""tackbox lint / doctor CLI entry point."""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+from . import __version__, cache, doctor
+from .engines import (
+    EngineResult,
+    EngineSpec,
+    active_engines,
+    dispatch,
+    engines_hash_hermetic,
+    is_hermetic,
+    parse_erclint_findings,
+    resolve_dev_versions,
+    resolve_hermetic_versions,
+    run_engines,
+)
+from .source_set import (
+    PathspecMagicError,
+    filter_source_set,
+    parse_git_diff_names,
+    parse_ls_files_stage,
+    parse_ls_files_untracked,
+)
+
+
+class ChangedScopeError(ValueError):
+    """Raised when the git commands backing --changed / --since fail."""
+
+_BANNER_ORDER = ("erclint", "opengrep", "node", "eslint", "markdownlint")
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse_argv(sys.argv[1:] if argv is None else argv)
+    if args.command == "lint":
+        try:
+            return _run_lint(
+                args.path,
+                no_cache=args.no_cache,
+                changed=args.changed,
+                since=args.since,
+            )
+        except (PathspecMagicError, ChangedScopeError) as e:
+            print(f"tackbox: {e}", file=sys.stderr)
+            return 2
+    if args.command == "doctor":
+        _print_banner(_tackbox_root())
+        return doctor.run(sys.stdout)
+    print(f"tackbox: unknown command {args.command!r}", file=sys.stderr)
+    return 2
+
+
+def _parse_argv(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(prog="tackbox")
+    sub = parser.add_subparsers(dest="command", required=True)
+    lint = sub.add_parser("lint", help="lint the source set")
+    lint.add_argument("path", nargs="?", default=".", help="scope path (default: .)")
+    lint.add_argument(
+        "--no-cache",
+        action="store_true",
+        help="ignore and do not write the (unit, engine) cache",
+    )
+    lint.add_argument(
+        "--changed",
+        action="store_true",
+        help="restrict to dirty tree (staged + unstaged + untracked)",
+    )
+    lint.add_argument(
+        "--since",
+        metavar="<ref>",
+        default=None,
+        help="restrict to three-dot diff <ref>...HEAD unioned with dirty tree",
+    )
+    sub.add_parser("doctor", help="verify the hermetic install is functional")
+    return parser.parse_args(argv)
+
+
+def _tackbox_root() -> Path:
+    return Path(__file__).resolve().parents[2]
+
+
+def _run_lint(scope: str, no_cache: bool, changed: bool, since: str | None) -> int:
+    repo_root = _find_repo_root()
+    tackbox_root = _tackbox_root()
+
+    changed_scope: set[str] | None = None
+    if changed or since is not None:
+        changed_scope = _compute_changed_scope(repo_root, since)
+
+    files, warnings = _collect_source_set(repo_root, scope, changed_scope)
+    for w in warnings:
+        print(f"tackbox: warning: {w.reason}: {w.path}", file=sys.stderr)
+
+    if not files:
+        print(
+            f"tackbox: scope {scope!r} matched no files in the source set",
+            file=sys.stderr,
+        )
+        return 2
+
+    _print_banner(tackbox_root)
+
+    plan = dispatch(files, active_engines())
+    if not plan:
+        return 0
+
+    # Self-lint: tackbox lints itself. Cache is disabled so tackbox never
+    # self-caches its own bugs (plan: "чтобы tackbox не самокэшировал").
+    if tackbox_root.resolve() == repo_root.resolve():
+        no_cache = True
+
+    if no_cache:
+        results = run_engines(plan, repo_root, tackbox_root)
+    else:
+        cache_root = cache.default_cache_root()
+        engines_hash = engines_hash_hermetic() if is_hermetic() else cache.engines_hash_dev(tackbox_root)
+        cache.gc_stale_engines(engines_hash, cache_root)
+
+        filtered_plan, pending = _apply_cache(plan, repo_root, engines_hash, cache_root)
+        results = run_engines(filtered_plan, repo_root, tackbox_root)
+        _mark_clean_units(results, pending, engines_hash, cache_root)
+        cache.gc_soft_cap(engines_hash, cache.SOFT_CAP, cache_root)
+
+    for r in results:
+        sys.stdout.write(f"== {r.engine_id} ==\n")
+        if r.stdout:
+            sys.stdout.write(r.stdout)
+            if not r.stdout.endswith("\n"):
+                sys.stdout.write("\n")
+        if r.stderr:
+            sys.stderr.write(r.stderr)
+            if not r.stderr.endswith("\n"):
+                sys.stderr.write("\n")
+
+    return _aggregate_exit(results)
+
+
+# -- Cache wiring ---------------------------------------------------------
+
+
+def _apply_cache(
+    plan: list[tuple[EngineSpec, list[str]]],
+    repo_root: Path,
+    engines_hash: str,
+    cache_root: Path,
+) -> tuple[list[tuple[EngineSpec, list[str]]], dict[str, dict]]:
+    """Filter cached units out of each engine's args.
+
+    Returns:
+    - filtered_plan: engines that still have uncached args.
+    - pending[engine_id] = {
+        "arg_digest": [(arg, digest), ...],   # uncached args passed to engine
+        "arg_ip": {arg: import_path, ...},     # erclint-only mapping
+      }
+      Used post-run to translate engine output into per-unit success and
+      write markers for the clean units.
+    """
+    filtered_plan: list[tuple[EngineSpec, list[str]]] = []
+    pending: dict[str, dict] = {}
+    for engine, args in plan:
+        arg_digest, extras = _digests_for_engine(engine, args, repo_root)
+        uncached: list[tuple[str, str]] = []
+        for arg, digest in arg_digest:
+            if digest is None:
+                uncached.append((arg, digest))
+                continue
+            key = cache.CacheKey(engines_hash, digest, engine.id)
+            if not cache.is_cached(key, cache_root):
+                uncached.append((arg, digest))
+        pending[engine.id] = {"arg_digest": uncached, **extras}
+        if uncached:
+            filtered_plan.append((engine, [a for a, _ in uncached]))
+    return filtered_plan, pending
+
+
+def _digests_for_engine(
+    engine: EngineSpec, args: list[str], repo_root: Path
+) -> tuple[list[tuple[str, str]], dict]:
+    if engine.id == "erclint":
+        digest_map = cache.erclint_package_digests(repo_root, args)
+        ip_map = cache.erclint_import_paths(repo_root, args)
+        # digest None = lint always, cache never; dropping the arg instead
+        # would silently skip linting the package.
+        arg_digest = [(a, digest_map.get(a)) for a in args]
+        return arg_digest, {"arg_ip": ip_map}
+    arg_digest = [(a, cache.sha256_file(repo_root / a)) for a in args]
+    return arg_digest, {}
+
+
+def _mark_clean_units(
+    results: list[EngineResult],
+    pending: dict[str, dict],
+    engines_hash: str,
+    cache_root: Path,
+) -> None:
+    for r in results:
+        info = pending.get(r.engine_id)
+        if not info:
+            continue
+        clean_args = _clean_args(r, info)
+        digest_of = dict(info["arg_digest"])
+        for arg in clean_args:
+            digest = digest_of.get(arg)
+            if digest is None:
+                continue
+            cache.mark_clean(
+                cache.CacheKey(engines_hash, digest, r.engine_id), cache_root
+            )
+
+
+def _clean_args(r: EngineResult, info: dict) -> list[str]:
+    args = [a for a, _ in info["arg_digest"]]
+    if r.engine_id == "erclint":
+        try:
+            findings = parse_erclint_findings(r.stdout)
+        except ValueError:
+            return []
+        dirty_ips = {f.get("pkg") for f in findings}
+        ip_map = info.get("arg_ip", {})
+        # Unknown import path -> cannot attribute findings -> never clean.
+        return [
+            a for a in args
+            if ip_map.get(a) is not None and ip_map[a] not in dirty_ips
+        ]
+    if r.exit_code == 0:
+        return args
+    return []
+
+
+def _aggregate_exit(results: list[EngineResult]) -> int:
+    """Aggregate engine exit codes; promote erclint findings to nonzero.
+
+    erclint in `-json` mode returns exit 0 even when findings exist -
+    handover #2 pinned this contract. tackbox is the layer that translates
+    findings into a failing aggregate exit.
+    """
+    max_code = 0
+    for r in results:
+        code = r.exit_code
+        if code == 0 and r.engine_id == "erclint" and _erclint_has_findings(r.stdout):
+            code = 1
+        if code > max_code:
+            max_code = code
+    return max_code
+
+
+def _erclint_has_findings(stdout: str) -> bool:
+    try:
+        return bool(parse_erclint_findings(stdout))
+    except ValueError:
+        # Analyzer-load errors surface as a failing aggregate.
+        return True
+
+
+def _collect_source_set(
+    repo_root: Path, scope: str, changed_scope: set[str] | None = None
+):
+    stage_raw = subprocess.run(
+        ["git", "ls-files", "-s", "-z"],
+        cwd=repo_root,
+        capture_output=True,
+        check=True,
+    ).stdout
+    untracked_raw = subprocess.run(
+        ["git", "ls-files", "--others", "--exclude-standard", "-z"],
+        cwd=repo_root,
+        capture_output=True,
+        check=True,
+    ).stdout
+    return filter_source_set(
+        parse_ls_files_stage(stage_raw),
+        parse_ls_files_untracked(untracked_raw),
+        scope,
+        exists=lambda p: (repo_root / p).exists(),
+        is_symlink=lambda p: (repo_root / p).is_symlink(),
+        changed_scope=changed_scope,
+    )
+
+
+def _compute_changed_scope(repo_root: Path, since: str | None) -> set[str]:
+    """Union of dirty tree with (optional) three-dot diff against <since>.
+
+    Dirty tree = files that differ from HEAD in the index or worktree,
+    plus untracked. Three-dot diff = files changed on this branch since
+    the merge-base with <since>; matches the PR-style question "what did
+    I change on my branch." A two-dot diff would leak reverse-changes
+    when <since> progresses after fork.
+    """
+    scope: set[str] = set()
+    completed = subprocess.run(
+        ["git", "diff", "--name-only", "-z", "HEAD"],
+        cwd=repo_root,
+        capture_output=True,
+    )
+    if completed.returncode != 0:
+        # Fresh repo without any commits: HEAD does not resolve. Fail with a
+        # clean tackbox message instead of a Python traceback on onboarding.
+        err = completed.stderr.decode("utf-8", errors="replace").strip()
+        raise ChangedScopeError(
+            f"--changed / --since requires at least one commit ({err})"
+        )
+    scope.update(parse_git_diff_names(completed.stdout))
+    untracked = subprocess.run(
+        ["git", "ls-files", "--others", "--exclude-standard", "-z"],
+        cwd=repo_root,
+        capture_output=True,
+        check=True,
+    ).stdout
+    scope.update(parse_ls_files_untracked(untracked))
+    if since is not None:
+        completed = subprocess.run(
+            ["git", "diff", "--name-only", "-z", f"{since}...HEAD"],
+            cwd=repo_root,
+            capture_output=True,
+        )
+        if completed.returncode != 0:
+            err = completed.stderr.decode("utf-8", errors="replace").strip()
+            raise ChangedScopeError(f"--since={since}: {err or 'git diff failed'}")
+        scope.update(parse_git_diff_names(completed.stdout))
+    return scope
+
+
+def _find_repo_root() -> Path:
+    result = subprocess.run(
+        ["git", "rev-parse", "--show-toplevel"],
+        capture_output=True,
+        check=True,
+    )
+    return Path(result.stdout.decode().strip())
+
+
+def _print_banner(tackbox_root: Path) -> None:
+    if is_hermetic():
+        versions = resolve_hermetic_versions()
+        engines_id = f"sha256:{engines_hash_hermetic()}"
+    else:
+        versions = resolve_dev_versions(tackbox_root)
+        engines_id = "dev"
+    parts = " ".join(f"{k}={versions[k]}" for k in _BANNER_ORDER)
+    print(f"tackbox {__version__} engines={engines_id} {parts}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    sys.exit(main())