pi-dev-team 0.2.0

package/scripts/recon_inventory.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+"""Python port of scripts/recon-inventory.sh (#615 / #572 Phase 3).
+
+Canonical file enumeration for the RECON envelope's `file_inventory` field
+(primitives contract 1.2.0+). Single source of truth for the enumeration
+pipeline; invoked both by the codebase-recon agent and by the
+evals/codebase-recon/tests/ test harness.
+
+Usage:
+    recon_inventory.py <repo-root> [--slug <slug>]
+                                    [--force-filesystem-walk]
+                                    [--emit-main-inventory-json <path>]
+
+Output:
+    stdout: one repo-relative path per line, LC_ALL=C sorted, deduplicated,
+            LF-terminated, no blank lines.
+    stderr: human-readable progress + `# BROKEN_SYMLINK: <src> -> <dst>`
+            markers (one per broken link). Consumers capture these for the
+            envelope `notes` array.
+    --emit-main-inventory-json <path>: write a JSON fragment suitable for
+            splicing into the main RECON envelope:
+                { "source": "...", "count": N, "sibling_ref": "recon-<slug>.inventory.txt" }
+
+Exit codes:
+    0  success
+    2  usage error
+    3  repo root not a directory
+
+Stdlib-only. See docs/python-hook-contract.md.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def _usage() -> None:
+    print(
+        "Usage: recon-inventory.sh <repo-root> [--slug <slug>]\n"
+        "                                       [--force-filesystem-walk]\n"
+        "                                       [--emit-main-inventory-json <path>]",
+        file=sys.stderr,
+    )
+
+
+def _parse_argv(argv: list[str]):
+    root = ""
+    slug = ""
+    force_fs = False
+    emit_main = ""
+    i = 0
+    while i < len(argv):
+        tok = argv[i]
+        if tok == "--slug":
+            slug = argv[i + 1] if i + 1 < len(argv) else ""
+            i += 2
+        elif tok == "--force-filesystem-walk":
+            force_fs = True
+            i += 1
+        elif tok == "--emit-main-inventory-json":
+            emit_main = argv[i + 1] if i + 1 < len(argv) else ""
+            i += 2
+        elif tok in ("-h", "--help"):
+            _usage()
+            sys.exit(2)
+        elif tok.startswith("--"):
+            print(f"unknown flag: {tok}", file=sys.stderr)
+            _usage()
+            sys.exit(2)
+        else:
+            if not root:
+                root = tok
+            else:
+                print(f"unexpected positional arg: {tok}", file=sys.stderr)
+                _usage()
+                sys.exit(2)
+            i += 1
+    return root, slug, force_fs, emit_main
+
+
+def _slugify(name: str) -> str:
+    """basename → lowercased, [a-z0-9._-] only, no leading/trailing dashes."""
+    s = name.lower()
+    s = re.sub(r"[^a-z0-9._-]", "-", s)
+    s = re.sub(r"-{2,}", "-", s)
+    s = s.strip("-")
+    return s
+
+
+def _parse_excludes(path: Path) -> tuple[list[str], list[str]]:
+    """Parse the two-section excludes file into (prefixes, filenames)."""
+    prefixes: list[str] = []
+    filenames: list[str] = []
+    section = ""
+    if not path.is_file():
+        return prefixes, filenames
+    for raw_line in path.read_text().splitlines():
+        line = raw_line
+        if re.match(
+            r"^[[:space:]]*#[[:space:]]*prefix:[[:space:]]*$".replace(
+                "[[:space:]]", r"\s"
+            ),
+            line,
+        ):
+            section = "prefix"
+            continue
+        if re.match(
+            r"^[[:space:]]*#[[:space:]]*filename:[[:space:]]*$".replace(
+                "[[:space:]]", r"\s"
+            ),
+            line,
+        ):
+            section = "filename"
+            continue
+        if re.match(r"^\s*#", line):
+            continue
+        if not line.strip():
+            continue
+        entry = line.strip()
+        entry = entry.rstrip("/")
+        if section == "prefix":
+            prefixes.append(entry)
+        elif section == "filename":
+            filenames.append(entry)
+    return prefixes, filenames
+
+
+def _collect_git(root_abs: Path) -> list[str]:
+    """Git branch: `ls-files -z` emits repo-relative paths including
+    submodule gitlinks (once, not recursed)."""
+    try:
+        r = subprocess.run(
+            ["git", "-C", str(root_abs), "ls-files", "-z"],
+            capture_output=True,
+            check=False,
+        )
+    except (FileNotFoundError, OSError):
+        return []
+    if r.returncode != 0:
+        return []
+    # NUL-delimited output; drop trailing empty split.
+    return [line for line in r.stdout.decode(errors="replace").split("\0") if line]
+
+
+def _collect_fs(root_abs: Path, excludes_file: Path) -> list[str]:
+    """Filesystem walk with excludes. Prunes directories whose name matches
+    a `# prefix:` entry (anywhere in the tree) and skips filenames matching
+    `# filename:` entries."""
+    prefixes, filenames = _parse_excludes(excludes_file)
+    prefix_set: set[str] = set(prefixes)
+    filename_set: set[str] = set(filenames)
+
+    out: list[str] = []
+    root_str = str(root_abs)
+    for dirpath, dirnames, files in os.walk(root_abs):
+        # Prune matching directory names in-place so os.walk skips them.
+        dirnames[:] = [d for d in dirnames if d not in prefix_set]
+        for f in files:
+            if f in filename_set:
+                continue
+            abs_path = os.path.join(dirpath, f)
+            # Only include regular files (skip devices, fifos, sockets).
+            try:
+                st = os.lstat(abs_path)
+            except OSError:
+                continue
+            # Match the .sh's `find -type f`: only include REGULAR files.
+            # Symlinks are silently dropped in fs-walk mode (only the
+            # git-ls-files branch surfaces committed symlinks, which then
+            # go through _resolve_symlinks for BROKEN_SYMLINK logging).
+            import stat as st_mod
+
+            if not st_mod.S_ISREG(st.st_mode):
+                continue
+            rel = os.path.relpath(abs_path, root_str)
+            out.append(rel)
+    return out
+
+
+def _resolve_symlinks(root_abs: Path, entries: list[str]) -> list[str]:
+    """Substitute each symlink with its resolved real path; drop + log
+    broken / outside-repo links."""
+    resolved: list[str] = []
+    for rel in entries:
+        if not rel:
+            continue
+        abs_path = root_abs / rel
+        if abs_path.is_symlink():
+            if not abs_path.exists():
+                print(f"# BROKEN_SYMLINK: {rel} -> (missing target)", file=sys.stderr)
+                continue
+            try:
+                real = abs_path.resolve()
+            except OSError:
+                print(f"# BROKEN_SYMLINK: {rel} -> (unresolvable)", file=sys.stderr)
+                continue
+            try:
+                real_rel = real.relative_to(root_abs.resolve())
+            except ValueError:
+                print(
+                    f"# BROKEN_SYMLINK: {rel} -> {real} (outside repo)",
+                    file=sys.stderr,
+                )
+                continue
+            resolved.append(str(real_rel))
+        else:
+            resolved.append(rel)
+    return resolved
+
+
+def main(argv: list[str]) -> int:
+    parsed = _parse_argv(argv)
+    if not parsed[0]:
+        _usage()
+        return 2
+    root, slug, force_fs, emit_main = parsed
+
+    if not os.path.isdir(root):
+        print(f"repo-root is not a directory: {root}", file=sys.stderr)
+        return 3
+
+    # Normalize root to an absolute real path so relative computations are
+    # stable. bash used `cd "$ROOT" && pwd -P`; Path.resolve() gives the
+    # equivalent.
+    root_abs = Path(root).resolve()
+
+    if not slug:
+        slug = _slugify(root_abs.name)
+
+    # `plugins/dev-team/knowledge/recon-inventory-excludes.txt` — same
+    # location as the .sh (`dirname/../knowledge/...`).
+    excludes_file = (
+        Path(__file__).resolve().parent.parent
+        / "knowledge"
+        / "recon-inventory-excludes.txt"
+    )
+
+    # Decide branch.
+    is_git = (root_abs / ".git").is_dir() or (root_abs / ".git").is_file()
+    if force_fs or not is_git:
+        source = "filesystem-walk"
+        raw = _collect_fs(root_abs, excludes_file)
+    else:
+        source = "git-ls-files"
+        raw = _collect_git(root_abs)
+
+    resolved = _resolve_symlinks(root_abs, raw)
+
+    # Sort + dedupe under LC_ALL=C for cross-locale determinism.
+    unique = sorted(set(resolved))
+
+    # Emit stdout: the inventory (LF-terminated).
+    for entry in unique:
+        print(entry)
+
+    # Optionally emit the main-envelope JSON fragment.
+    if emit_main:
+        count = len(unique)
+        payload = (
+            f'{{ "source": "{source}", "count": {count}, '
+            f'"sibling_ref": "recon-{slug}.inventory.txt" }}\n'
+        )
+        Path(emit_main).write_text(payload)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))

package/scripts/review_findings_log.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Per-lens review finding categories at /build checkpoints (#2211).
+
+``append`` adds one JSONL row per finding to
+``.claude/metrics/review-findings.jsonl`` and is fail-open (never raises, never
+blocks a phase). ``report`` ranks ``(lens, category)`` by frequency and
+first-pass-fix rate, so authoring checklists promote only the categories that
+actually fire.
+
+Stdlib-only. See docs/python-hook-contract.md.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+DEFAULT_LOG = Path(".claude/metrics/review-findings.jsonl")
+
+
+def append(path: Path, lens: str, category: str, severity: str,
+           iteration: int, fixed: bool) -> bool:
+    row = {
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "lens": lens, "category": category, "severity": severity,
+        "iteration": iteration, "fixed": fixed,
+    }
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open("a", encoding="utf-8") as fh:
+            fh.write(json.dumps(row) + "\n")
+        return True
+    except OSError:
+        return False
+
+
+def rank(rows) -> list[dict]:
+    """Group by (lens, category); first-pass-fix = fixed at iteration 1."""
+    groups: dict = defaultdict(lambda: {"count": 0, "first_pass_fixed": 0})
+    for r in rows:
+        g = groups[(r["lens"], r["category"])]
+        g["count"] += 1
+        if r.get("fixed") and r.get("iteration") == 1:
+            g["first_pass_fixed"] += 1
+    out = [
+        {"lens": lens, "category": cat, "count": g["count"],
+         "first_pass_fix_rate": round(g["first_pass_fixed"] / g["count"], 2)}
+        for (lens, cat), g in groups.items()
+    ]
+    return sorted(out, key=lambda d: (-d["count"], d["lens"], d["category"]))
+
+
+def load(path: Path) -> list[dict]:
+    rows = []
+    try:
+        with path.open(encoding="utf-8") as fh:
+            for line in fh:
+                try:
+                    row = json.loads(line)
+                except ValueError:
+                    continue
+                if isinstance(row, dict) and {"lens", "category"} <= row.keys():
+                    rows.append(row)
+    except OSError:
+        pass
+    return rows
+
+
+def main(argv=None) -> int:
+    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    p.add_argument("--log", type=Path, default=DEFAULT_LOG)
+    sub = p.add_subparsers(dest="cmd", required=True)
+    a = sub.add_parser("append")
+    a.add_argument("--lens", required=True)
+    a.add_argument("--category", required=True)
+    a.add_argument("--severity", default="unknown")
+    a.add_argument("--iteration", type=int, default=1)
+    a.add_argument("--fixed", action="store_true")
+    sub.add_parser("report")
+    args = p.parse_args(argv)
+    if args.cmd == "append":
+        append(args.log, args.lens, args.category, args.severity,
+               args.iteration, args.fixed)
+        return 0  # fail-open
+    print(json.dumps(rank(load(args.log)), indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/run_invariants.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""run_invariants.py — run a slice's `**Invariants:**` commands (issue #865).
+
+Invariants are checks that must stay green *beyond* a slice's own new
+acceptance tests — "don't break X" as a runnable gate term instead of tribal
+knowledge. `/build`'s slice review checkpoint runs each command from the
+repo root after the slice's own suite is green; a non-zero exit fails the
+slice gate exactly like a red test (fix or escalate — never step over).
+
+Each command is run in its own subshell via `shell=True` (same trust model
+as the plan's own test commands — the plan author owns portability). Runs
+stop at the first failure so the failing command is unambiguous.
+
+Stdlib-only. (ADR 0014/0015).
+"""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+from typing import NamedTuple
+
+_HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(_HERE / "lib"))
+
+import plan_parse
+
+
+class InvariantResult(NamedTuple):
+    command: str
+    returncode: int
+    stdout: str
+    stderr: str
+
+
+def run_invariants(commands: list[str], repo_root: str) -> list[InvariantResult]:
+    """Run each command from `repo_root`, stopping at the first failure.
+    Returns the results for every command actually run (in order)."""
+    results: list[InvariantResult] = []
+    for command in commands:
+        proc = subprocess.run(
+            command,
+            shell=True,
+            cwd=repo_root,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        results.append(
+            InvariantResult(command, proc.returncode, proc.stdout, proc.stderr)
+        )
+        if proc.returncode != 0:
+            break
+    return results
+
+
+def invariants_for_slice(plan_path: Path, slice_id: str) -> list[str]:
+    """Return the parsed Invariants commands declared for `slice_id`, or
+    `[]` when the slice declares none."""
+    fields = plan_parse.parse_slice_optional_fields(
+        plan_path.read_text(encoding="utf-8").splitlines()
+    )
+    entry = fields.get(slice_id)
+    if not entry:
+        return []
+    return list(entry.get("invariants") or [])
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="run_invariants.py",
+        description="Run a slice's declared Invariants commands (issue #865).",
+    )
+    parser.add_argument("--repo", default=".", help="Repo root to run commands from.")
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        "--plan", type=Path, help="Plan file — used with --slice to look up commands."
+    )
+    group.add_argument(
+        "command",
+        nargs="*",
+        default=[],
+        help="Explicit invariant command(s) to run.",
+    )
+    parser.add_argument("--slice", help="Slice id (required with --plan).")
+    args = parser.parse_args(argv)
+
+    if args.plan is not None:
+        if not args.slice:
+            parser.error("--slice is required when --plan is given")
+        commands = invariants_for_slice(args.plan, args.slice)
+    else:
+        commands = [c for c in args.command if c]
+
+    if not commands:
+        print("No invariants declared — nothing to run.")
+        return 0
+
+    results = run_invariants(commands, args.repo)
+    for result in results:
+        status = "PASS" if result.returncode == 0 else "FAIL"
+        print(f"[{status}] {result.command}")
+        if result.returncode != 0:
+            if result.stdout.strip():
+                print(result.stdout)
+            if result.stderr.strip():
+                print(result.stderr, file=sys.stderr)
+
+    failed = [r for r in results if r.returncode != 0]
+    if failed:
+        sys.stderr.write(
+            f"run-invariants: invariant failed: {failed[0].command!r} "
+            f"(exit {failed[0].returncode})\n"
+        )
+        return failed[0].returncode or 1
+
+    print(f"All {len(results)} invariant(s) passed.")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())