fleetproof 0.1.0__py3-none-any.whl

fleetproof/__init__.py

@@ -0,0 +1,46 @@
+"""FleetProof — independent, out-of-band verification for agent fleets.
+
+The agent may author the checks; it never executes and grades its own work.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from .runlog import (
+    RunRecord,
+    SubInvocation,
+    child_run_id,
+    current_run_id,
+    list_run_records,
+    load_run,
+    project_root,
+    record,
+    recorded,
+    runs_dir,
+    set_runs_dir,
+)
+from .checks import Check, CheckSpecError, load_checks
+from .checker import CheckReport, CheckResult, run_check, run_checks
+
+__all__ = [
+    "__version__",
+    "recorded",
+    "record",
+    "current_run_id",
+    "child_run_id",
+    "project_root",
+    "runs_dir",
+    "set_runs_dir",
+    "list_run_records",
+    "load_run",
+    "RunRecord",
+    "SubInvocation",
+    "Check",
+    "CheckSpecError",
+    "load_checks",
+    "CheckReport",
+    "CheckResult",
+    "run_check",
+    "run_checks",
+]

fleetproof/__main__.py

@@ -0,0 +1,6 @@
+"""Allow ``python -m fleetproof`` as an alternative to the console script."""
+
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fleetproof/checker.py

@@ -0,0 +1,280 @@
+"""The independent checker.
+
+This is the part of FleetProof that no agent runs against itself. The agent that
+did the work authored ``checks.json``; *this* module executes those checks — each
+declared command in its own OS subprocess — grades the results deterministically,
+and appends the verdict to the run log. It is invoked from the Stop hook, i.e.
+from a process the working agent does not control.
+
+The separation is load-bearing:
+    - the working agent writes the run records and the check spec;
+    - the checker (a different process) executes and grades them.
+
+No LLM is in this path. Grading is pure comparison.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .checks import Check, default_checks_path, load_checks, short_spec_hash, spec_hash
+from .runlog import list_run_records, record, runs_dir
+
+# Per-check wall-clock ceiling. A check that hangs is a failed check, not a hung fleet.
+DEFAULT_TIMEOUT_S = 600
+
+
+@dataclass
+class CheckResult:
+    id: str
+    expectation: str
+    passed: bool
+    blocking: bool
+    returncode: int | None
+    detail: str
+    duration_ms: float
+    stdout_tail: str = ""
+    stderr_tail: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class CheckReport:
+    results: list[CheckResult] = field(default_factory=list)
+    run_id: str | None = None
+    spec_sha256: str | None = None
+
+    @property
+    def total(self) -> int:
+        return len(self.results)
+
+    @property
+    def passed(self) -> int:
+        return sum(1 for r in self.results if r.passed)
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for r in self.results if not r.passed)
+
+    @property
+    def blocking_failures(self) -> list[CheckResult]:
+        return [r for r in self.results if not r.passed and r.blocking]
+
+    @property
+    def verdict(self) -> str:
+        """'pass' unless a blocking check failed. Non-blocking failures do not gate."""
+        return "fail" if self.blocking_failures else "pass"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "verdict": self.verdict,
+            # SHA-256 of the checks.json that governed this verdict. Additive:
+            # records written before this field existed simply omit it, and every
+            # reader treats a missing value as None (no hash on record).
+            "spec_sha256": self.spec_sha256,
+            "summary": {
+                "total": self.total,
+                "passed": self.passed,
+                "failed": self.failed,
+                "blocking_failed": len(self.blocking_failures),
+            },
+            "checks": [r.to_dict() for r in self.results],
+        }
+
+
+def _tail(text: str, limit: int = 2000) -> str:
+    if len(text) <= limit:
+        return text
+    return "…(truncated)…\n" + text[-limit:]
+
+
+def _run_command(command: str, cwd: Path, timeout: int) -> tuple[int | None, str, str]:
+    """Execute a shell command in a separate process. Returns (returncode, stdout, stderr).
+
+    A timeout or spawn failure yields a None returncode, which every grader treats
+    as a failure — the tool never lets an unrunnable check silently pass.
+    """
+    try:
+        proc = subprocess.run(
+            command,
+            shell=True,
+            cwd=str(cwd),
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        return proc.returncode, proc.stdout or "", proc.stderr or ""
+    except subprocess.TimeoutExpired as e:
+        out = e.stdout or ""
+        err = e.stderr or ""
+        if isinstance(out, bytes):
+            out = out.decode(errors="replace")
+        if isinstance(err, bytes):
+            err = err.decode(errors="replace")
+        return None, out, (err + f"\n[fleetproof] command timed out after {timeout}s")
+    except OSError as e:
+        return None, "", f"[fleetproof] could not launch command: {e}"
+
+
+def _grade(check: Check, returncode: int | None, stdout: str, stderr: str, cwd: Path) -> tuple[bool, str]:
+    """Return (passed, human-readable detail) for one check. Pure comparison."""
+    kind = check.expect["kind"]
+
+    if kind in ("exit0", "exit"):
+        want = 0 if kind == "exit0" else check.expect["code"]
+        if returncode is None:
+            return False, f"command did not complete (expected exit {want})"
+        ok = returncode == want
+        return ok, f"exit {returncode} (expected {want})"
+
+    if kind == "regex":
+        if returncode is None:
+            return False, "command did not complete; cannot match output"
+        pattern = check.expect["pattern"]
+        combined = f"{stdout}\n{stderr}"
+        ok = re.search(pattern, combined) is not None
+        return ok, ("matched" if ok else "no match") + f" for /{pattern}/"
+
+    if kind == "file_exists":
+        target = (cwd / check.expect["path"]).resolve()
+        ok = target.exists()
+        return ok, ("present" if ok else "missing") + f": {check.expect['path']}"
+
+    return False, f"unknown expectation kind: {kind}"
+
+
+def run_check(check: Check, cwd: Path, timeout: int = DEFAULT_TIMEOUT_S) -> CheckResult:
+    """Execute and grade a single check in its own subprocess."""
+    import time as _time
+    started = _time.perf_counter()
+    returncode: int | None = None
+    stdout = stderr = ""
+    if check.run is not None:
+        returncode, stdout, stderr = _run_command(check.run, cwd, timeout)
+    passed, detail = _grade(check, returncode, stdout, stderr, cwd)
+    duration_ms = (_time.perf_counter() - started) * 1000.0
+    return CheckResult(
+        id=check.id,
+        expectation=check.describe_expectation(),
+        passed=passed,
+        blocking=check.block,
+        returncode=returncode,
+        detail=detail,
+        duration_ms=round(duration_ms, 3),
+        stdout_tail=_tail(stdout),
+        stderr_tail=_tail(stderr),
+    )
+
+
+def run_checks(
+    checks: list[Check] | None = None,
+    *,
+    cwd: Path | None = None,
+    timeout: int = DEFAULT_TIMEOUT_S,
+    record_to_log: bool = True,
+    spec_path: Path | None = None,
+) -> CheckReport:
+    """Run every check and (by default) append the verdict to the run log.
+
+    ``checks`` defaults to the loaded ``.fleetproof/checks.json``. ``cwd`` defaults
+    to the current working directory — the directory the fleet actually worked in.
+    ``spec_path`` is the check-spec file whose bytes get hashed onto the verdict;
+    when omitted it is resolved from ``cwd`` the same way the checker itself finds
+    the spec, so the recorded hash always describes the spec that governed the run.
+    """
+    if checks is None:
+        checks = load_checks(spec_path)
+    work_dir = Path(cwd) if cwd is not None else Path.cwd()
+    resolved_spec = Path(spec_path) if spec_path is not None else default_checks_path(work_dir)
+    sha = spec_hash(resolved_spec)
+
+    report = CheckReport(spec_sha256=sha)
+    if not record_to_log:
+        for check in checks:
+            report.results.append(run_check(check, work_dir, timeout))
+        return report
+
+    with record("fleetproof", "check", {"check_count": len(checks)}) as handle:
+        report.run_id = handle.run_id
+        for check in checks:
+            report.results.append(run_check(check, work_dir, timeout))
+        payload = report.to_dict()
+        payload["recorded_from_pid"] = _self_pid()
+        handle.set_output(payload)
+    return report
+
+
+def session_spec_baseline(session_id: str | None) -> str | None:
+    """The spec hash of the EARLIEST recorded checker verdict in ``session_id``.
+
+    This is the baseline every later verdict in the same session is compared
+    against: if a verdict's own spec hash differs from this, the checks.json was
+    edited mid-session (spec drift). Legacy verdicts with no hash on record are
+    skipped, so the baseline is the earliest verdict that actually carries one.
+    Returns None when there is no session context or no hashed verdict yet — in
+    which case there is nothing to drift *from*, so callers treat it as no drift.
+    """
+    if not session_id:
+        return None
+    hashed: list[tuple[str, str]] = []
+    for run in list_run_records():
+        if (run.session_id or None) != session_id:
+            continue
+        for sub in run.sub_invocations:
+            if sub.tool != "fleetproof" or sub.subcmd != "check":
+                continue
+            payload = sub.load_output()
+            if not isinstance(payload, dict):
+                continue
+            sha = payload.get("spec_sha256")
+            if sha:
+                order_key = sub.started_at or run.started_at or ""
+                hashed.append((order_key, sha))
+    if not hashed:
+        return None
+    hashed.sort(key=lambda item: item[0])
+    return hashed[0][1]
+
+
+def spec_drifted(current_hash: str | None, session_id: str | None) -> tuple[bool, str | None]:
+    """Return ``(drifted, baseline_hash)`` for a verdict's hash within a session.
+
+    ``drifted`` is True only when a session baseline exists and the current hash
+    differs from it — a v0.1 pass-with-drift still passes, this just flags it.
+    """
+    baseline = session_spec_baseline(session_id)
+    drifted = bool(baseline and current_hash and baseline != current_hash)
+    return drifted, baseline
+
+
+def _self_pid() -> int:
+    import os
+    return os.getpid()
+
+
+def format_report_text(report: CheckReport) -> str:
+    """A compact plain-text rendering for the CLI and hook feedback."""
+    lines = []
+    for r in report.results:
+        mark = "PASS" if r.passed else ("FAIL" if r.blocking else "warn")
+        lines.append(f"  [{mark}] {r.id}: {r.detail}")
+    s = report.to_dict()["summary"]
+    lines.append(
+        f"{report.verdict.upper()} - {s['passed']}/{s['total']} passed, "
+        f"{s['blocking_failed']} blocking failure(s)."
+    )
+    lines.append(f"spec: {short_spec_hash(report.spec_sha256)}")
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":  # pragma: no cover - manual smoke entry
+    rep = run_checks()
+    print(format_report_text(rep))
+    sys.exit(1 if rep.verdict == "fail" else 0)

fleetproof/checks.py

@@ -0,0 +1,218 @@
+"""Check-spec loader.
+
+A check spec is a JSON file — ``.fleetproof/checks.json`` by default — that
+declares, up front, what "done" is supposed to mean for this repo. An agent is
+allowed to *author* this file. It is never allowed to grade itself against it:
+that job belongs to :mod:`fleetproof.checker`, which runs in a separate process.
+
+Spec shape::
+
+    {
+      "checks": [
+        {
+          "id": "tests-pass",
+          "run": "python -m pytest -q",
+          "expect": "exit0",
+          "block": true,
+          "description": "Unit tests must pass."
+        }
+      ]
+    }
+
+``expect`` is one of:
+    "exit0"                        command must exit 0 (default if omitted)
+    {"exit": N}                    command must exit with code N
+    {"regex": "pattern"}           combined stdout+stderr must match the regex
+    {"file_exists": "path"}        path (relative to the run cwd) must exist
+
+JSON, not YAML, on purpose: the runtime is stdlib-only, so there is no third-
+party parser to trust in a tool whose entire job is being trustworthy.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .runlog import PROJECT_MARKER, project_root
+
+DEFAULT_CHECKS_FILENAME = "checks.json"
+
+_VALID_EXPECT_KEYS = {"exit", "regex", "file_exists"}
+
+# The line a drifted verdict carries everywhere it surfaces (gate output, report,
+# CLI). One canonical string so the three surfaces never disagree about wording.
+SPEC_DRIFT_NOTE = (
+    "NOTE: .fleetproof/checks.json was modified during this session "
+    "(spec drift). Review the diff before trusting this verdict."
+)
+
+
+class CheckSpecError(Exception):
+    """Raised when a check spec is missing or malformed."""
+
+
+@dataclass(frozen=True)
+class Check:
+    """One declared check. Immutable — the spec is a contract, not a scratchpad."""
+    id: str
+    run: str | None
+    expect: dict[str, Any]  # normalized: {"kind": ..., ...}
+    block: bool
+    description: str = ""
+
+    def describe_expectation(self) -> str:
+        kind = self.expect["kind"]
+        if kind == "exit0":
+            return "exit code 0"
+        if kind == "exit":
+            return f"exit code {self.expect['code']}"
+        if kind == "regex":
+            return f"output matches /{self.expect['pattern']}/"
+        if kind == "file_exists":
+            return f"file exists: {self.expect['path']}"
+        return kind
+
+
+def default_checks_path(start: Path | None = None) -> Path:
+    """Return the default check-spec location for the project rooted at ``start``."""
+    return project_root(start) / PROJECT_MARKER / DEFAULT_CHECKS_FILENAME
+
+
+def spec_hash(path: Path | None = None) -> str | None:
+    """SHA-256 (hex) of the check-spec bytes, or ``None`` if it can't be read.
+
+    Hashed over the raw file bytes, not the parsed spec: the point is to notice
+    that *the file the agent authored changed* mid-session, which a byte hash
+    catches even for edits (whitespace, key reordering) that parse identically.
+    Returns None rather than raising so a missing/unreadable spec degrades to
+    "no hash on record" instead of breaking a gate run.
+    """
+    spec_path = Path(path) if path is not None else default_checks_path()
+    try:
+        return hashlib.sha256(spec_path.read_bytes()).hexdigest()
+    except OSError:
+        return None
+
+
+def short_spec_hash(full: str | None, length: int = 12) -> str:
+    """A git-style short form of a spec hash for display; 'unknown' when absent."""
+    if not full:
+        return "unknown"
+    return full[:length]
+
+
+def load_checks(path: Path | None = None) -> list[Check]:
+    """Load and validate the check spec. Raises CheckSpecError on any problem."""
+    spec_path = Path(path) if path is not None else default_checks_path()
+    if not spec_path.exists():
+        raise CheckSpecError(
+            f"No check spec found at {spec_path}. Run `fleetproof init` to create one."
+        )
+    try:
+        raw = json.loads(spec_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as e:
+        raise CheckSpecError(f"Check spec {spec_path} is not valid JSON: {e}") from e
+
+    if not isinstance(raw, dict) or "checks" not in raw:
+        raise CheckSpecError("Check spec must be an object with a top-level 'checks' array.")
+    entries = raw["checks"]
+    if not isinstance(entries, list):
+        raise CheckSpecError("'checks' must be an array.")
+
+    checks: list[Check] = []
+    seen_ids: set[str] = set()
+    for i, entry in enumerate(entries):
+        check = _parse_check(entry, i)
+        if check.id in seen_ids:
+            raise CheckSpecError(f"Duplicate check id: {check.id!r}")
+        seen_ids.add(check.id)
+        checks.append(check)
+    return checks
+
+
+def _parse_check(entry: Any, index: int) -> Check:
+    where = f"checks[{index}]"
+    if not isinstance(entry, dict):
+        raise CheckSpecError(f"{where} must be an object.")
+
+    cid = entry.get("id")
+    if not cid or not isinstance(cid, str):
+        raise CheckSpecError(f"{where} is missing a string 'id'.")
+
+    run = entry.get("run")
+    if run is not None and not isinstance(run, str):
+        raise CheckSpecError(f"{where} ({cid}): 'run' must be a string or omitted.")
+
+    block = entry.get("block", True)
+    if not isinstance(block, bool):
+        raise CheckSpecError(f"{where} ({cid}): 'block' must be true or false.")
+
+    description = entry.get("description", "")
+    if not isinstance(description, str):
+        raise CheckSpecError(f"{where} ({cid}): 'description' must be a string.")
+
+    expect = _normalize_expect(entry.get("expect", "exit0"), cid, where)
+
+    # A file_exists check may have no command; every other kind needs one.
+    if run is None and expect["kind"] != "file_exists":
+        raise CheckSpecError(
+            f"{where} ({cid}): a '{expect['kind']}' check requires a 'run' command."
+        )
+
+    return Check(id=cid, run=run, expect=expect, block=block, description=description)
+
+
+def _normalize_expect(expect: Any, cid: str, where: str) -> dict[str, Any]:
+    if expect == "exit0":
+        return {"kind": "exit0"}
+    if not isinstance(expect, dict) or len(expect) != 1:
+        raise CheckSpecError(
+            f"{where} ({cid}): 'expect' must be \"exit0\" or a single-key object "
+            f"({sorted(_VALID_EXPECT_KEYS)})."
+        )
+    key, val = next(iter(expect.items()))
+    if key not in _VALID_EXPECT_KEYS:
+        raise CheckSpecError(f"{where} ({cid}): unknown expect kind {key!r}.")
+    if key == "exit":
+        if not isinstance(val, int) or isinstance(val, bool):
+            raise CheckSpecError(f"{where} ({cid}): expect.exit must be an integer.")
+        return {"kind": "exit", "code": val}
+    if key == "regex":
+        if not isinstance(val, str):
+            raise CheckSpecError(f"{where} ({cid}): expect.regex must be a string.")
+        return {"kind": "regex", "pattern": val}
+    # file_exists
+    if not isinstance(val, str):
+        raise CheckSpecError(f"{where} ({cid}): expect.file_exists must be a path string.")
+    return {"kind": "file_exists", "path": val}
+
+
+STARTER_SPEC: dict[str, Any] = {
+    "checks": [
+        {
+            "id": "tests-pass",
+            "run": "python -m pytest -q",
+            "expect": "exit0",
+            "block": True,
+            "description": "Unit tests must pass before a task can be called done.",
+        },
+        {
+            "id": "build-succeeds",
+            "run": "python -m build",
+            "expect": "exit0",
+            "block": True,
+            "description": "The package must build. Edit or delete if you do not ship a package.",
+        },
+        {
+            "id": "changelog-updated",
+            "run": None,
+            "expect": {"file_exists": "CHANGELOG.md"},
+            "block": False,
+            "description": "Advisory reminder — non-blocking example of a file_exists check.",
+        },
+    ]
+}