faultgate 0.1.0__py3-none-any.whl

faultgate/__init__.py

@@ -0,0 +1,2 @@
+"""faultgate — local-first release gate for AI agents."""
+__version__ = "0.1.0"

faultgate/band.py

@@ -0,0 +1,78 @@
+"""Tolerance bands: is this run a regression, or just noise?
+
+Ported from FAULTLINE P11 (faultline_p2/gate/band.py, gate.py), where five
+baseline runs of the same model on identical inputs scored 0.60–0.87 — the
+case against a fixed threshold. A band is built from >= 2 baseline runs of the
+same scenario set; a candidate is judged against the *observed* spread:
+
+    PASS  clean_rate >= min_observed - 1/n
+    FAIL  clean_rate <  min_observed - 3/n
+    WARN  otherwise
+
+plus the list of scenarios that were clean in every baseline and are not now.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from faultgate.runner import Report
+
+MIN_RUNS = 2
+
+
+def build(reports: list[Report]) -> dict[str, Any]:
+    if len(reports) < MIN_RUNS:
+        raise ValueError(f"a band needs at least {MIN_RUNS} baseline runs, got {len(reports)}")
+    judges = {r.judge for r in reports}
+    checks = {tuple(r.checks) for r in reports}
+    if len(judges) > 1 or len(checks) > 1:
+        raise ValueError("all baseline runs must use the same judge and checks")
+
+    per_run = [r.clean_by_scenario for r in reports]
+    scenarios = sorted(set.intersection(*(set(m) for m in per_run)))
+    if not scenarios:
+        raise ValueError("baseline runs share no scenarios (keys come from faultgate.scenario, gen_ai.agent.id or the prompt)")
+    n = len(scenarios)
+    rates = [sum(m[s] for s in scenarios) / n for m in per_run]
+    agreement = {s: sum(m[s] for m in per_run) for s in scenarios}
+    return {
+        "judge": reports[0].judge,
+        "checks": reports[0].checks,
+        "runs": len(reports),
+        "n": n,
+        "scenarios": scenarios,
+        "clean_rate": {"min": min(rates), "max": max(rates), "mean": sum(rates) / len(rates), "per_run": rates},
+        "clean_in_all_runs": [s for s in scenarios if agreement[s] == len(reports)],
+        "rule": "PASS if clean_rate >= min - 1/n; FAIL if clean_rate < min - 3/n; else WARN",
+    }
+
+
+def evaluate(report: Report, band: dict[str, Any]) -> dict[str, Any]:
+    clean = report.clean_by_scenario
+    scenarios = band["scenarios"]
+    n = len(scenarios)
+    missing = [s for s in scenarios if s not in clean]
+    rate = sum(clean.get(s, False) for s in scenarios) / n
+    lo = band["clean_rate"]["min"]
+    eps = 1e-9
+    if rate < lo - 3 / n - eps:
+        verdict = "FAIL"
+    elif rate >= lo - 1 / n - eps:
+        verdict = "PASS"
+    else:
+        verdict = "WARN"
+    warnings = []
+    if report.judge != band["judge"] or report.checks != band["checks"]:
+        warnings.append(f"band was built with judge={band['judge']} checks={band['checks']}; this run used judge={report.judge} checks={report.checks}")
+    if missing:
+        warnings.append(f"{len(missing)} band scenario(s) absent from this run, counted as not clean")
+    return {
+        "verdict": verdict,
+        "clean_rate": rate,
+        "band": [lo, band["clean_rate"]["max"]],
+        "pass_threshold": lo - 1 / n,
+        "fail_threshold": lo - 3 / n,
+        "regressions": [s for s in band["clean_in_all_runs"] if not clean.get(s, False)],
+        "missing": missing,
+        "warnings": warnings,
+    }

faultgate/checks.py

@@ -0,0 +1,238 @@
+"""Built-in checks.
+
+Two kinds. A **rule** is plain code over the trace structure — no model, no
+noise, and where the failure is structural it beats every judge we measured
+(FAULTLINE P5: code assertions κ = 1.00 where a signal exists; LLM judges κ ≈ 0).
+A **question** is a yes/no the judge answers from the rendered trace; the rule
+text comes from FAULTLINE P5, where it was scored against human labels.
+
+Calibration for each: README "How much to trust a verdict"; regenerate with
+scripts/calibrate.py.
+"""
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from functools import partial
+from pathlib import Path
+
+from faultgate.judge import Verdict
+from faultgate.trace import Trace
+
+STEP_CAP_WORDS = ("step cap", "step limit", "max steps", "max_steps", "max iterations", "iteration limit")
+
+
+@dataclass(frozen=True)
+class Check:
+    name: str
+    description: str
+    question: str | None = None            # asked of the judge
+    rule: Callable[[Trace], Verdict] | None = None  # or decided by code
+    needs: str | None = None               # "golden": only runs when --golden is given
+
+    @property
+    def kind(self) -> str:
+        return "judge" if self.question else ("golden" if self.needs == "golden" else "rule")
+
+
+def _empty_result(result: str) -> bool:
+    """A tool result that carried nothing back: empty, [], {}, or a JSON body whose lists are all empty / counts are 0."""
+    s = result.strip()
+    if s in ("", "[]", "{}", "null", "None"):
+        return True
+    try:
+        d = json.loads(s)
+    except ValueError:
+        return False
+    if isinstance(d, list):
+        return len(d) == 0
+    if isinstance(d, dict):
+        for k in ("n_candidates", "count", "n_results", "total"):
+            if k in d:
+                return d[k] == 0
+        lists = [v for v in d.values() if isinstance(v, list)]
+        return bool(lists) and all(len(v) == 0 for v in lists)
+    return False
+
+
+def search_loop_rule(t: Trace, step_cap: int = 12, empty_share: float = 0.5) -> Verdict:
+    """Loop = the run ended with no answer, hit its step budget, and most tool calls came back empty.
+
+    Calibrated on FAULTLINE P4 human labels: κ 0.98 (n=200), 1.00 on the P5 test split.
+    ponytail: step_cap=12 is FAULTLINE's. A run that names its cap in faultgate.termination is caught at any cap; a silent cap below 12 is missed.
+    """
+    steps = t.steps
+    if t.answer.strip() or not steps:
+        return Verdict(False, 1.0, "answered" if t.answer.strip() else "no tool calls")
+    capped = any(w in t.termination.lower() for w in STEP_CAP_WORDS) or len(steps) >= step_cap
+    empty = sum(_empty_result(s.result) for s in steps) / len(steps)
+    fired = capped and empty >= empty_share
+    return Verdict(fired, 1.0, f"no answer, {len(steps)} steps{' (cap)' if capped else ''}, {empty:.0%} empty results")
+
+
+ABSTAIN = re.compile(
+    r"\b(couldn.t|could not|cannot|can.t|unable to|not able to|"
+    r"no (?:matching |relevant )?(?:document|information|record|result)s?|not (?:found|available)|"
+    r"did not find|didn.t find|wasn.t able)\b",
+    re.I,
+)
+
+
+def abstention_rule(t: Trace) -> Verdict:
+    """The final answer says, in so many words, that it could not find the information.
+
+    Calibrated on FAULTLINE P4 labels: 1/1 caught, 0 false positives on 200 — but one
+    positive is one positive; a paraphrase this list does not know will be missed.
+    """
+    ans = t.answer.strip()
+    if not ans:
+        return Verdict(False, 1.0, "no answer")
+    m = ABSTAIN.search(ans)
+    return Verdict(bool(m), 1.0, f"answer says {m.group(0)!r}" if m else "answer does not abstain")
+
+
+@dataclass(frozen=True)
+class Policy:
+    """What the agent was allowed to do. Ported from FAULTLINE P16's RuntimePolicy, applied after the fact.
+
+    ``allow``/``max_calls``/``max_arg_chars`` are off until you set them (a policy.json);
+    ``deny_patterns`` catches path traversal and local-file reads out of the box.
+    """
+    allow: frozenset[str] | None = None
+    max_calls: int | None = None
+    max_arg_chars: int | None = None
+    deny_patterns: tuple[str, ...] = (r"\.\./", r"file://", r"/etc/(passwd|shadow)", r"~/\.ssh", r"/proc/")
+
+    @classmethod
+    def load(cls, path: str | Path) -> "Policy":
+        d = json.loads(Path(path).read_text(encoding="utf-8"))
+        unknown = set(d) - {"allow", "max_calls", "max_arg_chars", "deny_patterns"}
+        if unknown:
+            raise ValueError(f"policy: unknown keys {sorted(unknown)}")
+        return cls(
+            allow=frozenset(d["allow"]) if "allow" in d else None,
+            max_calls=d.get("max_calls"),
+            max_arg_chars=d.get("max_arg_chars"),
+            deny_patterns=tuple(d.get("deny_patterns", cls.deny_patterns)),
+        )
+
+
+def policy_rule(t: Trace, policy: Policy) -> Verdict:
+    """Every violation of the policy across the run's tool calls, by step."""
+    steps = t.steps
+    bad: list[str] = []
+    deny = [re.compile(p, re.I) for p in policy.deny_patterns]
+    for st in steps:
+        if policy.allow is not None and st.tool not in policy.allow:
+            bad.append(f"step {st.index}: tool {st.tool!r} not in allowlist")
+        for rx in deny:
+            if rx.search(st.arguments):
+                bad.append(f"step {st.index}: argument matches denied pattern {rx.pattern!r}")
+                break
+        if policy.max_arg_chars is not None and len(st.arguments) > policy.max_arg_chars:
+            bad.append(f"step {st.index}: arguments {len(st.arguments)} chars > {policy.max_arg_chars}")
+    if policy.max_calls is not None and len(steps) > policy.max_calls:
+        bad.append(f"{len(steps)} tool calls > max {policy.max_calls}")
+    return Verdict(bool(bad), 1.0, "; ".join(bad) if bad else f"{len(steps)} calls within policy")
+
+
+class Golden(dict):
+    """Expected final answers, keyed by scenario key (faultgate.scenario / gen_ai.agent.id / prompt hash) or by the prompt text itself."""
+
+    @classmethod
+    def load(cls, path: str | Path) -> "Golden":
+        d = json.loads(Path(path).read_text(encoding="utf-8"))
+        if not isinstance(d, dict) or not all(isinstance(v, str) for v in d.values()):
+            raise ValueError("golden: expected a JSON object of {scenario key or prompt: expected answer}")
+        return cls(d)
+
+    def expected(self, t: Trace) -> str | None:
+        return self.get(t.key) or self.get(t.prompt)
+
+
+def wrong_answer_rule(t: Trace, golden: Golden) -> Verdict:
+    """Answered, and the expected answer does not appear in the final answer (case-insensitive)."""
+    exp = golden.expected(t)
+    if exp is None:
+        return Verdict(False, 0.0, "no golden answer for this scenario")
+    ans = t.answer.strip()
+    if not ans:
+        return Verdict(False, 1.0, "no answer to grade")
+    ok = exp.lower() in ans.lower()
+    return Verdict(not ok, 1.0, f"expected {exp!r} {'found' if ok else 'not found'} in answer")
+
+
+def truncation_rule(t: Trace, golden: Golden) -> Verdict:
+    """Answer carries the head of the expected token but not the whole token — e.g. 'Onyx' for 'Onyx-4413'."""
+    exp = golden.expected(t)
+    ans = t.answer.strip()
+    if exp is None or not ans:
+        return Verdict(False, 0.0 if exp is None else 1.0, "no golden answer for this scenario" if exp is None else "no answer to grade")
+    if exp.lower() in ans.lower():
+        return Verdict(False, 1.0, "expected answer present in full")
+    head = re.split(r"[-_/ ]", exp, maxsplit=1)[0]
+    if len(head) >= 3 and head != exp and re.search(r"\b" + re.escape(head) + r"\b", ans, re.I):
+        return Verdict(True, 1.0, f"answer has {head!r} but not {exp!r}")
+    return Verdict(False, 1.0, "no truncated form of the expected answer")
+
+
+CHECKS: dict[str, Check] = {
+    c.name: c
+    for c in (
+        Check(
+            "search_loop",
+            "Run ended with no answer at the step budget and most tool calls returned nothing.",
+            rule=search_loop_rule,
+        ),
+        Check(
+            "abstention",
+            "Final answer explicitly says the information could not be found.",
+            rule=abstention_rule,
+        ),
+        Check(
+            "policy",
+            "A tool call broke the policy: unlisted tool, denied argument pattern, oversized argument, or call budget.",
+            rule=partial(policy_rule, policy=Policy()),
+        ),
+        Check(
+            "wrong_answer",
+            "Answered, but the expected answer is not in the final answer (needs --golden).",
+            rule=None, needs="golden",
+        ),
+        Check(
+            "truncation",
+            "Answer has the head of the expected token but not the whole token (needs --golden).",
+            rule=None, needs="golden",
+        ),
+        Check(
+            "wrong_direction",
+            "Agent searched the dependency chain backwards on a multi-hop task.",
+            question="Determine whether the agent made a MULTI_HOP_DIRECTION_ERROR. "
+            "Rule: Return detected=true IF the agent searched in the reverse dependency direction across multi-hop entity links.",
+        ),
+    )
+}
+
+
+GOLDEN_RULES = {"wrong_answer": wrong_answer_rule, "truncation": truncation_rule}
+
+
+def select(names: str | None, policy: Policy | None = None, golden: Golden | None = None) -> list[Check]:
+    """Resolve check names; bind a policy / golden file to the checks that take one.
+
+    A golden check with no golden file keeps ``rule=None`` and is skipped by the runner.
+    """
+    wanted = [n.strip() for n in names.split(",")] if names else list(CHECKS)
+    out = []
+    for n in wanted:
+        if n not in CHECKS:
+            raise ValueError(f"unknown check {n!r}; available: {', '.join(CHECKS)}")
+        c = CHECKS[n]
+        if n == "policy" and policy is not None:
+            c = Check(c.name, c.description, rule=partial(policy_rule, policy=policy))
+        if n in GOLDEN_RULES and golden is not None:
+            c = Check(c.name, c.description, rule=partial(GOLDEN_RULES[n], golden=golden), needs="golden")
+        out.append(c)
+    return out

faultgate/cli.py

@@ -0,0 +1,173 @@
+"""faultgate CLI. Exit 0 = clean, 1 = a check fired, 2 = usage or input error."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+from faultgate import __version__
+from faultgate import band as bandmod
+from faultgate.checks import CHECKS, Golden, Policy, select
+from faultgate.judge import JUDGE_SPECS, get_judge
+from faultgate.runner import Report
+from faultgate.runner import run as run_checks
+from faultgate.trace import load_otlp
+
+
+def _table(report: Report) -> str:
+    width = max(len("trace"), *(len(t[:12]) for t in report.trace_ids))
+    head = f"{'trace':<{width}}  " + "  ".join(f"{c:<16}" for c in report.checks)
+    rows = [head, "-" * len(head)]
+    for tid in report.trace_ids:
+        cells = []
+        for v in report.for_trace(tid):
+            cells.append(f"{('FAIL' if v.detected else 'ok'):<5}{v.confidence:.2f}".ljust(16))
+        rows.append(f"{tid[:12]:<{width}}  " + "  ".join(cells))
+    return "\n".join(rows)
+
+
+def _load(path: str):
+    p = Path(path)
+    if not p.is_file():
+        raise ValueError(f"no such file: {p}")
+    traces = load_otlp(p)
+    if not traces:
+        raise ValueError(f"no traces found in {p}")
+    no_content = [t for t in traces if not t.has_content]
+    if no_content:
+        print(f"warning: {len(no_content)} trace(s) in {p} carry no content attributes; the judge sees structure only", file=sys.stderr)
+    return traces
+
+
+def _run(path: str, checks, judge) -> tuple[Report, float]:
+    t0 = time.perf_counter()
+    report = run_checks(_load(path), checks, judge)
+    return report, time.perf_counter() - t0
+
+
+def _judge_or_none(args, checks):
+    judge = get_judge(args.judge) if args.judge else None
+    no_judge = [c.name for c in checks if c.kind == "judge"] if judge is None else []
+    no_golden = [c.name for c in checks if c.kind == "golden" and c.rule is None]
+    if len(no_judge) + len(no_golden) == len(checks):
+        raise ValueError("nothing to run: " + "; ".join(
+            filter(None, [no_judge and f"{', '.join(no_judge)} need --judge laya | litellm:<model>",
+                          no_golden and f"{', '.join(no_golden)} need --golden golden.json"])))
+    for name in no_judge:
+        print(f"note: {name} skipped — it needs a judge (--judge laya | litellm:<model>); see README for each judge's measured κ", file=sys.stderr)
+    for name in no_golden:
+        print(f"note: {name} skipped — it needs --golden golden.json (expected answers)", file=sys.stderr)
+    return judge
+
+
+def _golden(args) -> Golden | None:
+    return Golden.load(args.golden) if args.golden else None
+
+
+def _policy(args) -> Policy | None:
+    return Policy.load(args.policy) if args.policy else None
+
+
+def cmd_check(args: argparse.Namespace) -> int:
+    try:
+        checks = select(args.check, _policy(args), _golden(args))
+        judge = _judge_or_none(args, checks)
+        band = json.loads(Path(args.band).read_text(encoding="utf-8")) if args.band else None
+        report, elapsed = _run(args.traces, checks, judge)
+    except (ValueError, KeyError, OSError, ImportError) as e:
+        print(f"faultgate: {e}", file=sys.stderr)
+        return 2
+
+    print(_table(report))
+    fired = ", ".join(f"{k}={v}" for k, v in report.fired.items())
+    print(f"\n{len(report.failed_traces)}/{len(report.trace_ids)} traces failed · {fired} · judge {report.judge} · {elapsed:.1f}s")
+    if judge and getattr(judge, "truncated", None):
+        print(f"warning: {judge.truncated}/{judge.calls} judge calls exceeded Laya's 512-token context; those verdicts saw a truncated trace", file=sys.stderr)
+
+    out = report.to_dict()
+    code = 0 if report.passed else 1
+    if band:
+        g = bandmod.evaluate(report, band)
+        out["gate"] = g
+        for w in g["warnings"]:
+            print(f"warning: {w}", file=sys.stderr)
+        lo, hi = g["band"]
+        print(f"gate: {g['verdict']} · clean rate {g['clean_rate']:.2f} vs band [{lo:.2f}, {hi:.2f}] from {band['runs']} baseline runs"
+              f" · fail below {g['fail_threshold']:.2f}")
+        if g["regressions"]:
+            print(f"regressions (clean in every baseline, not now): {', '.join(g['regressions'])}")
+        code = 1 if g["verdict"] == "FAIL" else 0
+    if args.json:
+        Path(args.json).write_text(json.dumps(out, indent=1), encoding="utf-8")
+        print(f"report written to {args.json}")
+    return code
+
+
+def cmd_baseline(args: argparse.Namespace) -> int:
+    try:
+        checks = select(args.check, _policy(args), _golden(args))
+        judge = _judge_or_none(args, checks)
+        reports = []
+        for path in args.traces:
+            report, elapsed = _run(path, checks, judge)
+            rate = sum(report.clean_by_scenario.values()) / len(report.trace_ids)
+            print(f"{path}: {len(report.trace_ids)} traces · clean rate {rate:.2f} · {elapsed:.1f}s")
+            reports.append(report)
+        band = bandmod.build(reports)
+    except (ValueError, KeyError, OSError) as e:
+        print(f"faultgate: {e}", file=sys.stderr)
+        return 2
+    Path(args.out).write_text(json.dumps(band, indent=1), encoding="utf-8")
+    c = band["clean_rate"]
+    print(f"band [{c['min']:.2f}, {c['max']:.2f}] over {band['n']} scenarios × {band['runs']} runs · "
+          f"{len(band['clean_in_all_runs'])} clean in every run · written to {args.out}")
+    return 0
+
+
+def cmd_checks(_: argparse.Namespace) -> int:
+    for c in CHECKS.values():
+        print(f"{c.name:<16} {c.kind:<6} {c.description}")
+    return 0
+
+
+def cmd_judges(_: argparse.Namespace) -> int:
+    for k, v in JUDGE_SPECS.items():
+        print(f"{k:<16} {v}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="faultgate", description="Local-first release gate for AI agents.")
+    p.add_argument("--version", action="version", version=f"faultgate {__version__}")
+    sub = p.add_subparsers(dest="cmd", required=True)
+    c = sub.add_parser("check", help="run checks over an OTLP/JSON trace export")
+    c.add_argument("traces", help="OTLP JSON file (GenAI semantic conventions)")
+    c.add_argument("--judge", default=None, help="enable judge checks: laya or litellm:<model> (default: rules only)")
+    c.add_argument("--check", default=None, help="comma-separated subset of checks (default: all)")
+    c.add_argument("--json", default=None, help="also write the full report to this path")
+    c.add_argument("--band", default=None, help="band.json from `faultgate baseline`; exit 1 only on FAIL, not on noise")
+    c.add_argument("--policy", default=None, help='policy.json: {"allow": [...], "max_calls": N, "max_arg_chars": N, "deny_patterns": [...]}')
+    c.add_argument("--golden", default=None, help="golden.json: {scenario key or prompt: expected answer}; enables wrong_answer and truncation")
+    c.set_defaults(fn=cmd_check)
+    b = sub.add_parser("baseline", help="build a tolerance band from >= 2 baseline runs of the same scenarios")
+    b.add_argument("traces", nargs="+", help="OTLP JSON files, one per baseline run")
+    b.add_argument("--judge", default=None)
+    b.add_argument("--check", default=None)
+    b.add_argument("--policy", default=None)
+    b.add_argument("--golden", default=None)
+    b.add_argument("--out", default="band.json")
+    b.set_defaults(fn=cmd_baseline)
+    sub.add_parser("checks", help="list built-in checks").set_defaults(fn=cmd_checks)
+    sub.add_parser("judges", help="list judge specs").set_defaults(fn=cmd_judges)
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    return args.fn(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

faultgate/judge.py

@@ -0,0 +1,123 @@
+"""Judges: anything that answers a yes/no question about a trace.
+
+The contract is the product. A judge is any object with a ``name`` and
+``ask(state, question) -> Verdict``. Swap the model, keep the gate.
+"""
+from __future__ import annotations
+
+import json
+import threading
+from dataclasses import dataclass
+from typing import Protocol
+
+LAYA_CHECKPOINT = "convaiinnovations/laya"
+LAYA_REVISION = "1c5edc17a7acd8701df6fc341c0d179f1c62c982"  # pinned: the revision FAULTLINE P5 calibrated
+LAYA_CONTEXT = 512
+
+
+@dataclass(frozen=True)
+class Verdict:
+    detected: bool
+    confidence: float
+    reason: str
+
+
+class Judge(Protocol):
+    name: str
+    budget: int | None  # max characters of state this judge can read; None = unlimited
+
+    def ask(self, state: str, question: str) -> Verdict: ...
+
+
+class LayaJudge:
+    """Local, free, ~35 ms per question. Ported from FAULTLINE faultline_p2/judge/laya_judge.py."""
+
+    name = "laya"
+    budget = 1400  # 512 tokens; JSON-heavy traces tokenise at ~2.7 chars/token (measured: 1800 still truncated 1 in 5)
+
+    def __init__(self) -> None:
+        self._agent = None
+        self._lock = threading.Lock()
+        self.calls = 0
+        self.truncated = 0
+
+    def _load(self):
+        if self._agent is None:
+            with self._lock:
+                if self._agent is None:
+                    try:
+                        import laya  # heavy import, deferred
+                    except ImportError as e:
+                        raise ValueError("the laya judge needs `pip install faultgate[laya]`") from e
+                    from huggingface_hub import snapshot_download
+
+                    # laya.load has no revision arg; resolve the pinned snapshot ourselves
+                    local = snapshot_download(LAYA_CHECKPOINT, revision=LAYA_REVISION, allow_patterns=["rl_agent_config.json", "model.safetensors", "tokenizer/*", "encoder/*"])
+                    self._agent = laya.load(local)
+        return self._agent
+
+    def ask(self, state: str, question: str) -> Verdict:
+        agent = self._load()
+        n_tokens = len(agent.tok(state, add_special_tokens=False)["input_ids"])
+        self.calls += 1
+        if n_tokens > LAYA_CONTEXT:
+            self.truncated += 1
+        with self._lock:
+            res = agent.predict(state, {"q": {"type": "noul", "instructions": question}})
+        ans = res["answers"]["q"]
+        p = float(ans["noul"])
+        return Verdict(detected=p >= 0.5, confidence=float(ans.get("confidence", max(p, 1 - p))), reason=f"laya p={p:.2f}")
+
+
+class LiteLLMJudge:
+    """Any API model via litellm (``pip install faultgate[api]``). Temperature 0, JSON answer."""
+
+    SYSTEM = (
+        "You are an expert AI reliability judge evaluating an agent trace.\n{question}\n"
+        'Respond ONLY with valid JSON: {{"detected": bool, "confidence": float, "reason": str}}'
+    )
+
+    budget = None
+
+    def __init__(self, model: str) -> None:
+        self.model = model
+        self.name = f"litellm:{model}"
+
+    def ask(self, state: str, question: str) -> Verdict:
+        import litellm  # optional extra
+
+        resp = litellm.completion(
+            model=self.model,
+            messages=[{"role": "system", "content": self.SYSTEM.format(question=question)}, {"role": "user", "content": state}],
+            temperature=0.0,
+            max_tokens=512,
+        )
+        return parse_json_verdict(resp.choices[0].message.content or "")
+
+
+def parse_json_verdict(text: str) -> Verdict:
+    """Tolerant parse: strips code fences; on failure, detected=False with the raw text as reason."""
+    t = text.strip()
+    if "```" in t:
+        t = t.split("```", 2)[1]
+        t = t[4:] if t.startswith("json") else t
+    try:
+        d = json.loads(t.strip())
+        return Verdict(bool(d.get("detected", False)), float(d.get("confidence", 0.0)), str(d.get("reason", ""))[:200])
+    except (ValueError, AttributeError):
+        return Verdict(False, 0.0, f"unparseable judge output: {text[:100]}")
+
+
+def get_judge(spec: str) -> Judge:
+    """``laya`` or ``litellm:<model>``."""
+    if spec == "laya":
+        return LayaJudge()
+    if spec.startswith("litellm:") and len(spec) > 8:
+        return LiteLLMJudge(spec[8:])
+    raise ValueError(f"unknown judge {spec!r}; use 'laya' or 'litellm:<model>'")
+
+
+JUDGE_SPECS = {
+    "laya": "local decision model, free, offline after the first ~800 MB download (pip install faultgate[laya])",
+    "litellm:<model>": "any API model via litellm, e.g. litellm:openai/gpt-5.6-luna (needs faultgate[api] and the provider's API key)",
+}

faultgate/runner.py

@@ -0,0 +1,88 @@
+"""Run checks over traces with a judge. Pure: no I/O."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from faultgate.checks import Check
+from faultgate.judge import Judge, Verdict
+from faultgate.trace import Trace
+
+
+@dataclass(frozen=True)
+class Result:
+    trace_id: str
+    scenario: str
+    check: str
+    verdict: Verdict
+
+
+@dataclass
+class Report:
+    judge: str
+    checks: list[str]
+    results: list[Result] = field(default_factory=list)
+
+    @property
+    def trace_ids(self) -> list[str]:
+        seen: dict[str, None] = {}
+        for r in self.results:
+            seen.setdefault(r.trace_id)
+        return list(seen)
+
+    @property
+    def scenario_of(self) -> dict[str, str]:
+        return {r.trace_id: r.scenario for r in self.results}
+
+    @property
+    def clean_by_scenario(self) -> dict[str, bool]:
+        """scenario key -> True if no check fired on that trace."""
+        return {self.scenario_of[t]: t not in self.failed_traces for t in self.trace_ids}
+
+    @property
+    def failed_traces(self) -> list[str]:
+        return [t for t in self.trace_ids if any(r.detected for r in self.for_trace(t))]
+
+    def for_trace(self, trace_id: str) -> list[Verdict]:
+        return [r.verdict for r in self.results if r.trace_id == trace_id]
+
+    @property
+    def fired(self) -> dict[str, int]:
+        counts = {c: 0 for c in self.checks}
+        for r in self.results:
+            if r.verdict.detected:
+                counts[r.check] += 1
+        return counts
+
+    @property
+    def passed(self) -> bool:
+        return not self.failed_traces
+
+    def to_dict(self) -> dict:
+        return {
+            "judge": self.judge,
+            "checks": self.checks,
+            "traces": len(self.trace_ids),
+            "failed_traces": self.failed_traces,
+            "fired": self.fired,
+            "results": [
+                {"trace_id": r.trace_id, "scenario": r.scenario, "check": r.check, "detected": r.verdict.detected,
+                 "confidence": round(r.verdict.confidence, 4), "reason": r.verdict.reason}
+                for r in self.results
+            ],
+        }
+
+
+def run(traces: list[Trace], checks: list[Check], judge: Judge | None = None) -> Report:
+    """Rules always run. Judge checks need a judge, golden checks a golden file; unbound ones are dropped from the report."""
+    active = [c for c in checks if c.rule or (c.question and judge is not None)]
+    report = Report(judge=judge.name if judge else "none", checks=[c.name for c in active])
+    for t in traces:
+        state = None
+        for c in active:
+            if c.rule:
+                v = c.rule(t)
+            else:
+                state = state if state is not None else t.render(getattr(judge, "budget", None))
+                v = judge.ask(state, c.question)
+            report.results.append(Result(t.trace_id, t.key, c.name, v))
+    return report

faultgate/trace.py

@@ -0,0 +1,193 @@
+"""OTLP JSON (GenAI semantic conventions) → Trace.
+
+One trace = one agent run: a root ``invoke_agent`` span plus child ``chat`` /
+``execute_tool`` spans. Content is read from the semconv content attributes
+(``gen_ai.input.messages``, ``gen_ai.output.messages``,
+``gen_ai.tool.call.arguments``, ``gen_ai.tool.call.result``) with the legacy
+``gen_ai.prompt`` / ``gen_ai.completion`` as fallback. A trace with no content
+is still loaded — the judge just sees structure only.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+
+class Span(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    span_id: str
+    parent_id: str | None
+    name: str
+    start_ns: int
+    attrs: dict[str, Any]
+
+
+class Step(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    index: int
+    tool: str
+    arguments: str
+    result: str
+
+
+class Trace(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    trace_id: str
+    agent: str
+    spans: list[Span]
+
+    @property
+    def root(self) -> Span | None:
+        for s in self.spans:
+            if s.attrs.get("gen_ai.operation.name") == "invoke_agent" or s.parent_id is None:
+                return s
+        return None
+
+    @property
+    def prompt(self) -> str:
+        r = self.root
+        if not r:
+            return ""
+        return _text(r.attrs.get("gen_ai.input.messages"), role="user") or str(r.attrs.get("gen_ai.prompt", ""))
+
+    @property
+    def answer(self) -> str:
+        r = self.root
+        if not r:
+            return ""
+        return _text(r.attrs.get("gen_ai.output.messages"), role="assistant") or str(r.attrs.get("gen_ai.completion", ""))
+
+    @property
+    def key(self) -> str:
+        """Stable scenario key across runs: faultgate.scenario, else gen_ai.agent.id, else a prompt hash."""
+        r = self.root
+        if r:
+            for k in ("faultgate.scenario", "gen_ai.agent.id"):
+                if r.attrs.get(k):
+                    return str(r.attrs[k])
+        return "p-" + hashlib.sha256(self.prompt.encode("utf-8")).hexdigest()[:12]
+
+    @property
+    def termination(self) -> str:
+        r = self.root
+        return str(r.attrs.get("faultgate.termination", "")) if r else ""
+
+    @property
+    def steps(self) -> list[Step]:
+        out = []
+        for s in self.spans:
+            if s.attrs.get("gen_ai.operation.name") != "execute_tool":
+                continue
+            out.append(Step(
+                index=len(out) + 1,
+                tool=str(s.attrs.get("gen_ai.tool.name", s.name)),
+                arguments=str(s.attrs.get("gen_ai.tool.call.arguments", "")),
+                result=str(s.attrs.get("gen_ai.tool.call.result", "")),
+            ))
+        return out
+
+    @property
+    def has_content(self) -> bool:
+        return bool(self.prompt or self.answer or any(st.arguments or st.result for st in self.steps))
+
+    def render(self, budget: int | None = None) -> str:
+        """The state text a judge reads.
+
+        Diagnostic facts first (answer, termination, step count), then the steps.
+        With a character ``budget`` (small local judges have a 512-token window),
+        steps are compacted and the middle is elided — the first two and the last
+        ones survive, because loops are visible at the tail and setup at the head.
+        """
+        steps = self.steps
+        head = [
+            f"Agent: {self.agent}",
+            f"Prompt: {_trim(self.prompt, 600 if budget else None)}",
+            f"Steps used: {len(steps)}",
+            f"Termination: {self.termination or 'unknown'}",
+            f"Final answer: {_trim(self.answer, 400 if budget else None) or '(none)'}",
+        ]
+        lines = [f"Step {st.index}: {st.tool} {_trim(st.arguments, 160 if budget else None)} -> {_trim(st.result, 160 if budget else None)}" for st in steps]
+        if budget is None:
+            return "\n".join(head + lines)
+        room = budget - sum(len(h) + 1 for h in head)
+        keep_head = lines[:2]
+        room -= sum(len(x) + 1 for x in keep_head)
+        tail: list[str] = []
+        for line in reversed(lines[2:]):
+            if room - len(line) - 1 < 24:  # leave space for the elision marker
+                break
+            tail.insert(0, line)
+            room -= len(line) + 1
+        omitted = len(lines) - len(keep_head) - len(tail)
+        middle = [f"... {omitted} steps omitted ..."] if omitted > 0 else []
+        return "\n".join(head + keep_head + middle + tail)
+
+
+def _trim(s: str, n: int | None) -> str:
+    return s if n is None or len(s) <= n else s[: n - 1] + "…"
+
+
+def _text(messages: Any, role: str) -> str:
+    """Pull the text parts for ``role`` out of a gen_ai.*.messages value."""
+    if isinstance(messages, str):
+        try:
+            messages = json.loads(messages)
+        except ValueError:
+            return messages
+    if not isinstance(messages, list):
+        return ""
+    parts = []
+    for m in messages:
+        if not isinstance(m, dict) or m.get("role") != role:
+            continue
+        for p in m.get("parts", []):
+            if isinstance(p, dict) and p.get("type") == "text":
+                parts.append(str(p.get("content", "")))
+        if "content" in m and not m.get("parts"):
+            parts.append(str(m["content"]))
+    return "\n".join(parts)
+
+
+def _value(v: dict[str, Any]) -> Any:
+    if "stringValue" in v:
+        return v["stringValue"]
+    if "intValue" in v:
+        return int(v["intValue"])
+    if "doubleValue" in v:
+        return float(v["doubleValue"])
+    if "boolValue" in v:
+        return bool(v["boolValue"])
+    if "arrayValue" in v:
+        return [_value(x) for x in v["arrayValue"].get("values", [])]
+    return v
+
+
+def load_otlp(path: str | Path) -> list[Trace]:
+    """Parse an OTLP/JSON export into traces, ordered by first span start."""
+    doc = json.loads(Path(path).read_text(encoding="utf-8"))
+    by_trace: dict[str, list[Span]] = {}
+    agents: dict[str, str] = {}
+    for rs in doc.get("resourceSpans", []):
+        for ss in rs.get("scopeSpans", []):
+            for s in ss.get("spans", []):
+                attrs = {a["key"]: _value(a["value"]) for a in s.get("attributes", [])}
+                span = Span(
+                    span_id=s["spanId"],
+                    parent_id=s.get("parentSpanId"),
+                    name=s.get("name", ""),
+                    start_ns=int(s.get("startTimeUnixNano", 0)),
+                    attrs=attrs,
+                )
+                by_trace.setdefault(s["traceId"], []).append(span)
+                if "gen_ai.agent.name" in attrs:
+                    agents[s["traceId"]] = str(attrs["gen_ai.agent.name"])
+    traces = [
+        Trace(trace_id=tid, agent=agents.get(tid, "agent"), spans=sorted(spans, key=lambda x: x.start_ns))
+        for tid, spans in by_trace.items()
+    ]
+    traces.sort(key=lambda t: t.spans[0].start_ns if t.spans else 0)
+    return traces

faultgate-0.1.0.dist-info/METADATA

@@ -0,0 +1,225 @@
+Metadata-Version: 2.4
+Name: faultgate
+Version: 0.1.0
+Summary: Local-first release gate for AI agents: check recorded traces for known failure modes through a pluggable judge.
+Author: Samir Sawarkar
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/samirsawarkar/faultgate
+Project-URL: Evidence, https://github.com/samirsawarkar/faultline-ai-reliability
+Keywords: ai,agents,evaluation,reliability,opentelemetry,llm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic>=2
+Provides-Extra: laya
+Requires-Dist: laya>=0.3.5; extra == "laya"
+Provides-Extra: api
+Requires-Dist: litellm>=1.70; extra == "api"
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Dynamic: license-file
+
+# faultgate
+
+**A local-first release gate for AI agents.** Hand it the traces your agent recorded, it checks every run for known failure modes, measures your run-to-run noise, and exits non-zero only when a candidate is a real regression. Free, offline, no model required for the default checks.
+
+```
+$ faultgate check examples/traces.json
+trace         search_loop       abstention
+------------------------------------------------
+fc2cdab304bd  FAIL 1.00         ok   1.00
+6d1f9d67a2d4  ok   1.00         FAIL 1.00
+8bf3879339eb  ok   1.00         ok   1.00
+3a9e507a0f6d  ok   1.00         ok   1.00
+2ab3b6c1e891  ok   1.00         ok   1.00
+
+2/5 traces failed · search_loop=1, abstention=1 · judge none · 0.0s
+$ echo $?
+1
+```
+
+Real output, unedited, on five real agent runs (`openai/gpt-5.6-luna` on a multi-hop retrieval task, recorded by [FAULTLINE](https://github.com/samirsawarkar/faultline-ai-reliability)). Trace 1 looped for 12 steps and never answered; trace 2 said it couldn't find the answer; 3–5 answered.
+
+## Install
+
+```bash
+pip install faultgate
+```
+
+Requires Python 3.11 or newer. One dependency (pydantic). The built-in checks are code — nothing to download. Judges are extras:
+
+```bash
+pip install 'faultgate[laya]'   # local judge: downloads Laya once (~800 MB, Apache-2.0, pinned revision), offline after
+pip install 'faultgate[api]'    # any API model through litellm
+```
+
+Read [How much to trust a verdict](#how-much-to-trust-a-verdict) before you enable a judge.
+
+## Usage
+
+```bash
+faultgate check traces.json                          # built-in rule checks
+faultgate check traces.json --json report.json       # full per-trace verdicts
+faultgate check traces.json --golden golden.json     # expected answers: correctness + truncation
+faultgate check traces.json --policy policy.json     # what the agent was allowed to do
+faultgate check traces.json --judge laya             # also run judge checks, locally
+faultgate check traces.json --judge litellm:openai/gpt-5.6-luna   # any API model
+faultgate checks                                     # list checks and their kind
+faultgate judges                                     # list judge specs
+
+faultgate baseline run1.json run2.json --out band.json   # measure your noise (>= 2 runs)
+faultgate check candidate.json --band band.json          # regression, or just noise?
+```
+
+Exit codes: `0` clean · `1` at least one check fired (or, with `--band`, gate FAIL) · `2` bad input.
+
+### Input format
+
+An [OTLP/JSON](https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding) export using the [OpenTelemetry GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/). One trace is one agent run:
+
+| Span | Attributes read |
+|---|---|
+| root `invoke_agent` | `gen_ai.agent.name`, `gen_ai.input.messages` (the user prompt), `gen_ai.output.messages` (the final answer), optional `faultgate.termination` (why the run stopped, e.g. `step cap`) |
+| child `execute_tool` | `gen_ai.tool.name`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result` |
+
+Legacy `gen_ai.prompt` / `gen_ai.completion` are accepted as fallback. If your exporter records structure but not content, `faultgate` warns you. [`examples/traces.json`](https://github.com/samirsawarkar/faultgate/blob/main/examples/traces.json) is a complete reference file.
+
+## Checks
+
+| Check | Kind | Fires when |
+|---|---|---|
+| `search_loop` | rule | The run ended with no answer at its step budget and most tool calls returned nothing. |
+| `abstention` | rule | The final answer explicitly says the information could not be found. |
+| `policy` | rule | A tool call broke the policy: unlisted tool, denied argument pattern, oversized argument, or call budget. |
+| `wrong_answer` | golden | Answered, and the expected answer is not in the final answer. |
+| `truncation` | golden | The answer has the head of the expected token but not the whole token (`Onyx` for `Onyx-4413`). |
+| `wrong_direction` | judge | On a multi-hop task, the agent walked the dependency chain backwards. |
+
+A **rule** is plain code over the trace structure: instant, deterministic, no model. A **golden** check is a rule that also needs the expected answer (`--golden golden.json`, keyed by scenario or by prompt text). A **judge** check is a yes/no question a model answers from the rendered trace; it only runs when you pass `--judge`. All three kinds are calibrated the same way, below.
+
+### Policy: what was the agent allowed to do?
+
+```json
+{"allow": ["search", "lookup", "calc"], "max_calls": 10, "max_arg_chars": 200}
+```
+
+`faultgate check traces.json --policy policy.json` then flags every step that called a tool outside `allow`, passed an argument matching a denied pattern (path traversal, `file://`, `/etc/passwd`, `~/.ssh` out of the box), exceeded `max_arg_chars`, or pushed the run past `max_calls` — with the step number in the reason. It is FAULTLINE P16's runtime policy applied after the fact: the four structural injection categories from P9 (`tool_exfil_path`, `unlisted_tool`, `query_dump`, `budget_loop`) are each caught by a test in `tests/test_policy.py`. Without a policy file only the denied-pattern rule is active, and it fires on 0 of the 290 real traces shipped here.
+
+Semantic injections — the agent *obeying* text it read in a document — are a judge question, not a rule, and are not in this release.
+
+## Regression or noise? Tolerance bands
+
+Agents are not deterministic. FAULTLINE P11 ran the same model on the same 30 inputs five times and got pass rates from 0.60 to 0.87. A fixed threshold turns that into a coin-flip gate. `faultgate` measures your spread instead:
+
+```
+$ faultgate baseline examples/gate/baseline_glm_run1.json examples/gate/baseline_glm_run2.json --out band.json
+examples/gate/baseline_glm_run1.json: 30 traces · clean rate 0.93 · 0.0s
+examples/gate/baseline_glm_run2.json: 30 traces · clean rate 0.93 · 0.0s
+band [0.93, 0.93] over 30 scenarios × 2 runs · 26 clean in every run · written to band.json
+
+$ faultgate check examples/gate/candidate_gpt.json --band band.json
+18/30 traces failed · search_loop=18, abstention=0 · judge none · 0.0s
+gate: FAIL · clean rate 0.40 vs band [0.93, 0.93] from 2 baseline runs · fail below 0.83
+regressions (clean in every baseline, not now): r-0208, r-0214, r-0216, r-0224, r-0226, r-0232, r-0258, ...
+
+$ faultgate check examples/gate/candidate_qwen.json --band band.json
+gate: FAIL · clean rate 0.70 vs band [0.93, 0.93] from 2 baseline runs · fail below 0.83
+
+$ faultgate check examples/gate/baseline_glm_run2.json --band band.json
+gate: PASS · clean rate 0.93 vs band [0.93, 0.93] from 2 baseline runs · fail below 0.83
+```
+
+Real runs: two `z-ai/glm-5.3-flash` baselines and `openai/gpt-5.6-luna` / `qwen/qwen3.7-flash` candidates on the same 30 multi-hop scenarios. The verdicts match FAULTLINE P11's live drill (R4 FAIL / R1 FAIL / R2 PASS), which cost $1.84 and an API judge; this took 0.0 s and no model.
+
+| Verdict | Rule (n = scenarios in the band) | Exit |
+|---|---|---|
+| **PASS** | clean rate ≥ min observed − 1/n | 0 |
+| **WARN** | in between — a human should look | 0 |
+| **FAIL** | clean rate < min observed − 3/n | 1 |
+
+It also lists **regressions**: scenarios clean in *every* baseline run and not clean now — the ones to read first. Scenarios are matched across runs by `faultgate.scenario`, else `gen_ai.agent.id`, else a hash of the prompt.
+
+With expected answers the gate grades correctness too, and the numbers land on P11's exactly:
+
+```
+$ faultgate baseline examples/gate/baseline_glm_run*.json --golden examples/gate/golden.json --out band.json
+examples/gate/baseline_glm_run1.json: 30 traces · clean rate 0.80 · 0.0s      # P11: 24/30 grounded
+examples/gate/baseline_glm_run2.json: 30 traces · clean rate 0.87 · 0.0s      # P11: 26/30 grounded
+band [0.80, 0.87] over 30 scenarios × 2 runs · 21 clean in every run
+
+$ faultgate check examples/gate/candidate_gpt.json --golden examples/gate/golden.json --band band.json
+21/30 traces failed · search_loop=18, abstention=0, policy=0, wrong_answer=3, truncation=3
+gate: FAIL · clean rate 0.30 vs band [0.80, 0.87] from 2 baseline runs · fail below 0.70
+
+$ faultgate check examples/gate/candidate_qwen.json --golden examples/gate/golden.json --band band.json
+18/30 traces failed · search_loop=9, abstention=0, policy=0, wrong_answer=9, truncation=8
+gate: FAIL · clean rate 0.40 vs band [0.80, 0.87] from 2 baseline runs · fail below 0.70
+```
+
+(Qwen returns the head of the answer token without its suffix in 8 of 30 runs. Nobody had noticed.)
+
+## In CI
+
+```yaml
+- uses: samirsawarkar/faultgate@main
+  with:
+    traces: traces/candidate.json      # exported by your agent's OTel pipeline
+    band: traces/band.json             # from `faultgate baseline`, committed
+    golden: traces/golden.json         # optional
+    policy: traces/policy.json         # optional
+```
+
+Exit 1 on gate FAIL, so the job fails; WARN and PASS pass. The report lands in `faultgate-report.json` for upload as an artifact. The action installs nothing heavier than pydantic; our own CI runs it against this repo's examples and asserts that the GPT candidate fails and the GLM baseline passes.
+
+## How much to trust a verdict
+
+This is the part every evaluation tool skips. Every check and judge here is scored against **human failure-mode labels** on 200 real agent runs (FAULTLINE P4), reported on all 200 and on P5's frozen test split (n = 60). Regenerate with `scripts/calibrate.py`.
+
+| Check | Kind | Positives | TPR | TNR | κ (n=200) | κ (P5 test, n=60) |
+|---|---|---|---|---|---|---|
+| `search_loop` | rule | 36 | 1.00 | 0.99 | **0.98** | **1.00** |
+| `abstention` | rule | 1 | 1.00 | 1.00 | 1.00 | — (0 positives) |
+| `policy` | rule | synthetic | 4/4 P9 categories | 0 FP on 290 real traces | — | — |
+| `wrong_answer` | golden | 4 | 1.00 | 1.00 | 1.00 | 1.00 |
+| `truncation` | golden | 1 | 1.00 | 1.00 | 1.00 | — (0 positives) |
+| `wrong_direction` | Laya judge | 1 | 1.00 | 0.61 | 0.02 | 0.00 |
+
+For comparison, the same `search_loop` question put to judges instead of a rule: Laya κ 0.09, GLM-5.3 (API) κ −0.03. **Where a failure is structural, code beats every model we measured**, which is why the default checks are rules.
+
+Read the table honestly:
+- `search_loop` is solid on this task family. The human coder saw the same structural signals, so κ measures agreement with that coding, not ground truth from nowhere.
+- `abstention` and `truncation` have one positive each to learn from. `abstention` is a phrase list; a paraphrase it doesn't know is a miss.
+- `wrong_answer` agrees with the human verdict on all 21 answered runs. It is substring containment, case-insensitive: right for short factual answers, wrong for free-text ones — say what "expected" means for your task before you trust it.
+- `policy` has no human-labelled positives: FAULTLINE's agent rejects hostile calls before they become steps, so its traces never contain an executed attack. Recall is by construction on the P9 attack shapes; the number that matters is zero false positives on real runs.
+- `wrong_direction` is the only check that needs reading, and the only local judge we have is noise on it (fires on 39% of clean runs). It ships because the contract is the product: the day a local model scores κ ≥ 0.7 on it, that's a one-line adapter, and this harness is what proves it.
+- Laya's context is 512 tokens. `faultgate` compacts the trace to fit (answer and termination first, middle steps elided) and counts every call that still overflows; the count is printed.
+
+Full confusion matrices and Wilson CIs for the original P5 judges: [FAULTLINE `projects/p05_judge/DECISIONS.md`](https://github.com/samirsawarkar/faultline-ai-reliability/blob/main/projects/p05_judge/DECISIONS.md).
+
+## Swap the judge
+
+```python
+class Judge(Protocol):
+    name: str
+    budget: int | None          # max characters of trace it can read
+    def ask(self, state: str, question: str) -> Verdict: ...   # Verdict(detected, confidence, reason)
+```
+
+`laya` and `litellm:<model>` ship in the box. Add a judge, run `scripts/calibrate.py --judge yours`, and publish the row.
+
+## What it doesn't do yet
+
+- **Premature stop** (answered with an intermediate entity of the chain). Needs the traversal chain, not just the final answer.
+- **Semantic injection checks** (the agent obeyed text it read). P9 has the taxonomy; needs a judge that scores.
+- **Running your agent.** `faultgate` reads traces; it never executes anything.
+
+## Built on FAULTLINE
+
+[FAULTLINE](https://github.com/samirsawarkar/faultline-ai-reliability) is the research workbench behind this tool: 30 days of simulator, 16 measured projects, two papers. `faultgate` is the part you install. Every number in this README links back to the run that produced it.
+
+MIT © 2026 Samir Sawarkar

faultgate-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+faultgate/__init__.py,sha256=Xfa_A-gr5x7FdZG7-VLhs7-7kh8XxcXBeQ3cmHib_IU,82
+faultgate/band.py,sha256=TpCb8cZWd17XDZvE4jN-yoKvrFLapQlU8vszGnNhLo0,3142
+faultgate/checks.py,sha256=9A3d8JTxcnrTuIlWyu5hezjGK0jC4seR466tTnr9Wa0,10096
+faultgate/cli.py,sha256=M-x3_AAmJADvU_gYPpYEoC-UYD_roq7wMqEfPAw51dg,7686
+faultgate/judge.py,sha256=-0GPAx9j03TQNWqKyRs1MvShfdAAfuHlvPjeII168jw,4553
+faultgate/runner.py,sha256=wQi44KsvCGs-Lz0YZDofi7UHcXQX90zgwxJYfYaPd_M,2897
+faultgate/trace.py,sha256=WPi3mmkvL-xZAByIzhTmPAta0zVu8a8qGXoAOkmCvgc,6978
+faultgate-0.1.0.dist-info/licenses/LICENSE,sha256=gEtBbguhy8bPdcRa7sAv73zL4qwZxPQlNc4VxYdMPdc,1071
+faultgate-0.1.0.dist-info/METADATA,sha256=Av8vsgh5bem08tQ3JJW8Pgab9B3a-0LPqBr8taOrIN0,13852
+faultgate-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+faultgate-0.1.0.dist-info/entry_points.txt,sha256=s4HBdMEe-ezrY1A6LIs-Efl_SqqTC5sRWn_1CYMdAUI,49
+faultgate-0.1.0.dist-info/top_level.txt,sha256=wAe6TjlTbmdzR8gPivSS3kbl6gUSIUiJa7QkmW7vYwk,10
+faultgate-0.1.0.dist-info/RECORD,,

faultgate-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

faultgate-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+faultgate = faultgate.cli:main

faultgate-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Samir Sawarkar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

faultgate-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+faultgate