faithgate 0.1.0__tar.gz

faithgate-0.1.0/.github/workflows/eval-gate.yml

@@ -0,0 +1,68 @@
+name: eval-gate
+
+# Two jobs, both offline/deterministic (heuristic judge, no API key):
+#   gate               — baseline vs candidate must stay green (the normal PR gate)
+#   proves-detection   — baseline vs a suite with PLANTED hallucinations must go RED;
+#                        the step INVERTS the exit code, so this job failing means the gate
+#                        LOST its ability to catch regressions. A green badge here is proof.
+# To score with the real Claude judge instead: pip install ".[claude]", swap --judge claude,
+# and set the ANTHROPIC_API_KEY repository secret.
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  gate:
+    runs-on: ubuntu-latest
+    env:
+      PYTHONPATH: .
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Score the baseline version
+        run: python -m faithgate.gate.cli --db ci.db run --suite examples/rag_app/suite_baseline.jsonl --label baseline --judge heuristic
+
+      - name: Score the candidate version
+        run: python -m faithgate.gate.cli --db ci.db run --suite examples/rag_app/suite_candidate.jsonl --label candidate --judge heuristic
+
+      - name: Gate — fail if faithfulness regressed
+        run: |
+          set +e
+          python -m faithgate.gate.cli --db ci.db gate --base baseline --head candidate \
+            --max-regression 0.05 --min-score 0.5 | tee gate.txt
+          status=${PIPESTATUS[0]}
+          { echo '### faithgate — baseline vs candidate'; echo '```'; cat gate.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
+          exit "$status"
+
+  proves-detection:
+    runs-on: ubuntu-latest
+    env:
+      PYTHONPATH: .
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Score the baseline version
+        run: python -m faithgate.gate.cli --db proof.db run --suite examples/rag_app/suite_baseline.jsonl --label baseline --judge heuristic
+
+      - name: Score a version with PLANTED hallucinations
+        run: python -m faithgate.gate.cli --db proof.db run --suite examples/rag_app/suite_regressed.jsonl --label regressed --judge heuristic
+
+      - name: Assert the gate catches the planted regression (inverted exit)
+        run: |
+          set +e
+          python -m faithgate.gate.cli --db proof.db gate --base baseline --head regressed \
+            --max-regression 0.05 --min-score 0.5 | tee proof.txt
+          status=${PIPESTATUS[0]}
+          { echo '### faithgate — planted-regression detection proof'; echo '```'; cat proof.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
+          if [ "$status" -eq 0 ]; then
+            echo '::error::the gate FAILED to catch a known planted regression'
+            exit 1
+          fi
+          echo "gate correctly went red (exit $status) — detection proven"

faithgate-0.1.0/.github/workflows/licenses.yml

@@ -0,0 +1,34 @@
+name: licenses
+
+# Guards two promises at once:
+#   1. MIT cleanliness — fails if any dependency ships under a non-permissive license
+#      (e.g. Elastic License 2.0 like Phoenix Evals, or SSPL).
+#   2. The RAGAS contract — installs the [claude] extra and runs the contract tests, so an
+#      incompatible ragas release breaks THIS job loudly instead of silently green runs.
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install package with the claude extra + license scanner
+        run: |
+          python -m pip install --upgrade pip
+          pip install ".[claude]"
+          pip install pip-licenses
+      - name: Entry point smoke
+        run: faithgate --help
+      - name: RAGAS contract tests (run only where ragas is installed)
+        run: python -m unittest tests.test_ragas_contract -v
+      - name: Fail on non-permissive licenses
+        run: |
+          pip-licenses --format=csv --with-system \
+            | grep -Ei 'elastic|sspl|server side public|business source|bsl' \
+            && (echo "Non-permissive license found" && exit 1) || echo "All dependency licenses are permissive."

faithgate-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,27 @@
+name: release
+
+# Publishes to PyPI on a version tag via TRUSTED PUBLISHING (OIDC) — no long-lived API token
+# stored anywhere. One-time setup on pypi.org: add this repo + workflow as a trusted publisher
+# for the 'faithgate' project (environment: pypi).
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  pypi:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Build sdist + wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+      - name: Publish to PyPI (trusted publishing)
+        uses: pypa/gh-action-pypi-publish@release/v1

faithgate-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,19 @@
+name: tests
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Run unit tests (stdlib only — no install needed)
+        run: python -m unittest discover -s tests -v

faithgate-0.1.0/.gitignore

@@ -0,0 +1,30 @@
+# Secrets — the user's own API key lives here, NEVER commit it.
+.env
+.env.*
+
+# Local-only working docs — internal notes, never part of the public repo.
+PLAN.md
+HANDOFF.md
+AUDIT_*.md
+
+# Local trace store
+*.db
+*.db-wal
+*.db-shm
+.faithgate/
+
+# Python
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+
+# OS / editors
+.DS_Store
+.idea/
+.vscode/

faithgate-0.1.0/DESIGN.md

@@ -0,0 +1,94 @@
+# FaithGate — design rationale
+
+Why the tool is shaped the way it is: the locked decisions, the traps they avoid, and what was
+deliberately cut. Companion to [README.md](README.md) (what it does) and [ROADMAP.md](ROADMAP.md)
+(what comes next).
+
+## The one principle
+
+**Never present a number as more certain than it is.** Every design call below is this principle
+applied somewhere: to the judge, to the gate verdict, to the docs, to the dependency list.
+
+## Positioning
+
+The "simple + local + eval-first" lane is well served (promptfoo, Arize Phoenix, DeepEval — see the
+README's comparison). FaithGate deliberately does NOT compete there. It is a *complement* with one
+job: **fail-closed blocking of faithfulness regressions**, version-to-version, with measured honesty
+about the judge that produced every number.
+
+## Locked decisions
+
+| # | Decision | Why |
+|---|---|---|
+| D1 | No per-claim breakdown in v1 | RAGAS publicly returns only a scalar; scraping its internals for per-claim verdicts would be a fragile reimplementation that breaks on minor releases. The panel shows the score + a one-line reason and stays honest. (RAGAS's newer collections API exposes `.reason` — the v3 path.) |
+| D2 | "Local" mode = `local-verification`, precisely labeled | RAGAS's HHEM variant still requires an LLM for claim *extraction* — HHEM only replaces the entailment check. A "fully offline trusted judge" cannot honestly be delivered, so it isn't advertised. |
+| D3 | Frontier judge (Claude) as the trusted default | Laptop-size local models score ~61% on faithfulness benchmarks — near coin-flip on hard cases. The offline heuristic exists so the pipeline runs keyless, but it is deliberately distrusted and its blindness is asserted by a unit test. |
+| D4 | Capture = span ingest + `capture()` helper + suite files | No custom proxy (single point of failure in the user's request path) and no per-provider SDK (unmaintainable). The ingest adapter is one file, isolating the still-experimental GenAI semconv; a startup self-test fails loudly if extraction breaks. |
+| D5 | SQLite + WAL, single file | Single-writer is a non-issue for a single-user tool. Analytical layers (DuckDB etc.) were cut when plain SQL proved sufficient — see "Changed during build". |
+| D6 | Embed RAGAS; never re-implement metric math | The metric is battle-tested and citable. This project's value is the harness around it: the gate, the diffing, the calibration, the honesty surfaces. A CI contract-test job imports the exact RAGAS surface used, so an incompatible release breaks loudly. |
+| D7 | Abstention is a distinct state | NaN / zero-extracted-statements / judge errors are never a `0.0` and never a regression — but if *everything* abstained, the gate fails closed instead of blessing an unjudged run. |
+| D8 | Manifests + judge-change guard | Every run pins judge id/model/kind, RAGAS version, runner version, suite hash. The gate refuses to compare runs whose judges differ (exit 3): a judge swap must never masquerade as a model regression. Corrupted manifests fail closed — corruption is not absence. |
+| D9 | Content-keyed diffing | Runs are matched by `SHA256(normalized question + contexts)` (or an explicit per-case `id`), never by row id — so the gate distinguishes "same case, score moved" from "different case". |
+| D10 | Zero-dependency base install | Capture, the offline judge, the gate, and the stdlib web panel run with no third-party packages. RAGAS + the Claude client live in the `[claude]` extra; torch/HHEM in `[local]`. |
+| D11 | Version identity at capture time | Every trace can carry a `prompt_version_id` (see `faithgate.keys.version_key`) — the join key for the replay/drift layers on the roadmap. System-under-test identity is kept separate from judge identity: merging them would destroy the "did my app change or did my judge change?" distinction. |
+
+## Gate semantics (fail-closed)
+
+The verdict is a function of *(matched, regressed, abstained, new, missing, duplicates)* — not just
+"any regressions?":
+
+- **Zero matched cases → FAIL** ("nothing compared"). A renamed suite, an unscored run, or 100%
+  abstention can never turn CI green.
+- **The score floor guards every scored case** — new cases and abstained-baseline cases included.
+- **Duplicates:** the head keeps the *lowest* duplicate score (strict on the new version); the
+  baseline keeps the *highest* (a stray low baseline entry must not quietly lower the bar).
+- **Policy knobs** close the quiet channels: `--fail-on-missing` (deleted cases), `--max-abstained`
+  (targeted abstention). Exit codes: 0 ok · 1 fail · 2 usage/input error · 3 judge changed.
+- Known, documented limitation: without an explicit `id`, rewording a question mints a new case —
+  inherent to content-keyed matching; the `id` field is the escape hatch.
+
+## Data model (score-centric, forward-wired)
+
+Seven tables: `run`, `trace`, `span`, `judge_run`, `eval_score`, `dataset`, `dataset_item`.
+`eval_score` is the hub linking a captured response to its number, grounding context, and judge
+provenance. Three forward-looking choices cost nothing now and unblock the roadmap without
+migrations:
+
+- `eval_score.metric` is a column (second metric = INSERT, not ALTER).
+- A captured response and a curated test case share row shape; `dataset_item.origin_trace_id` ↔
+  `trace.source_dataset_item_id` pre-wire "promote a bad production answer into a regression test"
+  as two INSERTs (roadmap v2).
+- Contexts live on the trace (not only inside raw span attributes), so replay and promotion never
+  re-parse experimental span formats. Raw spans are retained verbatim as churn insurance.
+
+## The calibration harness
+
+The judge is an AI too, so it takes an exam: a 40-example hand-labeled golden set, stratified across
+failure types (faithful paraphrases, partial support, date/entity swaps, negations, unsupported
+additions). `faithgate calibrate` reports agreement-with-humans per judge, with n shown, and its
+error surface is loud: per-sample timeouts, live progress, first-error printed, exit 2 when nothing
+was actually judged. Measured results live in the README and are re-measurable by anyone with a key.
+
+## Changed during build (deliberate descopes)
+
+Recorded because pretending the original plan was built would violate the one principle:
+
+- **Panel:** stdlib `http.server` instead of a web framework — this is what makes the base install
+  dependency-free.
+- **DuckDB analytics layer:** cut; plain SQL over SQLite is sufficient at this scale.
+- **Async scoring worker:** deferred; `run` scores synchronously after capture, ingested spans are
+  scored via `faithgate score`.
+- **OTLP wire format:** deferred; `/v1/spans` accepts OpenInference-shaped JSON and is documented as
+  such, not as OTLP compatibility.
+- **Per-call hardening added after real-world testing:** RAGAS injects `temperature=0.01` per call
+  unless bypassed (Sonnet-5-class models reject non-default sampling params) and retries every
+  exception 10× with up to 60s waits — turning any persistent error into a multi-minute silent
+  hang. FaithGate bypasses the injection, caps retries, bounds every judge call with a timeout, and
+  surfaces the first real error instead.
+
+## What proves the claims
+
+Every capability sentence in the README maps to a test or CI job. The flagship: the `eval-gate`
+workflow's **proves-detection** job scores a suite with planted hallucinations and *asserts the
+gate fails* by inverting the exit code — if the gate ever loses its ability to catch a known
+regression, CI itself goes red.

faithgate-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 albertofettucini
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

faithgate-0.1.0/PKG-INFO

@@ -0,0 +1,251 @@
+Metadata-Version: 2.4
+Name: faithgate
+Version: 0.1.0
+Summary: Fails your CI when your LLM app starts making things up — local-first faithfulness regression gate.
+Project-URL: Homepage, https://github.com/albertofettucini/faithgate
+Author: albertofettucini
+License: MIT
+License-File: LICENSE
+Keywords: ci,evaluation,faithfulness,llm,llm-as-judge,local-first,observability,rag,ragas,regression
+Requires-Python: >=3.10
+Provides-Extra: claude
+Requires-Dist: langchain-anthropic>=1.0; extra == 'claude'
+Requires-Dist: langchain-community<0.4,>=0.3; extra == 'claude'
+Requires-Dist: ragas<0.5,>=0.4; extra == 'claude'
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: local
+Requires-Dist: langchain-anthropic>=1.0; extra == 'local'
+Requires-Dist: langchain-community<0.4,>=0.3; extra == 'local'
+Requires-Dist: langchain-ollama>=0.2; extra == 'local'
+Requires-Dist: ragas<0.5,>=0.4; extra == 'local'
+Requires-Dist: sentence-transformers>=3.0; extra == 'local'
+Requires-Dist: torch>=2.2; extra == 'local'
+Requires-Dist: transformers<5,>=4.40; extra == 'local'
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="assets/logo.svg" width="110" alt="FaithGate — a pixel shield, half green half red, split by the gate">
+</p>
+
+# FaithGate
+
+> **You changed a prompt. Did any answer quietly start making things up?**
+>
+> FaithGate scores every answer against its sources, diffs versions, and **fails CI on
+> regression** — with a judge whose trustworthiness is *measured* (85%, n=40), never assumed.
+> Zero infra, fully local, **pytest for prompts.**
+
+![tests](https://github.com/albertofettucini/faithgate/actions/workflows/tests.yml/badge.svg)
+![eval-gate](https://github.com/albertofettucini/faithgate/actions/workflows/eval-gate.yml/badge.svg)
+![license](https://img.shields.io/badge/license-MIT-blue)
+![python](https://img.shields.io/badge/python-3.10%2B-blue)
+
+---
+
+## What it is
+
+It captures your app's question → answer → retrieved-context turns, scores each for **faithfulness**
+(is the answer supported by the context it was given?), and **fails the build when a new version
+scores worse than the last**. Everything runs locally — traces never leave your machine.
+
+**Zero telemetry.** FaithGate itself sends nothing anywhere: no analytics, no phoning home, no
+account. The only network traffic is the API call to the judge **you** configured (plus a one-time
+HuggingFace model download if you opt into the `[local]` HHEM extra).
+
+## See it catch a regression (60 seconds, offline, no API key)
+
+```bash
+git clone https://github.com/albertofettucini/faithgate && cd faithgate
+
+PYTHONPATH=. python3 examples/demo_gate.py
+```
+
+```
+faithgate regression gate: FAIL ❌
+
+  matched 12 · 3 regressed · 0 improved · 9 unchanged · 0 abstained · 0 new · 0 missing
+
+  Regressions:
+    ❌ How many moons does Earth have?  1.00 → 0.29  (below floor)
+    ❌ When was Pluto reclassified as a dwarf planet?  1.00 → 0.12  (below floor)
+    ❌ At what altitude does the ISS orbit?  0.90 → 0.20  (below floor)
+
+CI exit code would be: 1
+```
+
+That's a real end-to-end run: a demo RAG suite where three answers started hallucinating, scored by
+the offline judge, caught by the gate. No scripted numbers.
+
+Drive it yourself:
+
+```bash
+PYTHONPATH=. python3 -m faithgate.gate.cli --db demo.db run \
+  --suite examples/rag_app/suite_baseline.jsonl  --label baseline  --judge heuristic
+PYTHONPATH=. python3 -m faithgate.gate.cli --db demo.db run \
+  --suite examples/rag_app/suite_regressed.jsonl --label regressed --judge heuristic
+PYTHONPATH=. python3 -m faithgate.gate.cli --db demo.db gate --base baseline --head regressed
+PYTHONPATH=. python3 -m faithgate.gate.cli --db demo.db up    # browse it → http://127.0.0.1:7654
+```
+
+The **base install is stdlib-only**: capture, the offline judge, the gate, and the web panel need
+zero dependencies. The real judge lives in an extra (below).
+
+## Why it exists (the honest version)
+
+The "simple + local + eval-first" lane is already well served:
+
+| Tool | Reality |
+|---|---|
+| [promptfoo](https://github.com/promptfoo/promptfoo) | MIT, one-command, local, eval-first. Acquired by OpenAI (2026). |
+| [Arize Phoenix](https://github.com/Arize-ai/phoenix) | `pip install` + SQLite, local, ships judge templates. Core is Elastic-2.0 (source-available). |
+| [DeepEval](https://github.com/confident-ai/deepeval) | Apache-2.0, `pip install` + Ollama, 50+ metrics. |
+
+**Why not just promptfoo?** You probably should use promptfoo — FaithGate is not a replacement
+for your eval stack, it's a **complement** that does exactly one thing: fail-closed blocking of
+*faithfulness* regressions, version-to-version, with honesty guarantees about the judge that
+produced every number. Keep your existing suites and tools; FaithGate runs beside them as the
+narrow tripwire for "did my app quietly start making things up." And the roadmap inverts the one
+thing hand-written suites can't do: promptfoo's golden sets are authored by you — FaithGate's v2
+mines them **from your production traffic** (the schema already makes promotion two INSERTs).
+
+FaithGate does **not** try to out-feature anyone. It targets the narrow seam:
+
+1. **A regression *gate*, not a dashboard** — run the suite on every change, diff the scores, fail
+   the build. Fail-closed: zero matched cases or total abstention can never turn CI green.
+2. **Radical honesty about the judge** — a frontier judge (Claude) by default; a *precisely labeled*
+   local mode; a measured judge-vs-human agreement number; abstention instead of fake zeros; and a
+   judge swap between runs is **flagged and blocked**, never mistaken for a model regression.
+3. **Truly zero-infra** — the base tool is Python stdlib only. No Postgres, no ClickHouse, no
+   Docker, no Node, no web framework. The one metric library (RAGAS) is an opt-in extra.
+
+## Real scoring with Claude
+
+The trusted default judge is Claude, using **your own** API key (never hardcoded, `.env` is
+gitignored):
+
+```bash
+# needs Python 3.10+ — on stock macOS (3.9) the easiest path is uv:
+#   brew install uv && uv venv --python 3.12 .venv && source .venv/bin/activate
+uv pip install -e ".[claude]"            # or: python3 -m pip install --upgrade pip && pip install -e ".[claude]"
+export ANTHROPIC_API_KEY=sk-ant-...      # your own key, read from the environment only
+faithgate run --suite my_suite.jsonl --label v2 --judge claude
+```
+
+A suite is JSONL, one turn per line (`id` is optional — it pins the case's identity so a reworded
+question keeps its baseline instead of being treated as a new case):
+
+```json
+{"id": "capital-q", "question": "...", "answer": "...", "contexts": ["retrieved chunk 1", "..."]}
+```
+
+## Honest judging
+
+FaithGate never presents a score as more certain than it is.
+
+- **Frontier judge by default.** Small local models are weak judges — an 8B model scores ~61% on
+  faithfulness benchmarks. So Claude is the trusted default; everything else is a labeled trade-off.
+- **`local-verification` is labeled precisely.** `--judge claude-local` runs the entailment check
+  on-device (Vectara HHEM, via `faithgate[local]`) but still uses Claude for claim extraction. It
+  is **not** "fully offline", and the tool says so. Its cost is measured, not guessed: on-device
+  verification trades ~8 points of balanced agreement vs the full Claude judge (see the table).
+- **The offline judge is deliberately distrusted.** `--judge heuristic` is a token-overlap proxy so
+  the whole pipeline runs keyless. Its measured weakness — it cannot see contradictions built from
+  the context's own words — is **asserted by a unit test** (`test_heuristic_misses_contradiction`),
+  not hidden in a footnote.
+- **Abstention, not fake zeros.** When the judge can't score, the result is `abstained` — excluded
+  from the gate math and reported separately. If *everything* abstained or errored, the gate and the
+  CLI **fail loudly** instead of laundering a broken judge into a green run.
+- **Judge changes are flagged.** Every run records a manifest (judge id/model/kind, RAGAS version,
+  runner version, suite hash). `gate` refuses to compare runs whose judges differ (exit 3) unless
+  you pass `--allow-judge-change`.
+- **Measured agreement, sample size shown.** `faithgate calibrate` runs the judge over a 40-example
+  hand-labeled golden set (faithful paraphrases, partial support, date/entity swaps, negations,
+  unsupported additions):
+
+  | judge | agreement (balanced) | faithful kept | unfaithful caught |
+  |---|---|---|---|
+  | `claude` (default, claude-sonnet-5) | **85%** *(n=40 — directional)* | 20/20 | 14/20 |
+  | `claude-local` (HHEM verify on-device) | 77% *(n=40 — directional)* | 19/20 | 12/20 |
+  | `heuristic` (offline proxy) | 68% *(n=40 — directional)* | 18/20 | 9/20 |
+
+  Even the trusted judge is not an oracle — 85% agreement is consistent with published
+  frontier-judge benchmarks, and that's exactly why the number is measured and shown instead
+  of assumed. Re-measure against your own key anytime: `faithgate calibrate --judge claude`.
+
+## In CI
+
+`.github/workflows/eval-gate.yml` runs **two** offline, deterministic jobs on every push/PR:
+
+- **gate** — baseline vs candidate must stay green (the normal PR gate).
+- **proves-detection** — baseline vs a suite with *planted hallucinations* must go **red**; the job
+  inverts the exit code. If the gate ever loses its ability to catch a known regression, **this job
+  fails the build.** A green badge here isn't decoration — it's continuously re-proven detection.
+
+Both jobs post the score-diff table to the Actions run summary.
+
+## How it works
+
+```
+your app ──spans──► POST /v1/spans ─┐        (or: faithgate.capture(...) / faithgate run --suite)
+                                     ├─► SQLite (WAL) ──► scoring (RAGAS or offline proxy)
+     ingest adapter (one file,       │        │                    │
+     isolates experimental semconv)  │        ▼                    ▼
+                                     │   web panel      version-to-version diff (by content key)
+                                     └────────┴────────────────────┴──► gate: pass / fail / not-comparable
+```
+
+- **Capture:** ingest OpenInference-shaped JSON spans, call `faithgate.capture()` directly, or feed
+  a suite file. Every trace can carry a `prompt_version_id` (see `faithgate.keys.version_key`) — the
+  join key for the replay/drift layers on the [roadmap](ROADMAP.md).
+- **Score:** [RAGAS](https://github.com/explodinggradients/ragas) computes faithfulness — embedded,
+  not re-implemented. A CI contract-test job imports the exact RAGAS surface we use, so an
+  incompatible release breaks loudly.
+- **Store:** SQLite + WAL, one file, zero server.
+- **Gate:** runs are matched by content (never row id); new, missing, duplicate, and abstained cases
+  are always visible in the report.
+
+## Design notes (the non-obvious parts)
+
+Full rationale — locked decisions, fail-closed semantics, schema pre-wiring, deliberate descopes —
+lives in [DESIGN.md](DESIGN.md). Highlights:
+
+- **We show the score, not a faked per-claim breakdown.** RAGAS only exposes a scalar publicly;
+  scraping its internals would be a fragile reimplementation. v1 shows the score + a one-line
+  reason and stays honest. (The per-claim path returns via RAGAS's newer API — see ROADMAP v3.)
+- **"Local" can't mean "no LLM."** RAGAS's HHEM variant still needs an LLM to *extract* claims — so
+  faithgate ships `local-verification` (HHEM verifies, Claude extracts) and refuses to advertise a
+  keyless trusted mode it can't honestly deliver.
+- **Fail-closed gate semantics.** The gate's verdict is a function of (matched, regressed, abstained,
+  new, missing) — not just "any regressions?". A renamed suite, a broken judge, or 100% abstention
+  fails the gate instead of passing vacuously.
+- **Case identity is content-based by default.** Rewording a question mints a new case (only the
+  score floor guards it, not the delta). If your suite evolves, give cases a stable `id` — that
+  pins identity across rewordings.
+
+## Commands
+
+| Command | What it does |
+|---|---|
+| `faithgate run --suite S --label L [--judge]` | score a suite of answers into a named run |
+| `faithgate gate --base A --head B [--allow-judge-change]` | compare two runs; exit non-zero on regression |
+| `faithgate runs` | list captured runs |
+| `faithgate show --run R` | show a run's scored cases |
+| `faithgate score [--judge] [--run] [--retry-errors]` | score pending traces; optionally re-score errored ones |
+| `faithgate calibrate [--judge]` | judge agreement with the human-labeled golden set |
+| `faithgate up` | start the local web panel |
+
+Judges: `claude` (default, needs key + `[claude]` extra) · `claude-local` (HHEM, `[local]` extra) ·
+`heuristic` (offline, zero deps).
+Exit codes: `0` ok · `1` gate failed (regression / nothing compared) · `2` usage or input error ·
+`3` judge changed between runs.
+
+## Roadmap
+
+See [ROADMAP.md](ROADMAP.md) — v2 auto-eval generation (bad production answers become regression
+tests), v2.5 CI gates over the mined dataset, v3 prescriptive feedback + failure clustering, then
+replay / drift / cost-quality reports. The v1 schema already carries the join keys they need.
+
+## License
+
+MIT — see [LICENSE](LICENSE).