judgekeeper 0.1.0__tar.gz

judgekeeper-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,110 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        # The mlflow extra lets tests/test_import_mlflow.py build and read a real MLflow store.
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,mlflow]"
+      - name: Test
+        run: pytest
+      - name: Lint
+        run: ruff check src tests
+
+  # Builds both distributions, installs the wheels into a fresh venv and runs them from there,
+  # so a packaging mistake (missing demo_data or NOTICE, a broken entry point) fails here
+  # rather than on release day. The same steps run in release.yml before publishing.
+  package:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build both distributions
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build --outdir dist .
+          python -m build --outdir dist packages/pytest-judgekeeper
+          twine check --strict dist/*
+      - name: Install the wheel into a fresh venv
+        run: |
+          python -m venv "$RUNNER_TEMP/venv"
+          "$RUNNER_TEMP/venv/bin/pip" install dist/judgekeeper-*.whl
+      - name: Run the installed judgekeeper outside the checkout
+        working-directory: ${{ runner.temp }}
+        run: |
+          set -x
+          venv/bin/judgekeeper --help
+          venv/bin/judgekeeper demo --out demo
+          test -f demo/report.html -a -f demo/report.json
+          venv/bin/python -c 'import judgekeeper.demo as d; assert (d.DEMO_DATA / "NOTICE").is_file()'
+      - name: Run the test suite against the installed wheels
+        run: |
+          "$RUNNER_TEMP/venv/bin/pip" install --find-links dist dist/pytest_judgekeeper-*.whl \
+            "pytest>=8" "pyyaml>=6"
+          "$RUNNER_TEMP/venv/bin/python" -c 'import judgekeeper, sys; assert "site-packages" in judgekeeper.__file__, judgekeeper.__file__'
+          "$RUNNER_TEMP/venv/bin/python" -m pytest -p no:cacheprovider
+
+  # Runs action.yml from this checkout with the replay runner: no API key needed.
+  action:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Gate a judge that agrees with every human label (expect PASS)
+        id: agree
+        uses: ./
+        with:
+          anchors: tests/fixtures/pairwise/anchors.jsonl
+          runner: replay
+          fixture: tests/fixtures/pairwise/replay-agree.jsonl
+          baseline: tests/fixtures/pairwise/baseline.json
+          out-dir: gate-agree
+          artifact-name: judgekeeper-gate-agree
+
+      - name: Gate a judge with kappa 0.5 (expect FAIL and exit 1)
+        id: disagree
+        continue-on-error: true
+        uses: ./
+        with:
+          anchors: tests/fixtures/pairwise/anchors.jsonl
+          runner: replay
+          fixture: tests/fixtures/pairwise/runs/run-01.jsonl
+          baseline: tests/fixtures/pairwise/baseline.json
+          out-dir: gate-disagree
+          artifact-name: judgekeeper-gate-disagree
+
+      - name: Check the action's status and exit code
+        env:
+          AGREE_STATUS: ${{ steps.agree.outputs.status }}
+          AGREE_CODE: ${{ steps.agree.outputs.exit-code }}
+          DISAGREE_OUTCOME: ${{ steps.disagree.outcome }}
+          DISAGREE_STATUS: ${{ steps.disagree.outputs.status }}
+          DISAGREE_CODE: ${{ steps.disagree.outputs.exit-code }}
+        run: |
+          set -x
+          test "$AGREE_STATUS" = PASS
+          test "$AGREE_CODE" = 0
+          test "$DISAGREE_OUTCOME" = failure
+          test "$DISAGREE_STATUS" = FAIL
+          test "$DISAGREE_CODE" = 1
+          test -f gate-agree/report.html -a -f gate-agree/report.json -a -f gate-agree/gate.json

judgekeeper-0.1.0/.github/workflows/pages.yml

@@ -0,0 +1,63 @@
+# Publishes the website (website/) to GitHub Pages at the site root, with the HTML reports
+# under docs/examples/ at examples/<name>/, e.g.
+# https://<owner>.github.io/judgekeeper/ and https://<owner>.github.io/judgekeeper/examples/llmbar-haiku/
+# Inert until the maintainer enables Pages (Settings > Pages > Source: GitHub Actions; see
+# docs/RELEASING.md): until then the build job sees no Pages site and the deploy job is skipped.
+name: Pages
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pages: read
+    outputs:
+      enabled: ${{ steps.check.outputs.enabled }}
+    steps:
+      - name: Is Pages enabled for this repository?
+        id: check
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if gh api "repos/$GITHUB_REPOSITORY/pages" --silent 2>/dev/null; then
+            echo "enabled=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "enabled=false" >> "$GITHUB_OUTPUT"
+            echo "Pages is not enabled for this repository: nothing to publish." >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build the site
+        run: python scripts/build_pages.py website docs/examples _site
+      - if: steps.check.outputs.enabled == 'true'
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: _site
+
+  deploy:
+    needs: build
+    if: needs.build.outputs.enabled == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v4

judgekeeper-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,86 @@
+# Publishes judgekeeper and pytest-judgekeeper to PyPI when a GitHub release is published.
+# Nothing else triggers it. PyPI Trusted Publishing (OIDC) authenticates the upload: there is
+# no API token anywhere. One-time setup by the maintainer: docs/RELEASING.md.
+name: Release
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install build tools
+        run: python -m pip install --upgrade pip build twine
+      - name: Build both distributions
+        run: |
+          python -m build --outdir dist .
+          python -m build --outdir dist packages/pytest-judgekeeper
+      - name: Check the metadata PyPI will render
+        run: twine check --strict dist/*
+      - name: Check the tag matches the version
+        env:
+          TAG: ${{ github.event.release.tag_name }}
+        run: |
+          version=$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')
+          test "$TAG" = "v$version"
+          ls dist/judgekeeper-"$version"-py3-none-any.whl dist/pytest_judgekeeper-"$version"-py3-none-any.whl
+      - name: Run the test suite against the built wheel in a clean venv
+        run: |
+          python -m venv "$RUNNER_TEMP/venv"
+          "$RUNNER_TEMP/venv/bin/pip" install --find-links dist dist/judgekeeper-*.whl \
+            dist/pytest_judgekeeper-*.whl "pytest>=8" "pyyaml>=6"
+          # src/ is not on the path: every test imports the installed wheel.
+          "$RUNNER_TEMP/venv/bin/python" -c 'import judgekeeper, sys; assert "site-packages" in judgekeeper.__file__, judgekeeper.__file__'
+          "$RUNNER_TEMP/venv/bin/python" -m pytest -p no:cacheprovider
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  # One upload job per PyPI project, each behind its own environment (pypi, pypi-plugin):
+  # PyPI refuses two pending Trusted Publishers with the same repository, workflow and
+  # environment, and both projects are new.
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/judgekeeper/
+    permissions:
+      id-token: write  # Trusted Publishing: PyPI exchanges this OIDC token for a short upload token
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Keep only judgekeeper
+        run: rm dist/pytest_judgekeeper-*
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  # pytest-judgekeeper pins judgekeeper==<version>, so it goes up second.
+  publish-plugin:
+    needs: [build, publish]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi-plugin
+      url: https://pypi.org/project/pytest-judgekeeper/
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Keep only pytest-judgekeeper
+        run: rm dist/judgekeeper-*
+      - uses: pypa/gh-action-pypi-publish@release/v1

judgekeeper-0.1.0/.gitignore

@@ -0,0 +1,18 @@
+__pycache__/
+*.pyc
+.venv/
+venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.env
+.DS_Store
+# judge outputs and reports are reproducible artifacts; commit only curated examples under docs/
+/runs/
+/reports/
+anchors/
+# `judgekeeper demo` and `judgekeeper check` write here by default
+/judgekeeper-demo/
+/judgekeeper-report/

judgekeeper-0.1.0/CHANGELOG.md

@@ -0,0 +1,29 @@
+# Changelog
+
+## 0.1.0 (2026-10-02)
+
+The first release: alpha, so commands, flags and `report.json` keys may still change.
+
+- Validate an LLM judge against a frozen human-labeled anchor set: TPR and TNR with 95%
+  intervals, Cohen's kappa, the noise floor across repeated runs, AB/BA position bias, per-slice
+  numbers and label-quality warnings, in `report.json` and a self-contained `report.html`.
+- Judges: built-in Anthropic and OpenAI-compatible runners, your own Python function
+  (`check_judge`, `--callable`), any program (`--exec`), and a replay runner for tests. Every
+  judgment carries a judge fingerprint; API keys are read from environment variables only and
+  scrubbed from everything judgekeeper prints or writes.
+- No framework needed: `check` for a CSV or JSONL of verdicts and labels, `demo` with bundled
+  synthetic data, `template` and `import-labels` for spreadsheet labels, and `label`, a local
+  labeling page.
+- Readers for promptfoo, DeepEval, Inspect AI, MLflow and Langfuse results, and the ScoreRecord
+  import and export format.
+- Keep checking: `baseline`, `gate` (PASS, FAIL, FLAKY, JUDGE_CHANGED, ANCHORS_CHANGED, never
+  failing inside the noise band), a GitHub Action, a pytest plugin (also published as
+  `pytest-judgekeeper`), `migrate` to compare an old and a new judge, and `attribute` to tell
+  judge drift from a change in your system.
+- A skill file for coding agents: `skills/judgekeeper/SKILL.md`.
+- A static website in `website/`: what judgekeeper does, step by step, with an interactive
+  confusion matrix, a setup guide for API keys, a hands-on tutorial and the reference.
+  Published by GitHub Pages once enabled; preview with `python3 -m http.server --directory website`.
+
+Measured so far: one live run, LLMBar judged by `claude-haiku-4-5-20251001`
+(`docs/examples/llmbar-haiku/`). The live judge-migration demo has not been run yet.

judgekeeper-0.1.0/CLAUDE.md

@@ -0,0 +1,28 @@
+# judgekeeper
+
+Open-source Python toolkit that validates, monitors and migrates an LLM-as-judge against a frozen human-labeled anchor set. Read `README.md` for positioning and `docs/research/00-judge-reliability-synthesis.md` for the evidence behind every design choice.
+
+## How work is organised
+
+- Each build session has a spec in `docs/specs/`. Do the spec in front of you, nothing beyond it. Non-goals in a spec are real.
+- Package code lives in `src/judgekeeper/`. Tests live in `tests/`.
+- Public human-labeled datasets used for demos are downloaded by loader code, never committed. Curated example reports go under `docs/examples/`.
+
+## Commands
+
+```
+uv venv && source .venv/bin/activate
+uv pip install -e ".[dev]"
+pytest
+ruff check src tests
+```
+
+## Rules
+
+- Tests first. Unit tests never call an LLM API. Judge calls go behind a runner interface with a replay runner that reads recorded judgments from JSONL fixtures.
+- Every judgment written to disk carries a judge fingerprint (provider, model id, snapshot, prompt hash, rubric version, temperature, timestamp). No exceptions.
+- Report kappa, TPR and TNR. Never report raw agreement on its own.
+- Keep the core dependency-free where practical. Metrics are implemented in plain Python; numpy is acceptable, scikit-learn is not required.
+- Anthropic and OpenAI runners read API keys from `ANTHROPIC_API_KEY` and `OPENAI_API_KEY`. Never write keys to files.
+- Do not invent numbers in README or docs. Every figure must come from a run whose report is committed under `docs/examples/`.
+- Commit messages: short imperative subject, body explains why.

judgekeeper-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sathvik Thota
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

judgekeeper-0.1.0/PKG-INFO

@@ -0,0 +1,207 @@
+Metadata-Version: 2.5
+Name: judgekeeper
+Version: 0.1.0
+Summary: Validate, monitor and migrate your LLM-as-judge against a frozen human reference set.
+Project-URL: Homepage, https://github.com/judgekeeper/judgekeeper
+Project-URL: Changelog, https://github.com/judgekeeper/judgekeeper/blob/main/CHANGELOG.md
+Author: Sathvik Thota
+License-Expression: MIT
+License-File: LICENSE
+Keywords: calibration,drift,evaluation,llm,llm-as-judge
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.40; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: build>=1; extra == 'dev'
+Requires-Dist: hatchling>=1.26; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: pyyaml>=6; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: inspect
+Requires-Dist: inspect-ai>=0.3; extra == 'inspect'
+Provides-Extra: mlflow
+Requires-Dist: mlflow>=3.1; extra == 'mlflow'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# judgekeeper
+
+**Checks whether your LLM-as-judge agrees with human labels, and keeps checking:** TPR, TNR and kappa against a frozen human-labeled set, run-to-run noise, position bias, drift and a CI gate.
+
+```
+pip install judgekeeper
+# before the first PyPI release: pip install "judgekeeper @ git+https://github.com/judgekeeper/judgekeeper"
+judgekeeper demo
+```
+
+`demo` validates a recorded judge on 100 bundled synthetic items: no API key, no network, a few seconds. It prints TPR, TNR and kappa and writes `judgekeeper-demo/report.html`. Without installing: `uvx judgekeeper demo` (before the PyPI release: `uvx --from git+https://github.com/judgekeeper/judgekeeper judgekeeper demo`).
+
+See a real report without installing anything: [LLMBar judged by Claude Haiku 4.5](docs/examples/llmbar-haiku/report.html), also published at https://judgekeeper.github.io/judgekeeper/examples/llmbar-haiku/ (the pattern is `https://<owner>.github.io/judgekeeper/examples/llmbar-haiku/`).
+
+New to judge evaluation? The website in [`website/`](website/) explains it step by step, with a setup guide for API keys and a hands-on tutorial. Preview it locally: `python3 -m http.server --directory website 8000`, then open http://localhost:8000.
+
+Already have judge verdicts and human labels in a CSV? One command:
+
+```
+judgekeeper check results.csv --judge verdict --human label --out reports/my-judge/
+```
+
+One row per judgment. `id`, `run`, `input`, `output` and `reason` columns are used when present. Verdicts can be `pass`/`fail`, `true`/`false`, `yes`/`no`, `correct`/`incorrect`, `1`/`0` or text starting with `PASS`/`FAIL`; scores need a rule (`--pass-if "score>=0.5"`); other spellings need `--label-map "good=pass,bad=fail"`. judgekeeper never guesses.
+
+- No labels yet? `judgekeeper label items.jsonl` opens a local labeling page (keys 1/2, defer, undo, notes) that writes `labels.csv` as you go. [Details](docs/reference.md#label-a-local-labeling-page).
+- Gate from pytest: `pytest --judgekeeper-report reports/my-judge/report.json`, a `judgekeeper_gate` fixture or a `@pytest.mark.judgekeeper` marker. [Details](docs/reference.md#pytest-plugin).
+- Let a coding agent wire it in: [`skills/judgekeeper/SKILL.md`](skills/judgekeeper/SKILL.md) tells Claude Code and other agents how, step by step.
+
+> Status: alpha (0.1.0). The validation report, CI gate and GitHub Action, pytest plugin, labeling page, judge migration, drift attribution and the promptfoo, DeepEval, Inspect AI, MLflow and Langfuse readers work. [`docs/reference.md`](docs/reference.md) has every command, flag, exit code and config key; [`CHANGELOG.md`](CHANGELOG.md) what is in 0.1.0.
+
+## Why
+
+Teams grade AI outputs with another model, the "judge". Judges disagree with humans more than raw agreement suggests, flip verdicts between identical runs, and change silently when the provider updates the model. judgekeeper measures a judge against a frozen set of human labels, reports TPR and TNR (with 95% intervals) and kappa rather than raw agreement, and tells you, later, whether the judge is still right.
+
+The report leads with TPR (how often the judge passes what humans passed) and TNR (how often it fails what humans failed). Either below 0.80 is "not trustworthy as a gate"; 0.80 to 0.90 is "usable with care". It also shows kappa, a confusion matrix, every disagreement with the judge's rationale, the noise floor across repeated runs ("unknown" with one run, never zero), AB/BA position bias for pairwise items, per-slice numbers, and warnings when there are fewer than 60 labels or the classes split worse than 80/20.
+
+## I have a spreadsheet
+
+Judge verdicts and human labels already in one table: `judgekeeper check` above. Give it a `run` column (or several rows per id with different runs) to measure run-to-run noise; with one run the noise floor is reported as unknown and `gate` returns `FLAKY`. Without an `id` column, ids are a hash of input and output and the report says so. In a notebook:
+
+```python
+import judgekeeper
+report = judgekeeper.check_table(df, judge="verdict", human="label")   # a DataFrame, a list of dicts or a path
+```
+
+No labels yet? Label in the local page or in Excel or Google Sheets, then freeze the labels as an anchor set:
+
+```
+judgekeeper label items.jsonl --out labels.csv        # or: judgekeeper template items.jsonl -o labels.csv
+judgekeeper import-labels labels.csv -o anchors.jsonl # checks every label, skips and lists unlabeled rows, freezes
+```
+
+## I have a judge function
+
+Run it 3 times over the anchor set (pairwise items in both AB and BA order) and get a report:
+
+```python
+import judgekeeper
+def my_judge(item):          # item: id, input, output (never the human label)
+    return call_my_model(item) # bool, "PASS: ...", 0.8, (verdict, reason) or {"verdict": ..., "reason": ...}
+report = judgekeeper.check_judge(my_judge, "anchors.jsonl", runs=3, fingerprint={"model": "my-model"})
+```
+
+Async functions work too. A judge that raises or returns nothing is recorded as an error, excluded from the metrics and counted, never scored as a fail. From the command line, a Python function or a program in any language (one item as JSON on stdin, a verdict on stdout):
+
+```
+judgekeeper judge anchors.jsonl --callable mypkg.judges:my_judge --runs 3 --out runs/mine/
+judgekeeper judge anchors.jsonl --exec "node judge.js" --runs 3 --out runs/mine/
+judgekeeper validate anchors.jsonl runs/mine/ --out reports/mine/
+```
+
+judgekeeper prints the number of judge calls first and asks for `--yes` above 1,000. The `--exec` contract, with 10-line Node and Python judges, is in [`docs/reference.md`](docs/reference.md). Built-in Anthropic and OpenAI-compatible runners (`--runner anthropic|openai`, any OpenAI-compatible endpoint via `--base-url`) are documented there too. Keys stay in environment variables; nothing judgekeeper writes or prints contains one.
+
+## I use a framework
+
+Point judgekeeper at the files your eval tool already writes and add human labels. Your eval code does not change.
+
+promptfoo (human labels from web-UI ratings, or `--labels`; repeats from `--repeat 3`):
+
+```
+promptfoo eval -o results.json --repeat 3
+judgekeeper import promptfoo results.json --metric helpfulness --out reports/helpfulness/
+```
+
+DeepEval (one `test_run_*.json` per run in a results folder; labels from `--labels`):
+
+```
+export DEEPEVAL_RESULTS_FOLDER=deepeval-results   # then run your DeepEval tests 3 times
+judgekeeper import deepeval deepeval-results/ --metric "Correctness [GEval]" --labels labels.csv --out reports/correctness/
+```
+
+Inspect AI (epochs are runs; labels from score edits or `--labels`; `.eval` logs need the `judgekeeper[inspect]` extra):
+
+```
+inspect eval task.py --epochs 3 --log-format json
+judgekeeper import inspect logs/ --metric model_graded_qa --labels labels.csv --out reports/qa/
+```
+
+MLflow (judge and human assessments on your traces; each evaluation run is a run; needs the `judgekeeper[mlflow]` extra):
+
+```
+judgekeeper import mlflow --experiment my-app-eval --metric correctness --out reports/correctness/
+```
+
+Langfuse (judge and human scores from the public API; keys from `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY`):
+
+```
+judgekeeper import langfuse --judge-score helpfulness --human-score helpfulness_human --from 2026-09-01 --pass-if "score>=0.5" --out reports/helpfulness/
+```
+
+Each writes the same `report.json` and `report.html` as `check`. With `--anchors-out anchors.jsonl`, the MLflow and Langfuse imports also freeze the labeled items as an anchor set, so you can re-judge them with `judgekeeper judge` for a real noise floor. How to get each file, how labels get in and each tool's traps: [`docs/integrations/`](docs/integrations/). Any other tool can export ScoreRecords (`target_id, name, annotator_kind, label, score, explanation, run, input, output, evaluator, created_at`) and use `judgekeeper import records`, with `--map` for renamed columns; `judgekeeper export records` writes judgekeeper's runs in that format.
+
+## Keep checking
+
+```
+judgekeeper baseline set reports/my-judge/report.json   # commit .judgekeeper/baseline.json
+judgekeeper gate reports/my-judge/report.json           # PASS, FAIL, FLAKY, JUDGE_CHANGED, ANCHORS_CHANGED
+judgekeeper migrate anchors.jsonl runs/old/ runs/new/ --out reports/migration/
+judgekeeper attribute reports/now/report.json --app-score-before X --app-score-after Y
+```
+
+`gate` never fails on a change inside the noise band. In CI, the GitHub Action (`action.yml`) runs judge, validate and gate, and the pytest plugin gates a report your pipeline already wrote. A judge field that is unknown on either side (imported data rarely records temperature or snapshot) is a warning, not a block, unless you pass `--require-fingerprint`. `migrate` compares an old and a new judge item by item; `attribute` says whether a score moved because your system changed or the judge did. Details: [`docs/reference.md`](docs/reference.md).
+
+## First results
+
+The judge is strong on ordinary items and weak on adversarial ones: `claude-haiku-4-5-20251001` reaches kappa 0.94 against human labels on LLMBar's Natural slice but 0.56 on Adversarial/GPTOut ([report](docs/examples/llmbar-haiku/report.html), data in [`docs/examples/llmbar-haiku/`](docs/examples/llmbar-haiku/)).
+
+The run: LLMBar (Natural and Adversarial subsets, 419 human-labeled pairwise items) judged by `claude-haiku-4-5-20251001` at temperature 0, 3 runs, AB and BA. `scripts/llmbar_haiku_demo.sh` runs it end to end and writes the report to `docs/examples/llmbar-haiku/`.
+
+Run on 2026-10-01 against `api.anthropic.com`. Every number below is quoted from `docs/examples/llmbar-haiku/report.json` (human-readable version: `report.html` in the same folder). TPR and TNR treat human label `A` as the positive class.
+
+Verdict: **Usable as a gate: kappa 0.83, TPR 0.95, TNR 0.88 against human labels.**
+
+| Run | Kappa | TPR | TNR |
+|---|---|---|---|
+| 1 | 0.82 | 0.95 | 0.87 |
+| 2 | 0.85 | 0.96 | 0.89 |
+| 3 | 0.82 | 0.95 | 0.87 |
+| Mean | 0.83 | 0.95 | 0.88 |
+
+By slice (mean over runs):
+
+| Slice | Items | Kappa | TPR | TNR |
+|---|---|---|---|---|
+| Natural | 100 | 0.94 | 0.98 | 0.97 |
+| Adversarial/GPTInst | 92 | 0.92 | 1.00 | 0.92 |
+| Adversarial/Neighbor | 134 | 0.84 | 0.96 | 0.88 |
+| Adversarial/Manual | 46 | 0.64 | 0.91 | 0.74 |
+| Adversarial/GPTOut | 47 | 0.56 | 0.86 | 0.71 |
+
+- Noise floor: mean pairwise kappa between runs 0.95. 3.6% of items changed verdict in at least one run.
+- Position bias: 9.9% of judgments changed when the two outputs were swapped (AB vs BA). Kappa on the BA order alone was 0.80.
+- The judge's TNR is lower than its TPR. Its errors cluster in the hardest adversarial slices (GPTOut, Manual), where kappa drops to 0.56 and 0.64.
+
+## Why this and not X
+
+- Vendor "align evals" features (LangSmith, Arize, MLflow, Ragas, Confident AI) do one-off calibration. judgekeeper covers what happens after: time, change, and gating.
+- RAND's Judge Reliability Harness generates bias probes but has no human-kappa, no time series, no CI, and only supports OpenAI.
+- Several zero-star scripts from Aug to Sep 2026 sketch anchor-set drift checks. judgekeeper aims to be the maintained, multi-provider, framework-integrated version with published numbers.
+
+Full landscape and sources: `docs/research/00-judge-reliability-synthesis.md`.
+
+## Roadmap
+
+1. Core validation report on a public human-labeled dataset (done; LLMBar above).
+2. GitHub Action and noise-floor gate (done).
+3. Judge migration and drift attribution (built; the live migration demo is pending a key).
+4. Use without a framework: `check`, `check_table`, `check_judge`, `--callable`, `--exec`, `demo`, spreadsheet labels (done).
+5. Readers for promptfoo, DeepEval and Inspect AI files, and the ScoreRecord import/export format (done). MLflow and Langfuse readers, with re-judging of imported labels (done).
+6. Launch readiness: labeling page, pytest plugin, agent skill, release and Pages workflows (done; publishing is the maintainer's manual step, [`docs/RELEASING.md`](docs/RELEASING.md)).
+7. Demo agent (Claude Agent SDK), clarification (ask-vs-act) judge, MCP server.
+
+## License
+
+MIT.