eval-builder 0.1.0__tar.gz

eval_builder-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: ci
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python: ["3.11", "3.12", "3.13"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv sync --python ${{ matrix.python }}
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run mypy src
+      - run: uv run pytest -q
+      - run: uv build

eval_builder-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,34 @@
+name: release
+
+# Publishes to PyPI when a version tag (v*) is pushed, using PyPI trusted publishing (no API token).
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

eval_builder-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+__pycache__/
+*.py[cod]
+.venv/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+dist/
+build/
+*.egg-info/
+.DS_Store
+evalset/
+.deepeval/
+# Example inputs that prepare.py downloads or derives (reproducible from the pinned dataset)
+examples/mt-bench/data/
+# Large or derived files inside example workspaces
+examples/mt-bench/*/judge_requests.jsonl
+examples/mt-bench/*/traces.jsonl

eval_builder-0.1.0/AGENTS.md

@@ -0,0 +1,45 @@
+# Working on eval-builder
+
+This file is for people and coding agents changing eval-builder itself. The agent
+workflow for *using* the tool lives in `skills/eval-builder/SKILL.md`.
+
+## Layout
+
+```
+src/eval_builder/
+  schema.py        Trace dataclass and helpers every ingest format maps into
+  ingest/          format detection and parsers (openai, anthropic, langfuse, otel, generic)
+  redact.py        default-on secret and PII redaction with counts
+  select.py        dedupe, near-duplicates, k-means clusters, strata, failure oversampling
+  draft.py         case and rubric skeletons, validation, update_case
+  judge/plan.py    judge requests with repeated trials and swap/pad probes
+  judge/run.py     opt-in judge plugin runner (off by default, runs a user command)
+  judge/check.py   flip rate, kappa, accuracy, probes, verdicts
+  judge/stats.py   Wilson interval, Cohen's kappa, majority vote
+  export.py        promptfoo, DeepEval, Inspect AI, JSONL
+  report.py        report.md and report.json
+  setup_agents.py  `eval-builder setup` for Claude Code, Codex, Cursor
+  mcp_server.py    MCP tools over stdio
+  cli.py           argparse CLI, every command supports --json
+```
+
+## Rules
+
+- No model calls and no network access in the package. The judge runner only starts a
+  command the user names, and only with an explicit flag.
+- Deterministic: same inputs and seed give the same selection and the same files.
+- Every number the tool reports must be traceable to a file in the workspace.
+- Tests are offline, fast, and never touch the real home directory (`conftest.py`
+  points HOME at a temp dir). Add a fixture when you add a format.
+- Statistics changes need a test against a value computed by hand, with the arithmetic
+  in a comment.
+- Writing style: no em dashes or en dashes, no hype words, plain sentences.
+
+## Commands
+
+```
+uv sync
+uv run pytest -q
+uv run ruff check . && uv run ruff format --check .
+uv run mypy src
+```

eval_builder-0.1.0/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+
+## 0.1.0 (2026-10-08)
+
+First release.
+
+- `ingest`: OpenAI chat JSONL (fine-tuning style and request/response logs), Anthropic
+  messages, Langfuse trace exports, OpenTelemetry GenAI spans (OTLP JSON and flat span
+  JSON), and generic input/output JSONL. Default-on redaction with a per-kind report.
+- `select`: exact and near-duplicate removal, TF-IDF k-means clusters, stratum
+  coverage, failure oversampling, a recorded reason for every pick.
+- `draft`, `validate`: case and rubric templates that the agent fills in with the user.
+- `judge-plan`, `judge-run` (opt-in), `judge-check`: repeated trials, flip rate,
+  agreement with human labels (accuracy with Wilson intervals, Cohen's kappa), answer
+  order and padding probes, a verdict per judge.
+- `export`: promptfoo, DeepEval, Inspect AI, JSONL.
+- `report`: Markdown and JSON.
+- `setup`: registers the MCP server with Claude Code, Codex and Cursor.

eval_builder-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,11 @@
+# Contributing
+
+Thanks for helping. Before opening a pull request:
+
+1. `uv sync`
+2. `uv run pytest -q` (offline, under a minute)
+3. `uv run ruff check . && uv run ruff format --check . && uv run mypy src`
+
+New log formats need a small fixture in `tests/fixtures/` and a test. Changes to the
+judge statistics need a test against a hand-computed value. See `AGENTS.md` for the
+layout and the rules (no model calls, deterministic output, no em dashes in text).

eval_builder-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Abel Yagubyan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

eval_builder-0.1.0/PKG-INFO

@@ -0,0 +1,154 @@
+Metadata-Version: 2.5
+Name: eval-builder
+Version: 0.1.0
+Summary: Turn real LLM app logs into an eval suite and measure which LLM judges you can trust.
+Author: Abel Yagubyan
+License-Expression: MIT
+License-File: LICENSE
+Keywords: deepeval,evals,inspect-ai,llm,llm-as-a-judge,mcp,promptfoo
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: mcp>=1.2
+Requires-Dist: numpy>=1.24
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: scikit-learn>=1.3
+Requires-Dist: scipy>=1.10
+Description-Content-Type: text/markdown
+
+# eval-builder
+
+Your agent turns your app's real logs into an eval suite, and tells you which of its judges you can actually trust.
+
+```sh
+uvx eval-builder ingest ./logs                 # OpenAI, Anthropic, Langfuse, OpenTelemetry or JSONL
+uvx eval-builder select -n 40 && uvx eval-builder draft
+uvx eval-builder judge-check                   # after your agent has run each judge a few times
+```
+
+> Not on PyPI yet. Until the first release, run it straight from GitHub by replacing `uvx eval-builder` with
+> `uvx --from git+https://github.com/Abelo9996/eval-builder eval-builder`. `setup` registers `uvx eval-builder mcp`, so it works once the
+> package is on PyPI.
+
+eval-builder is not another eval platform. It builds the suite and checks the judges,
+then exports to the tools you already run: promptfoo, DeepEval, Inspect AI, or plain
+JSONL. It never calls a model. Your coding agent (Claude Code, Codex, Cursor) does the
+thinking through the MCP server; eval-builder does the selection, the bookkeeping and
+the statistics, and writes down the evidence.
+
+## Example: real output
+
+Everything below comes from runs on this machine (Apple M4, 16 GB) that are committed
+in [`examples/mt-bench/`](examples/mt-bench/). The data is the LMSYS
+[MT-Bench human judgments](https://huggingface.co/datasets/lmsys/mt_bench_human_judgments)
+dataset (CC BY 4.0): 80 two-turn questions answered by 6 models, plus pairwise verdicts
+from expert human judges.
+
+**Logs to suite.** `ingest` read 480 conversations (OpenAI chat format, redaction on,
+0 matches). `select -n 24 --stratify category` removed 400 exact duplicates (each
+question was asked to six models), clustered the 80 unique conversations into 9 topics
+and picked 24 covering all 8 categories, 19 of them failures. From
+[`suite/report.md`](examples/mt-bench/suite/report.md):
+
+```
+| trace           | cluster | represents | why it was picked |
+| q81-alpaca-13b  | 0 | 6 | failure (negative user feedback); 62% of unique traces are failures and at least 30% of picks are reserved for them |
+| q82-alpaca-13b  | 0 | 6 | adds variety within cluster 0 (11 traces, 14%; response, previous response, previous); least similar to cases already picked there |
+| q156-alpaca-13b | 3 | 6 | covers category=humanities (10 unique traces, 12%) |
+```
+
+The agent wrote expected behavior for the 24 cases, and `export` produced files that
+Inspect AI 0.3.277, DeepEval 4.2.8 and promptfoo 0.124.0 all load (see
+[`verify_exports.sh`](examples/mt-bench/verify_exports.sh)).
+
+**Which judges can you trust?** Five local LLM judge setups (four models through
+Ollama, one of them also at temperature 0) and two controls that are not language models judged 80 answer pairs where human experts
+picked a winner. Each judge ran 5 times per pair, 5 more with the answers swapped, and
+5 more with an irrelevant paragraph appended to one answer: 8,400 calls, 0 errors.
+`eval-builder judge-check`, from [`judges/report.md`](examples/mt-bench/judges/report.md):
+
+| judge | verdict | flip rate | accuracy vs humans | kappa | survives answer swap | first-shown answer picked | padding helped |
+|---|---|---|---|---|---|---|---|
+| qwen2.5:7b-instruct, temp 0 | biased | 0% [0%, 5%] | 72% [62%, 81%] | 0.45 [0.26, 0.65] | 75% [65%, 83%] | 44% | 0% |
+| qwen2.5:7b-instruct, temp 0.8 | biased | 12% [7%, 22%] | 70% [59%, 79%] | 0.40 [0.20, 0.60] | 75% [65%, 83%] | 42% | 1% |
+| qwen2.5:3b-instruct | biased | 12% [7%, 22%] | 56% [45%, 67%] | 0.13 [-0.08, 0.35] | 44% [33%, 55%] | 35% | 9% |
+| gemma2:2b | unstable | 59% [48%, 69%] | 56% [45%, 67%] | 0.14 [-0.07, 0.36] | 38% [28%, 48%] | 35% | 11% |
+| llama3.2:3b | unstable | 56% [45%, 67%] | 51% [40%, 62%] | 0.04 [-0.17, 0.26] | 14% [8%, 23%] | 20% | 10% |
+| control: longer answer wins | biased | 0% [0%, 5%] | 66% [55%, 76%] | 0.32 [0.12, 0.53] | 100% [95%, 100%] | 50% | 30% |
+| control: coin flip | unstable | 95% [88%, 98%] | 51% [40%, 62%] | 0.03 [-0.19, 0.24] | 55% [44%, 65%] | 50% | 25% |
+
+n = 80 pairs per judge; brackets are 95% intervals. No judge passed. The closest,
+qwen2.5 7B, reaches kappa 0.40 to 0.45 with the human experts, but its verdict
+flips on a quarter of the pairs when the answer order is swapped, and setting the
+temperature to 0 removes the run-to-run flips without removing that order effect.
+llama3.2 3B picked whichever answer it saw second 80% of the time. The longer-answer
+rule never flips and agrees with humans 66% of the time, which is why stability and
+accuracy alone are not enough: the padding probe catches it.
+
+The same check on the suite's own pass/fail judge (qwen2.5 7B, 24 cases, no human
+labels) gives `unstable`: its verdict changed across 5 identical calls on 7 of 24
+cases (29%, interval 15% to 49%).
+
+## How it works
+
+| step | what it does | what it uses |
+|---|---|---|
+| `ingest` | Reads OpenAI chat JSONL (fine-tuning style or request/response logs), Anthropic messages, Langfuse trace exports, OpenTelemetry GenAI spans (OTLP JSON, current `gen_ai.input.messages` and older `gen_ai.prompt.N` attributes), or generic input/output JSONL. Normalizes to one schema with input, output, earlier turns, tools called, error flag, user feedback, route and model. Redacts emails, API keys, Luhn-valid card numbers, phone numbers and SSN-shaped numbers by default and reports counts. | Python standard library |
+| `select` | Removes exact duplicates (normalized hash of the user turns) and near-duplicates (character n-gram cosine), clusters the rest by topic, then picks: failures first (errors, negative feedback), at least one case per stratum (route, tool, feedback, any metadata key you name), one central case per cluster, and fills the rest by cluster size with the least similar remaining cases. Every pick records why it was picked. | scikit-learn (TF-IDF, k-means) |
+| `draft`, `validate` | Writes `cases.yaml` and `rubric.yaml` with TODO markers. The agent fills in expected behavior and criteria with you. Validation refuses ready cases that still contain TODO or reference unknown criteria. | PyYAML |
+| `judge-plan` | Lists every judge call to make: each case N times, plus probes that swap the answer order (pairwise judges) and pad an answer with an irrelevant paragraph. | |
+| `judge-run` | Optional and off by default. Sends each request as a JSON line to a command you name (your script, your provider, your keys) and records the verdicts. eval-builder ships no API keys and no provider code. | your command |
+| `judge-check` | Per judge: flip rate across repeated calls (with a Wilson interval), self-agreement, majority-of-3 vote stability, accuracy and Cohen's kappa against your human labels (with intervals), position consistency and first-shown preference, and how often padding moved the verdict toward the padded answer. Verdict: `trustworthy`, `unstable`, `biased`, `misaligned`, or `not_enough_data`, with the numbers behind it. | |
+| `export` | promptfoo `promptfooconfig.yaml` (llm-rubric asserts), DeepEval dataset plus a `deepeval test run` file, Inspect AI dataset plus `task.py`, plain JSONL. The manifest lists file hashes and which judges passed. | |
+| `report` | `report.md` and `report.json`: sources with sha256, counts, redactions, selection reasons, the judge table, and the limits. | |
+
+The verdict thresholds are explicit flags with defaults: flip rate at most 20% of cases,
+position consistency at least 80%, padding helps at most 10% of cases, kappa at least
+0.4 on at least 20 human-labeled cases. A judge without human labels is never called
+trustworthy.
+
+## Setup for agents
+
+```sh
+uvx eval-builder setup          # shows what it would change
+uvx eval-builder setup --yes    # applies it
+```
+
+`setup` registers the MCP server (`uvx eval-builder mcp`) with Claude Code (`claude mcp
+add --scope user`), Codex (`[mcp_servers.eval-builder]` in `~/.codex/config.toml`) and
+Cursor (`~/.cursor/mcp.json`), and copies the agent instructions to
+`~/.claude/skills/eval-builder/` and `~/.codex/skills/eval-builder/`. It backs up any
+file it edits and does nothing on a second run. The workflow the agent follows is in
+[`skills/eval-builder/SKILL.md`](skills/eval-builder/SKILL.md).
+
+MCP tools: `ingest`, `select`, `draft`, `list_cases`, `update_case`, `set_rubric`,
+`validate`, `judge_plan`, `judge_check`, `export`, `report`, `status`. The judge runner
+is CLI only, because it executes a command.
+
+## What it can't do
+
+- It does not write expected behavior or human labels. The agent drafts expected
+  behavior with you; labels must come from people. Without labels, judge-check can
+  tell you a judge is unstable or biased, but not that it is right.
+- Selection is lexical. Two requests that mean the same thing in different words can
+  land in different clusters, and near-duplicate detection only catches close textual
+  matches.
+- Redaction is pattern-based. Names, street addresses and free-form secrets get
+  through. Look at `traces.jsonl` before sharing a workspace.
+- The bias probes cover answer order and irrelevant length only. Self-preference,
+  style bias and rubric misreadings are not measured.
+- Small samples give wide intervals. The report prints them; read them.
+- It does not run your app or your eval. The exported files do that in promptfoo,
+  DeepEval or Inspect AI.
+
+## Privacy and safety
+
+Everything runs locally. eval-builder makes no network calls and sends nothing
+anywhere. The only process it starts is the judge command you name with
+`judge-run --enable-judge-plugin`. Redaction is on unless you pass `--no-redact`, and
+the report lists what was replaced (counts and kinds, never the values).
+
+## License
+
+MIT. See [LICENSE](LICENSE).

eval_builder-0.1.0/README.md

@@ -0,0 +1,135 @@
+# eval-builder
+
+Your agent turns your app's real logs into an eval suite, and tells you which of its judges you can actually trust.
+
+```sh
+uvx eval-builder ingest ./logs                 # OpenAI, Anthropic, Langfuse, OpenTelemetry or JSONL
+uvx eval-builder select -n 40 && uvx eval-builder draft
+uvx eval-builder judge-check                   # after your agent has run each judge a few times
+```
+
+> Not on PyPI yet. Until the first release, run it straight from GitHub by replacing `uvx eval-builder` with
+> `uvx --from git+https://github.com/Abelo9996/eval-builder eval-builder`. `setup` registers `uvx eval-builder mcp`, so it works once the
+> package is on PyPI.
+
+eval-builder is not another eval platform. It builds the suite and checks the judges,
+then exports to the tools you already run: promptfoo, DeepEval, Inspect AI, or plain
+JSONL. It never calls a model. Your coding agent (Claude Code, Codex, Cursor) does the
+thinking through the MCP server; eval-builder does the selection, the bookkeeping and
+the statistics, and writes down the evidence.
+
+## Example: real output
+
+Everything below comes from runs on this machine (Apple M4, 16 GB) that are committed
+in [`examples/mt-bench/`](examples/mt-bench/). The data is the LMSYS
+[MT-Bench human judgments](https://huggingface.co/datasets/lmsys/mt_bench_human_judgments)
+dataset (CC BY 4.0): 80 two-turn questions answered by 6 models, plus pairwise verdicts
+from expert human judges.
+
+**Logs to suite.** `ingest` read 480 conversations (OpenAI chat format, redaction on,
+0 matches). `select -n 24 --stratify category` removed 400 exact duplicates (each
+question was asked to six models), clustered the 80 unique conversations into 9 topics
+and picked 24 covering all 8 categories, 19 of them failures. From
+[`suite/report.md`](examples/mt-bench/suite/report.md):
+
+```
+| trace           | cluster | represents | why it was picked |
+| q81-alpaca-13b  | 0 | 6 | failure (negative user feedback); 62% of unique traces are failures and at least 30% of picks are reserved for them |
+| q82-alpaca-13b  | 0 | 6 | adds variety within cluster 0 (11 traces, 14%; response, previous response, previous); least similar to cases already picked there |
+| q156-alpaca-13b | 3 | 6 | covers category=humanities (10 unique traces, 12%) |
+```
+
+The agent wrote expected behavior for the 24 cases, and `export` produced files that
+Inspect AI 0.3.277, DeepEval 4.2.8 and promptfoo 0.124.0 all load (see
+[`verify_exports.sh`](examples/mt-bench/verify_exports.sh)).
+
+**Which judges can you trust?** Five local LLM judge setups (four models through
+Ollama, one of them also at temperature 0) and two controls that are not language models judged 80 answer pairs where human experts
+picked a winner. Each judge ran 5 times per pair, 5 more with the answers swapped, and
+5 more with an irrelevant paragraph appended to one answer: 8,400 calls, 0 errors.
+`eval-builder judge-check`, from [`judges/report.md`](examples/mt-bench/judges/report.md):
+
+| judge | verdict | flip rate | accuracy vs humans | kappa | survives answer swap | first-shown answer picked | padding helped |
+|---|---|---|---|---|---|---|---|
+| qwen2.5:7b-instruct, temp 0 | biased | 0% [0%, 5%] | 72% [62%, 81%] | 0.45 [0.26, 0.65] | 75% [65%, 83%] | 44% | 0% |
+| qwen2.5:7b-instruct, temp 0.8 | biased | 12% [7%, 22%] | 70% [59%, 79%] | 0.40 [0.20, 0.60] | 75% [65%, 83%] | 42% | 1% |
+| qwen2.5:3b-instruct | biased | 12% [7%, 22%] | 56% [45%, 67%] | 0.13 [-0.08, 0.35] | 44% [33%, 55%] | 35% | 9% |
+| gemma2:2b | unstable | 59% [48%, 69%] | 56% [45%, 67%] | 0.14 [-0.07, 0.36] | 38% [28%, 48%] | 35% | 11% |
+| llama3.2:3b | unstable | 56% [45%, 67%] | 51% [40%, 62%] | 0.04 [-0.17, 0.26] | 14% [8%, 23%] | 20% | 10% |
+| control: longer answer wins | biased | 0% [0%, 5%] | 66% [55%, 76%] | 0.32 [0.12, 0.53] | 100% [95%, 100%] | 50% | 30% |
+| control: coin flip | unstable | 95% [88%, 98%] | 51% [40%, 62%] | 0.03 [-0.19, 0.24] | 55% [44%, 65%] | 50% | 25% |
+
+n = 80 pairs per judge; brackets are 95% intervals. No judge passed. The closest,
+qwen2.5 7B, reaches kappa 0.40 to 0.45 with the human experts, but its verdict
+flips on a quarter of the pairs when the answer order is swapped, and setting the
+temperature to 0 removes the run-to-run flips without removing that order effect.
+llama3.2 3B picked whichever answer it saw second 80% of the time. The longer-answer
+rule never flips and agrees with humans 66% of the time, which is why stability and
+accuracy alone are not enough: the padding probe catches it.
+
+The same check on the suite's own pass/fail judge (qwen2.5 7B, 24 cases, no human
+labels) gives `unstable`: its verdict changed across 5 identical calls on 7 of 24
+cases (29%, interval 15% to 49%).
+
+## How it works
+
+| step | what it does | what it uses |
+|---|---|---|
+| `ingest` | Reads OpenAI chat JSONL (fine-tuning style or request/response logs), Anthropic messages, Langfuse trace exports, OpenTelemetry GenAI spans (OTLP JSON, current `gen_ai.input.messages` and older `gen_ai.prompt.N` attributes), or generic input/output JSONL. Normalizes to one schema with input, output, earlier turns, tools called, error flag, user feedback, route and model. Redacts emails, API keys, Luhn-valid card numbers, phone numbers and SSN-shaped numbers by default and reports counts. | Python standard library |
+| `select` | Removes exact duplicates (normalized hash of the user turns) and near-duplicates (character n-gram cosine), clusters the rest by topic, then picks: failures first (errors, negative feedback), at least one case per stratum (route, tool, feedback, any metadata key you name), one central case per cluster, and fills the rest by cluster size with the least similar remaining cases. Every pick records why it was picked. | scikit-learn (TF-IDF, k-means) |
+| `draft`, `validate` | Writes `cases.yaml` and `rubric.yaml` with TODO markers. The agent fills in expected behavior and criteria with you. Validation refuses ready cases that still contain TODO or reference unknown criteria. | PyYAML |
+| `judge-plan` | Lists every judge call to make: each case N times, plus probes that swap the answer order (pairwise judges) and pad an answer with an irrelevant paragraph. | |
+| `judge-run` | Optional and off by default. Sends each request as a JSON line to a command you name (your script, your provider, your keys) and records the verdicts. eval-builder ships no API keys and no provider code. | your command |
+| `judge-check` | Per judge: flip rate across repeated calls (with a Wilson interval), self-agreement, majority-of-3 vote stability, accuracy and Cohen's kappa against your human labels (with intervals), position consistency and first-shown preference, and how often padding moved the verdict toward the padded answer. Verdict: `trustworthy`, `unstable`, `biased`, `misaligned`, or `not_enough_data`, with the numbers behind it. | |
+| `export` | promptfoo `promptfooconfig.yaml` (llm-rubric asserts), DeepEval dataset plus a `deepeval test run` file, Inspect AI dataset plus `task.py`, plain JSONL. The manifest lists file hashes and which judges passed. | |
+| `report` | `report.md` and `report.json`: sources with sha256, counts, redactions, selection reasons, the judge table, and the limits. | |
+
+The verdict thresholds are explicit flags with defaults: flip rate at most 20% of cases,
+position consistency at least 80%, padding helps at most 10% of cases, kappa at least
+0.4 on at least 20 human-labeled cases. A judge without human labels is never called
+trustworthy.
+
+## Setup for agents
+
+```sh
+uvx eval-builder setup          # shows what it would change
+uvx eval-builder setup --yes    # applies it
+```
+
+`setup` registers the MCP server (`uvx eval-builder mcp`) with Claude Code (`claude mcp
+add --scope user`), Codex (`[mcp_servers.eval-builder]` in `~/.codex/config.toml`) and
+Cursor (`~/.cursor/mcp.json`), and copies the agent instructions to
+`~/.claude/skills/eval-builder/` and `~/.codex/skills/eval-builder/`. It backs up any
+file it edits and does nothing on a second run. The workflow the agent follows is in
+[`skills/eval-builder/SKILL.md`](skills/eval-builder/SKILL.md).
+
+MCP tools: `ingest`, `select`, `draft`, `list_cases`, `update_case`, `set_rubric`,
+`validate`, `judge_plan`, `judge_check`, `export`, `report`, `status`. The judge runner
+is CLI only, because it executes a command.
+
+## What it can't do
+
+- It does not write expected behavior or human labels. The agent drafts expected
+  behavior with you; labels must come from people. Without labels, judge-check can
+  tell you a judge is unstable or biased, but not that it is right.
+- Selection is lexical. Two requests that mean the same thing in different words can
+  land in different clusters, and near-duplicate detection only catches close textual
+  matches.
+- Redaction is pattern-based. Names, street addresses and free-form secrets get
+  through. Look at `traces.jsonl` before sharing a workspace.
+- The bias probes cover answer order and irrelevant length only. Self-preference,
+  style bias and rubric misreadings are not measured.
+- Small samples give wide intervals. The report prints them; read them.
+- It does not run your app or your eval. The exported files do that in promptfoo,
+  DeepEval or Inspect AI.
+
+## Privacy and safety
+
+Everything runs locally. eval-builder makes no network calls and sends nothing
+anywhere. The only process it starts is the judge command you name with
+`judge-run --enable-judge-plugin`. Redaction is on unless you pass `--no-redact`, and
+the report lists what was replaced (counts and kinds, never the values).
+
+## License
+
+MIT. See [LICENSE](LICENSE).

eval_builder-0.1.0/SECURITY.md

@@ -0,0 +1,13 @@
+# Security
+
+eval-builder reads log files you point it at and writes files into a workspace
+directory. It makes no network calls. The optional judge runner starts only the
+command you pass with `--command`, and only when you also pass
+`--enable-judge-plugin`.
+
+Logs often contain personal data. Redaction is on by default but is pattern-based and
+will miss things (names, addresses, free-form secrets). Review the workspace before
+sharing it.
+
+To report a vulnerability, open a private security advisory on the GitHub repository
+or email the maintainer. Please do not file a public issue for security problems.

eval_builder-0.1.0/examples/judges/control_judges.py

@@ -0,0 +1,34 @@
+"""Control judges for `eval-builder judge-run`. These are NOT language models.
+
+  --kind longer   picks whichever answer is longer (a deterministic heuristic)
+  --kind coin     picks A or B at random (seeded by request id), a pure-noise control
+
+They exist to show that judge-check separates "stable but biased" (longer) and
+"noise" (coin) from real judges. Never report them as LLM judges.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--kind", choices=["longer", "coin"], required=True)
+    args = ap.parse_args()
+    for line in sys.stdin:
+        req = json.loads(line)
+        shown = req["presented"]
+        if args.kind == "longer":
+            verdict = "A" if len(shown["answer_a"]) >= len(shown["answer_b"]) else "B"
+        else:
+            h = hashlib.sha256(req["request_id"].encode()).digest()[0]
+            verdict = "A" if h % 2 == 0 else "B"
+        print(json.dumps({"verdict": verdict}), flush=True)
+
+
+if __name__ == "__main__":
+    main()

eval_builder-0.1.0/examples/judges/ollama_judge.py

@@ -0,0 +1,88 @@
+"""Example judge plugin for `eval-builder judge-run`: a local model served by Ollama.
+
+This is an example, not part of the eval-builder package. It talks only to the
+Ollama server on localhost. Protocol: read one JSON request per line on stdin,
+write one JSON response per line on stdout: {"verdict": ..., "raw": ...}.
+
+Usage:
+  eval-builder judge-run --enable-judge-plugin \\
+    --command "qwen-3b=python examples/judges/ollama_judge.py --model qwen2.5:3b-instruct"
+
+Each trial uses seed = trial index, so a rerun on the same machine, model digest and
+Ollama version reproduces the same outputs.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.request
+
+
+def generate(
+    host: str,
+    model: str,
+    prompt: str,
+    temperature: float,
+    seed: int,
+    num_ctx: int,
+    num_predict: int,
+) -> dict:
+    body = json.dumps(
+        {
+            "model": model,
+            "prompt": prompt,
+            "stream": False,
+            "options": {
+                "temperature": temperature,
+                "seed": seed,
+                "num_ctx": num_ctx,
+                "num_predict": num_predict,
+            },
+        }
+    ).encode()
+    req = urllib.request.Request(f"{host}/api/generate", body, {"Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=300) as resp:
+        return json.load(resp)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--temperature", type=float, default=0.8)
+    ap.add_argument("--num-ctx", type=int, default=8192)
+    ap.add_argument("--num-predict", type=int, default=8)
+    ap.add_argument("--host", default="http://127.0.0.1:11434")
+    args = ap.parse_args()
+    for line in sys.stdin:
+        req = json.loads(line)
+        if not req.get("prompt"):
+            print(
+                json.dumps({"verdict": None, "raw": "request has no rendered prompt"}), flush=True
+            )
+            continue
+        out = generate(
+            args.host,
+            args.model,
+            req["prompt"],
+            args.temperature,
+            int(req.get("trial", 0)),
+            args.num_ctx,
+            args.num_predict,
+        )
+        text = out.get("response", "")
+        print(
+            json.dumps(
+                {
+                    "verdict": text.strip(),
+                    "raw": text,
+                    "prompt_tokens": out.get("prompt_eval_count"),
+                }
+            ),
+            flush=True,
+        )
+
+
+if __name__ == "__main__":
+    main()