benchspec 0.0.1__tar.gz

benchspec-0.0.1/.gitignore

@@ -0,0 +1,16 @@
+.venv/
+.worktrees/
+.pytest_cache/
+.ruff_cache/
+__pycache__/
+*.py[cod]
+*.swp
+.env
+tmp/
+dist/
+build/
+*.egg-info/
+.superpowers/
+.claude/scheduled_tasks.lock
+.claude/worktrees/
+.claude/settings.local.json

benchspec-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Swift
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

benchspec-0.0.1/Makefile

@@ -0,0 +1,41 @@
+.PHONY: help install test e2e evals assets lint lint\:ruff lint\:houserules clean
+.DEFAULT_GOAL := help
+
+help:  ## Show this help
+	@awk '/^[a-zA-Z0-9_:\\-]+:.*## / {t=$$0; sub(/:[ \t]*##.*/,"",t); gsub(/\\/,"",t); d=$$0; sub(/^.*## /,"",d); printf "  \033[36m%-11s\033[0m %s\n", t, d}' $(MAKEFILE_LIST)
+
+install:  ## Create the venv and install dev dependencies
+	uv sync
+
+test:  ## Run the unit test suite
+	PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 uv run pytest -p pytester
+
+e2e:  ## Run benchspec's own end-to-end suite (real microVMs; needs claude+codex CLIs and provider credentials)
+	uv run benchspec run --set e2e
+
+# Keep modest: high fan-out trips the Gemini call's ~60s timeout (12-way -> throttling).
+BINDER_WORKERS ?= 6
+evals:  ## Run the binder corpus (binder quality, not framework function). Pass EVAL_ARGS="--collect-only -q" to dry-run collection.
+	uv run pytest -m binder_corpus -n $(BINDER_WORKERS) evals/binder $(EVAL_ARGS)
+
+assets:  ## Re-render the raster brand assets in docs/assets (terminal mock PNG/GIF, social card)
+	uv run scripts/render_assets.py
+
+lint:  ## Lint with Ruff and houserules
+	$(MAKE) lint:ruff
+	$(MAKE) lint:houserules
+
+lint\:ruff:  ## Lint with Ruff
+	uv run ruff check .
+
+LINT_BASE ?= origin/dev
+lint\:houserules:  ## Lint changed and new Python files with houserules (needs GEMINI_API_KEY)
+	uv run houserules --base "$$(git merge-base $(LINT_BASE) HEAD)" --verbose .
+	@untracked_python_files="$$(git ls-files --others --exclude-standard -- '*.py')"; \
+	if [ -n "$$untracked_python_files" ]; then \
+		uv run houserules --verbose $$untracked_python_files; \
+	fi
+
+clean:  ## Remove the venv and Python caches
+	rm -rf .venv .pytest_cache .ruff_cache
+	find . -type d -name __pycache__ -prune -exec rm -rf {} +

benchspec-0.0.1/PKG-INFO

@@ -0,0 +1,210 @@
+Metadata-Version: 2.5
+Name: benchspec
+Version: 0.0.1
+Summary: benchspec is a framework for evaluating AI agents with repeatable, isolated benchmarks. Write evals as Markdown, run each eval across named benchmark arms, and compare how agent behavior changes by harness, model, effort, and environment.
+Project-URL: Homepage, https://github.com/theycallmeswift/benchspec
+Project-URL: Documentation, https://github.com/theycallmeswift/benchspec/tree/dev/docs
+Project-URL: Issues, https://github.com/theycallmeswift/benchspec/issues
+Author-email: Swift <swift@majorleaguehacking.com>
+License: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: pytest>=8
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: pyyaml>=6
+Provides-Extra: claude
+Provides-Extra: codex
+Provides-Extra: microsandbox
+Requires-Dist: microsandbox<0.7,>=0.6.16; extra == 'microsandbox'
+Provides-Extra: opencode
+Provides-Extra: testing
+Description-Content-Type: text/markdown
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="docs/assets/benchspec-wordmark-dark.svg">
+  <img alt="benchspec" src="docs/assets/benchspec-wordmark-light.svg" width="188" height="48">
+</picture>
+
+**Benchmark what your agent does, not what it says.**
+
+[![CI](https://github.com/theycallmeswift/benchspec/actions/workflows/ci.yml/badge.svg)](https://github.com/theycallmeswift/benchspec/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/benchspec)](https://pypi.org/project/benchspec/)
+[![Python](https://img.shields.io/pypi/pyversions/benchspec)](https://pypi.org/project/benchspec/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
+benchspec runs an agent (Claude Code, Codex, or OpenCode) against a task
+in a fresh microVM, checks what it actually did in the workspace, and reports the
+result as a comparison: with your skill versus without, one model versus another,
+one harness versus another.
+
+<img src="docs/assets/benchmark-terminal.gif" width="800" alt="Animated terminal output: benchspec run prints a benchmark matrix with evals as rows, arms as columns, color-coded rates, and percentage-point deltas">
+
+*End-of-run summary for a two-arm run of the in-repo [`hello`](evals/e2e/hello/)
+suite (illustrative numbers).* Rows are evals, columns are arms (`baseline` ran
+the agent bare, `trial` installed the skill), and every non-baseline cell shows
+its assertion pass rate plus the delta against the baseline in percentage
+points. The same matrix lands in `benchmark.md`, with machine-readable artifacts
+alongside.
+
+Teams pick harnesses, models, and prompts by anecdote: run it once, eyeball the
+transcript, trust the vibe. benchspec turns that guess into a measurement.
+Write the goal once, run it across the configurations you care about, and read
+off — in percentage points — how good each one actually is at accomplishing it.
+
+## Getting Started
+
+```bash
+pip install "benchspec[microsandbox]"
+```
+
+> **Pre-1.0.** The eval format and the artifact schemas are the surfaces most
+> likely to change. microsandbox is the only sandbox backend today; `docker` is
+> recognized in config but fails fast as not implemented.
+
+An eval is one Markdown file: a prompt, then a checklist of plain-prose claims
+about the workspace after the agent is done. There is no checker syntax to learn;
+the wording is the spec. `evals/hello/greets-by-name.eval.md`:
+
+```markdown
+---
+---
+
+## Prompt
+
+You are working in a workspace rooted at your current working directory.
+Greet Alice by name.
+
+## Assertions
+
+- [ ] ./Greetings/Alice.md contains the exact line 'Hello, Alice!'
+- [ ] Skill `hello` invoked
+- [ ] The greeting feels warm and personable, not curt or robotic
+```
+
+The benchmark is a block in `pyproject.toml`. Arms are the report columns; the
+baseline is what the others are measured against:
+
+```toml
+[tool.benchspec]
+default-set = "default"
+
+[tool.benchspec.sets.default]
+harness  = "claude-code"
+model    = "sonnet"
+baseline = "baseline"
+arms = [
+  { name = "baseline" },   # installs nothing
+  { name = "trial" },      # setup.sh installs the skill
+]
+```
+
+Then:
+
+```bash
+benchspec lint      # static checks on the assertions
+benchspec analyze   # which assertions grade deterministically, which go to the judge
+benchspec run       # every (eval × arm) in its own microVM, graded, reported
+```
+
+## What you need
+
+| | |
+|---|---|
+| Platform | Apple Silicon Mac, or Linux with `/dev/kvm`. Python 3.11+. |
+| Agent CLI | `claude`, `codex`, or `opencode` on `PATH`, with its credential (for Claude Code, `CLAUDE_CODE_OAUTH_TOKEN` or `ANTHROPIC_API_KEY`). |
+| `GEMINI_API_KEY` | The binder: a fixed Gemini call that classifies each assertion. Required for every `analyze` and `run`. |
+| Judge credential | The judge runs on the host through an agent CLI; the default is `claude-code` with `sonnet`. Prefer a different vendor from the arms (this repo's own suite judges Claude arms with Codex). |
+
+Credentials can live in a repo-root `.env`. A graded run can touch up to three
+vendors: the agent's, Gemini for the binder, and the judge's. `lint` is free;
+`analyze` and `run` spend API calls, and `run` also boots VMs. Preflight lists
+every missing piece and exits before anything is spent.
+
+## How a run works
+
+```mermaid
+flowchart LR
+    E["greets-by-name.eval.md<br/>prompt + assertions"] --> A1["arm: baseline<br/>fresh microVM,<br/>setup.sh installs nothing"]
+    E --> A2["arm: trial<br/>fresh microVM,<br/>setup.sh installs the hello skill"]
+    A1 --> F1["facts: files, SHAs,<br/>final message, tool calls"]
+    A2 --> F2["facts"]
+    F1 --> G["binder: deterministic checkers<br/>everything else: LLM judge"]
+    F2 --> G
+    G --> R["benchmark.md + benchmark.json<br/>meta.json + index.jsonl"]
+```
+
+Each `(eval × arm)` pair is one parametrized pytest test. A cell:
+
+1. **Boots a microVM** from a cached snapshot with the agent CLI already
+   installed. The first run builds the snapshot (a few minutes); later runs
+   reuse it, or pay the cost up front with `benchspec sandbox:build`.
+2. **Seeds the clean room** — the eval's optional `workspace/` files land in a
+   fresh directory mounted at `/workspace`, the agent's working directory.
+3. **Runs `setup.sh`**, where arms diverge: it sees `$BENCHSPEC_ARM`, so the
+   baseline branch exits early and the trial branch copies the skill into place.
+4. **Invokes the agent** on the eval's prompt.
+5. **Collects the facts** — file tree, contents, SHA-256s, the final message,
+   the tool calls.
+6. **Grades** — the binder maps each assertion to a deterministic checker where
+   it can do so without risk; the judge grades everything else from the
+   collected evidence alone.
+
+Two guarantees hold throughout. Nothing in the guest can write back to your
+checkout: `setup.sh` reaches the skill under test through a read-only staged
+copy of your repo at `/project` (what a `git clone` would contain — never
+`.env`, `.git`, or earlier runs' artifacts). And provider credentials are
+injected at the network boundary, never as readable environment variables in
+the guest.
+
+`benchspec run` is pytest underneath, and everything after `--` goes to
+pytest verbatim: `benchspec run -- -k greets-by-name` (equivalently
+`pytest -k greets-by-name`) runs one eval, `-n 8` fans cells across eight
+microVMs, and `--count 5` samples each cell five times so the report can flag a
+delta that sits within noise.
+
+## Why benchspec
+
+- **Comparison is first-class.** A single pass rate is a number without a
+  reference point. Arms and a baseline make the headline a delta; skip the
+  baseline when absolute rates are what you want.
+- **Deterministic where possible, judged where necessary.** The binder is tuned
+  so a false positive, a surface check passing on wrong output, is the one
+  unacceptable error; anything doubtful goes to the judge, which sees the
+  collected evidence and never grades from recall.
+- **Self-describing artifacts.** Every run writes `meta.json` (planned config
+  plus observed provenance, down to the agent version inside the guest),
+  `index.jsonl` (one row per sample), and `benchmark.json`, so other tools can
+  aggregate runs without knowing the directory layout.
+
+## Documentation
+
+| | |
+|---|---|
+| [`docs/quickstart.md`](docs/quickstart.md) | Empty directory to a graded two-arm run. |
+| [`docs/concepts.md`](docs/concepts.md) | The vocabulary: eval, arm, set, baseline, binder, judge. |
+| [`docs/writing-evals.md`](docs/writing-evals.md) | The eval format, workspaces, `setup.sh`, and how grading decides what binds. |
+| [`docs/configuration.md`](docs/configuration.md) | Sets, arms, the judge, every CLI flag, exit codes. |
+| [`docs/sandbox.md`](docs/sandbox.md) | Snapshots, mounts, credentials, host requirements. |
+| [`docs/results.md`](docs/results.md) | Reading `benchmark.md` and the machine-readable artifacts. |
+| [`docs/harnesses.md`](docs/harnesses.md) | `claude-code`, `codex`, `opencode`, and adding your own. |
+
+## Contributing
+
+Issues and pull requests are welcome at
+[github.com/theycallmeswift/benchspec](https://github.com/theycallmeswift/benchspec).
+Until `CONTRIBUTING.md` lands, [`docs/style/development.md`](docs/style/development.md)
+is the code style, and `make test` plus `make lint` are the bar.
+
+## License
+
+[MIT](LICENSE).

benchspec-0.0.1/README.md

@@ -0,0 +1,178 @@
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="docs/assets/benchspec-wordmark-dark.svg">
+  <img alt="benchspec" src="docs/assets/benchspec-wordmark-light.svg" width="188" height="48">
+</picture>
+
+**Benchmark what your agent does, not what it says.**
+
+[![CI](https://github.com/theycallmeswift/benchspec/actions/workflows/ci.yml/badge.svg)](https://github.com/theycallmeswift/benchspec/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/benchspec)](https://pypi.org/project/benchspec/)
+[![Python](https://img.shields.io/pypi/pyversions/benchspec)](https://pypi.org/project/benchspec/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
+benchspec runs an agent (Claude Code, Codex, or OpenCode) against a task
+in a fresh microVM, checks what it actually did in the workspace, and reports the
+result as a comparison: with your skill versus without, one model versus another,
+one harness versus another.
+
+<img src="docs/assets/benchmark-terminal.gif" width="800" alt="Animated terminal output: benchspec run prints a benchmark matrix with evals as rows, arms as columns, color-coded rates, and percentage-point deltas">
+
+*End-of-run summary for a two-arm run of the in-repo [`hello`](evals/e2e/hello/)
+suite (illustrative numbers).* Rows are evals, columns are arms (`baseline` ran
+the agent bare, `trial` installed the skill), and every non-baseline cell shows
+its assertion pass rate plus the delta against the baseline in percentage
+points. The same matrix lands in `benchmark.md`, with machine-readable artifacts
+alongside.
+
+Teams pick harnesses, models, and prompts by anecdote: run it once, eyeball the
+transcript, trust the vibe. benchspec turns that guess into a measurement.
+Write the goal once, run it across the configurations you care about, and read
+off — in percentage points — how good each one actually is at accomplishing it.
+
+## Getting Started
+
+```bash
+pip install "benchspec[microsandbox]"
+```
+
+> **Pre-1.0.** The eval format and the artifact schemas are the surfaces most
+> likely to change. microsandbox is the only sandbox backend today; `docker` is
+> recognized in config but fails fast as not implemented.
+
+An eval is one Markdown file: a prompt, then a checklist of plain-prose claims
+about the workspace after the agent is done. There is no checker syntax to learn;
+the wording is the spec. `evals/hello/greets-by-name.eval.md`:
+
+```markdown
+---
+---
+
+## Prompt
+
+You are working in a workspace rooted at your current working directory.
+Greet Alice by name.
+
+## Assertions
+
+- [ ] ./Greetings/Alice.md contains the exact line 'Hello, Alice!'
+- [ ] Skill `hello` invoked
+- [ ] The greeting feels warm and personable, not curt or robotic
+```
+
+The benchmark is a block in `pyproject.toml`. Arms are the report columns; the
+baseline is what the others are measured against:
+
+```toml
+[tool.benchspec]
+default-set = "default"
+
+[tool.benchspec.sets.default]
+harness  = "claude-code"
+model    = "sonnet"
+baseline = "baseline"
+arms = [
+  { name = "baseline" },   # installs nothing
+  { name = "trial" },      # setup.sh installs the skill
+]
+```
+
+Then:
+
+```bash
+benchspec lint      # static checks on the assertions
+benchspec analyze   # which assertions grade deterministically, which go to the judge
+benchspec run       # every (eval × arm) in its own microVM, graded, reported
+```
+
+## What you need
+
+| | |
+|---|---|
+| Platform | Apple Silicon Mac, or Linux with `/dev/kvm`. Python 3.11+. |
+| Agent CLI | `claude`, `codex`, or `opencode` on `PATH`, with its credential (for Claude Code, `CLAUDE_CODE_OAUTH_TOKEN` or `ANTHROPIC_API_KEY`). |
+| `GEMINI_API_KEY` | The binder: a fixed Gemini call that classifies each assertion. Required for every `analyze` and `run`. |
+| Judge credential | The judge runs on the host through an agent CLI; the default is `claude-code` with `sonnet`. Prefer a different vendor from the arms (this repo's own suite judges Claude arms with Codex). |
+
+Credentials can live in a repo-root `.env`. A graded run can touch up to three
+vendors: the agent's, Gemini for the binder, and the judge's. `lint` is free;
+`analyze` and `run` spend API calls, and `run` also boots VMs. Preflight lists
+every missing piece and exits before anything is spent.
+
+## How a run works
+
+```mermaid
+flowchart LR
+    E["greets-by-name.eval.md<br/>prompt + assertions"] --> A1["arm: baseline<br/>fresh microVM,<br/>setup.sh installs nothing"]
+    E --> A2["arm: trial<br/>fresh microVM,<br/>setup.sh installs the hello skill"]
+    A1 --> F1["facts: files, SHAs,<br/>final message, tool calls"]
+    A2 --> F2["facts"]
+    F1 --> G["binder: deterministic checkers<br/>everything else: LLM judge"]
+    F2 --> G
+    G --> R["benchmark.md + benchmark.json<br/>meta.json + index.jsonl"]
+```
+
+Each `(eval × arm)` pair is one parametrized pytest test. A cell:
+
+1. **Boots a microVM** from a cached snapshot with the agent CLI already
+   installed. The first run builds the snapshot (a few minutes); later runs
+   reuse it, or pay the cost up front with `benchspec sandbox:build`.
+2. **Seeds the clean room** — the eval's optional `workspace/` files land in a
+   fresh directory mounted at `/workspace`, the agent's working directory.
+3. **Runs `setup.sh`**, where arms diverge: it sees `$BENCHSPEC_ARM`, so the
+   baseline branch exits early and the trial branch copies the skill into place.
+4. **Invokes the agent** on the eval's prompt.
+5. **Collects the facts** — file tree, contents, SHA-256s, the final message,
+   the tool calls.
+6. **Grades** — the binder maps each assertion to a deterministic checker where
+   it can do so without risk; the judge grades everything else from the
+   collected evidence alone.
+
+Two guarantees hold throughout. Nothing in the guest can write back to your
+checkout: `setup.sh` reaches the skill under test through a read-only staged
+copy of your repo at `/project` (what a `git clone` would contain — never
+`.env`, `.git`, or earlier runs' artifacts). And provider credentials are
+injected at the network boundary, never as readable environment variables in
+the guest.
+
+`benchspec run` is pytest underneath, and everything after `--` goes to
+pytest verbatim: `benchspec run -- -k greets-by-name` (equivalently
+`pytest -k greets-by-name`) runs one eval, `-n 8` fans cells across eight
+microVMs, and `--count 5` samples each cell five times so the report can flag a
+delta that sits within noise.
+
+## Why benchspec
+
+- **Comparison is first-class.** A single pass rate is a number without a
+  reference point. Arms and a baseline make the headline a delta; skip the
+  baseline when absolute rates are what you want.
+- **Deterministic where possible, judged where necessary.** The binder is tuned
+  so a false positive, a surface check passing on wrong output, is the one
+  unacceptable error; anything doubtful goes to the judge, which sees the
+  collected evidence and never grades from recall.
+- **Self-describing artifacts.** Every run writes `meta.json` (planned config
+  plus observed provenance, down to the agent version inside the guest),
+  `index.jsonl` (one row per sample), and `benchmark.json`, so other tools can
+  aggregate runs without knowing the directory layout.
+
+## Documentation
+
+| | |
+|---|---|
+| [`docs/quickstart.md`](docs/quickstart.md) | Empty directory to a graded two-arm run. |
+| [`docs/concepts.md`](docs/concepts.md) | The vocabulary: eval, arm, set, baseline, binder, judge. |
+| [`docs/writing-evals.md`](docs/writing-evals.md) | The eval format, workspaces, `setup.sh`, and how grading decides what binds. |
+| [`docs/configuration.md`](docs/configuration.md) | Sets, arms, the judge, every CLI flag, exit codes. |
+| [`docs/sandbox.md`](docs/sandbox.md) | Snapshots, mounts, credentials, host requirements. |
+| [`docs/results.md`](docs/results.md) | Reading `benchmark.md` and the machine-readable artifacts. |
+| [`docs/harnesses.md`](docs/harnesses.md) | `claude-code`, `codex`, `opencode`, and adding your own. |
+
+## Contributing
+
+Issues and pull requests are welcome at
+[github.com/theycallmeswift/benchspec](https://github.com/theycallmeswift/benchspec).
+Until `CONTRIBUTING.md` lands, [`docs/style/development.md`](docs/style/development.md)
+is the code style, and `make test` plus `make lint` are the bar.
+
+## License
+
+[MIT](LICENSE).

benchspec-0.0.1/docs/concepts.md

@@ -0,0 +1,174 @@
+# Concepts
+
+This page is the vocabulary: one short paragraph per term, alphabetized, each
+with a one-line example and a link to the document that goes deepest. Skim the
+index, or come back when a term in another page is unfamiliar.
+
+[Arm](#arm) ·
+[Assertion](#assertion) ·
+[Baseline](#baseline) ·
+[Binder](#binder) ·
+[Cell](#cell) ·
+[Checker](#checker) ·
+[Clean room and /workspace](#clean-room-and-workspace) ·
+[Errored versus failed](#errored-versus-failed) ·
+[Eval](#eval) ·
+[Eval set](#eval-set) ·
+[Group](#group) ·
+[Harness](#harness) ·
+[Iteration](#iteration) ·
+[Judge](#judge) ·
+[/project](#project) ·
+[Sample](#sample) ·
+[Sandbox and snapshot](#sandbox-and-snapshot) ·
+[setup.sh](#setupsh) ·
+[Skill](#skill)
+
+## Arm
+
+One column of the benchmark: a named configuration of harness, model, effort,
+environment variables, and pass-through CLI arguments. Arms differ from each
+other only in that configuration and in what their `setup.sh` branch installs.
+Example: `{ name = "trial", model = "opus", effort = "high" }`. Reference:
+[configuration.md](configuration.md).
+
+## Assertion
+
+One `- [ ]` line in an eval: a single claim about the final workspace, the
+agent's final message, or what the agent did. Assertions are prose, not checker
+syntax; how each one is graded is decided at run time (see **binder**).
+Example: `- [ ] ./Greetings/Alice.md contains the exact line 'Hello, Alice!'`.
+
+## Baseline
+
+The arm every other arm's delta is measured against. With a baseline, the report
+headline is a percentage-point delta (`+14pp`); without one, arms report absolute
+pass rates. Example: `baseline = "baseline"`, an arm whose `setup.sh` installs
+nothing. Reading deltas: [results.md](results.md).
+
+## Binder
+
+A fixed, deliberately conservative classifier (one call to
+`gemini-3.5-flash-lite` per assertion, which is why `GEMINI_API_KEY` is always
+required) that decides, at grade time, whether an assertion can be checked
+mechanically. It either **binds** the line to one checker or **punts** it to the
+judge. It never grades anything itself. Reference:
+[writing-evals.md](writing-evals.md#how-assertions-are-graded).
+
+## Cell
+
+One `(eval × arm)` pair. Each cell becomes one parametrized pytest test that
+boots its own microVM, runs the agent, and grades the result. A two-eval,
+three-arm set has six cells. Example: `test_eval[hello-greets-by-name-trial]`.
+
+## Checker
+
+One deterministic check run on the host against the final workspace or the
+run's process facts: `file_exists`, `not_file_exists`, `glob_count`, `regex`,
+`frontmatter_has`, `sha256_match`, `skill_invoked`, `not_skill_invoked`. Zero
+variance, zero judge cost. Example: "./out/report.md exists" binds to
+`file_exists`.
+
+## Clean room and `/workspace`
+
+The clean room is a fresh temporary directory on the host, seeded from the
+eval's `workspace/` folder (or empty), that is mounted read-write into the
+microVM at `/workspace`. It is the agent's working directory, and the host
+grades the same directory afterward. Every path in an eval is written
+`./`-relative to it. Example: an eval with `workspace/request.md` starts the
+agent in a directory containing exactly `request.md`.
+
+## Errored versus failed
+
+A **failed** assertion is a measurement: the agent ran and the claim did not
+hold. An **errored** sample is infrastructure: the CLI crashed or timed out,
+`setup.sh` exited non-zero, or the judge failed at the transport level. Errored
+samples are excluded from pass rates but counted in the report, so a
+half-crashed run cannot read like a clean one. See
+[results.md](results.md#noise-samples-and-flakiness).
+
+## Eval
+
+One task and what success looks like: a prompt the agent receives, an optional
+`history:` of prior turns, and a checklist of plain-prose **assertions**. An eval
+is one Markdown file, `eval.md` or `<stem>.eval.md`, and its id is the folder
+name or the file stem. Example: `evals/hello/greets-by-name.eval.md` asks the
+agent to "Greet Alice by name" and asserts that `./Greetings/Alice.md` contains
+`Hello, Alice!`. Format reference: [writing-evals.md](writing-evals.md).
+
+## Eval set
+
+A named benchmark declared in `pyproject.toml`: its arms, the defaults they
+inherit, and an optional baseline. One run resolves exactly one set. Example:
+`[tool.benchspec.sets.default]` with arms `baseline` and `trial`. Reference:
+[configuration.md](configuration.md#eval-sets).
+
+## Group
+
+The folder an eval file sits in. Its name is the first half of an eval's
+identity, `(group, eval_id)`, and the artifact tree is keyed on it. Sibling
+`<stem>.eval.md` files in one folder share that folder's `workspace/` and
+`setup.sh`. Example: `evals/hello/greets-by-name.eval.md` is eval
+`greets-by-name` in group `hello`, and pytest names its cells
+`test_eval[hello-greets-by-name-<arm>]`.
+
+## Harness
+
+The agent CLI under test: `claude-code`, `codex`, or `opencode`. Each is
+one adapter that knows how to install the CLI into a microVM, which credential it
+needs, how to run it headless, and how to read its output stream. Chosen per arm.
+Details and how to add one: [harnesses.md](harnesses.md).
+
+## Iteration
+
+One run's artifact tree, `tmp/evals/iteration_NN/`, numbered once per run and
+shared by every cell in it. It holds `benchmark.md`, `benchmark.json`,
+`meta.json`, `index.jsonl`, and every sample's per-cell files. Reference:
+[results.md](results.md).
+
+## Judge
+
+The LLM that grades every punted assertion, from evidence only: the workspace
+tree, file contents and SHA-256s, the agent's final message, and the tools and
+skills it invoked. The judge runs on the host through one of the same harness
+adapters, is configured once per run, and is independent of the arms. Example:
+`[tool.benchspec.judge] harness = "codex"`. Reference:
+[configuration.md](configuration.md#the-judge).
+
+## `/project`
+
+A read-only mount of a staged copy of your repository (what a `git clone` would
+contain, minus `.env` files, `.git`, and `tmp/`). It exists so the eval's own
+`setup.sh` can copy the skill under test into the guest. The agent can read it
+too, so keep prompts pointed at `./`. Details:
+[sandbox.md](sandbox.md#what-project-contains).
+
+## Sample
+
+One execution of a cell. A plain run takes one sample per cell (`sample-0/`);
+`--count N` takes N, so the report can show flakiness and a noise band on each
+delta. Example: `benchspec run -- --count 5`. See
+[results.md](results.md#noise-samples-and-flakiness).
+
+## Sandbox and snapshot
+
+Every cell runs inside a microVM, a small hardware-isolated virtual machine
+booted from a **snapshot**: a sealed image with the base OS, the harness CLI,
+and any suite-wide tools already installed. Snapshots build once per
+configuration and are cached; cells boot from them in seconds. Reference:
+[sandbox.md](sandbox.md).
+
+## `setup.sh`
+
+An optional script beside the eval file that runs inside the microVM before the
+prompt, with `BENCHSPEC_ARM` set to the arm's name. It is the one place arms
+diverge: the canonical script installs a skill on `trial` and exits early on
+`baseline`. Reference:
+[writing-evals.md](writing-evals.md#setupsh-what-differs-per-arm).
+
+## Skill
+
+An instruction file (`SKILL.md`) an agent can load and dispatch; in the common
+"capability lift" benchmark, the skill is the thing under test. Skills install to
+the fixed guest path `/home/benchspec/skills`, which every harness's native
+skill directory links to. Example assertion: `` - [ ] Skill `hello` invoked ``.