static-fire 0.0.0-stage → 0.44.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Kenneth Liao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,231 @@
-# Temporary Holding Version
+# Static Fire (pre-release)
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Find out which **agent configuration** (harness + model + effort + apkit Profile) does your real work best,
+and what it costs. Same tasks, same graders, same judge for every config; the result is an
+Artificial Analysis-style score-vs-cost chart, the **Workflow Index**, for *your* workflows.
+
+**The whole point is real-world performance.** A trial reproduces the real job as closely as it can: the
+inputs, tools, context and instructions you would actually give the agent, and graders that read the
+output the way its real readers would. A score is only worth something if it predicts how the config does
+on your actual work. Every place a trial differs from the real job is a known gap, and a candidate to remove.
+The engine's gaps are listed below; the gaps of one task go in its project's README.
+
+## Quick start
+
+You need Node or Bun. The package brings its own Bun ([ADR-0008](docs/adr/0008-npm-package-brings-bun.md)).
+
+```sh
+mkdir my-evals && cd my-evals
+npx static-fire init                       # make a project in this folder
+```
+
+Then open your agent in that folder and tell it what you want to test, for example "I want to find out which
+agent is the best at making thumbnails". `init` writes an `AGENTS.md` that sends the agent to
+`static-fire guide`, which takes it from your goal to a dashboard with data. The guide also holds the file
+formats, so neither of you needs to read the engine source.
+
+A **project** is the folder that holds `static-fire.yaml`. Every command works on the project it finds by
+walking up from the current folder, so you can run it from any subfolder. The commands, with
+`npx static-fire` in place of `static-fire` when it is not installed:
+
+```sh
+static-fire init [--json]                  # make a project in the current folder; never overwrites a file
+static-fire guide                          # the guide for agents: from a goal to a dashboard, and the file formats
+static-fire new task <id>                  # a task folder with a commented task.yaml
+static-fire configs | judges | suites | tasks
+static-fire run --suites <suite> | --tasks <task>  --configs <config>,<config>|all \
+                [--trials 3] [--judge <judge>] [--votes 3] [--concurrency 4]   # one batch
+static-fire rerun <batch>                  # its failed trials again, as new trials (after a usage limit, for example)
+static-fire regrade [--tasks t] [--configs a,b] [--batch id] [--judge id] [--votes n] [--only-invalid]
+                                           # judge calls only: trials with no valid grading under the current grading hash
+static-fire show [--suites s | --tasks t] [--judge id] [--grading hash] [--since date] [--until date] [--json]
+                                           # the comparable trials, pooled across batches (JSON for agents)
+static-fire serve                          # dashboard at http://localhost:4477
+static-fire list                           # batches, newest first, with their progress
+static-fire export [path]                  # every record as JSON, for backup
+static-fire migrate                        # one-off import of an experiment folder from before 0.25.0
+```
+
+## How it works
+
+Every trial is a record; a chart is a query over them. Running one new config on one task adds its point to
+the existing chart, next to every earlier trial that is comparable: the same task hash, and a valid grading
+under the same grading hash. The dashboard has three pages. **Results** (`/`) shows the chart, or the same
+numbers as a table. Its scope chips and **Filter** menu (harness, model, effort, Profile, judge, grading
+version, date range) are in the page URL, so a view can be bookmarked. Harness, model, effort and Profile
+each take several values, for example `?model=opus&model=haiku`. When a question or the judge changes,
+earlier trials are left out of the view, and the notice under the chart offers to regrade them with judge
+calls only. **Runs** (`/runs`) lists every batch with its trials; start one from **New run** there. Runs
+started from the CLI appear live, and the run status in the top bar shows what is running. The **Compare** page (`/compare`) shows what changed between two
+grading versions of a task, and what that did to the scores and to each criterion. A suite's description is shown with
+every chart that draws the suite.
+Terms are defined in [`CONTEXT.md`](CONTEXT.md); the scoring design is
+[ADR-0001](docs/adr/0001-pass-fail-criteria-in-measures.md), and trials as records is
+[ADR-0006](docs/adr/0006-trials-as-records-charts-as-queries.md).
+
+## Develop the engine
+
+This repo is the engine. It holds no project: tests use the made-up project in `tests/fixtures/sample-project/`.
+
+```sh
+bun install
+bun test
+bun run typecheck
+cd <a project> && bun <this repo>/src/cli.ts <command>   # the engine from this checkout, on that project
+```
+
+## Planned work
+
+Planned work is tracked in [GitHub issues](https://github.com/kenneth-liao/static-fire/issues). A design
+written before the work starts is an issue labelled `spec`; decisions that stick are recorded as ADRs in
+[`docs/adr/`](docs/adr/).
+
+## How scoring works
+
+Every criterion is a **yes/no question**, decided by a deterministic **check** or an LLM **judge**.
+
+```
+Suite (e.g. Writing)                  Suite Index    = mean task score
+ └ Task (e.g. weekly-newsletter)      Task score     = 0 if a gate failed, else mean of measures
+    ├ Gates                           (output exists, frontmatter has a title, …)
+    └ Measure (e.g. "Gets opened")    Measure score  = % of its criteria passed
+       └ Criterion (yes/no, floor | strong | ceiling)
+Workflow Index = mean Suite Index over the Suites in view
+```
+
+- Judge criteria get several **votes** (3 by default). A criterion's score is its share of passing votes.
+  Votes are adaptive: a majority runs first (2 of 3), and the rest run only if it is split on any criterion
+  in the batch or a vote failed.
+  Judge criteria that need the same evidence (the output alone, or the output plus the inputs) share one
+  call per vote, across measures, with the order shuffled; `isolate: true` gives a criterion its own call
+  (ADR-0003). The judge writes each reason before its pass/fail. Answers must match a JSON Schema; one
+  retry, then an **error**.
+- A judge criterion can get **derived evidence**: an image a script makes from the output (see below).
+- A grader error makes the grading **invalid** (the trial is left out of the means until regraded), never a fail.
+- Points show the mean of valid trials with a **95% confidence interval**, pooled across batches, with n and
+  the date range. A human config counts only its latest trial per task. A config missing a task of a Suite
+  gets no Suite Index for it: its row reads "partial (2 of 3 tasks)".
+- **Dead criteria** (a strong/ceiling criterion every config passed, or any criterion every config failed)
+  are flagged: they carry no signal.
+
+### Derived evidence
+
+A judge criterion can name a script in the task's `grader/` folder. Before the judge is asked, the script
+makes an image from the output, and the judge gets that image as evidence, attached like any other image (by
+path for a judge that cannot take attached images). Use it to show the judge what the real reader sees, for
+example a thumbnail scaled to 320 px wide.
+
+```yaml
+- id: small-text-readable
+  question: Can every word of the text be read?
+  level: floor
+  grader: judge
+  derived: scale-320.py   # a path under evals/<task>/grader/
+  evidence: []            # optional: the judge sees only the derived image, not the full-size output
+```
+
+The contract for a script:
+
+- **Name:** `derived:` is a path relative to the task's `grader/` folder, one script per criterion. Task
+  validation reports a script that does not exist, and `derived` on a check or human criterion.
+- **How it runs:** by extension. `.ts` and `.js` with the Bun that runs the engine (so a Node-only install
+  needs no `bun` on the `PATH`), `.py` with `uv run --script` (declare packages such as Pillow in the script's
+  inline `# /// script` header), `.sh` with `sh`.
+- **Input:** argument 1 and `$OUTPUT` are the trial's output file. `$TASK_DIR` is the task folder, so a script
+  can read other task files, for example `$TASK_DIR/project/youtube/thumbnail-regions.json`. `$WORKSPACE` is
+  the trial workspace, which is also the working folder. The time limit is 60 s.
+- **Output:** write one PNG to the path in argument 2 (also `$DERIVED`). A non-zero exit, a time-out, no file
+  or a file that is not a PNG makes the criterion's verdict an error, so the grading is invalid.
+- **Where it stays:** `runs/trials/<trial-id>/gradings/<grading-id>/_derived/<script>.png`, with the script's
+  output next to it as `.log`. A script at the top of `grader/` keeps its name (`scale-320.py.png`); in the file
+  name, `/` is written as `%2F` and any other character outside letters, digits and `+ = . _ -` as `%` and its
+  hex code, so two scripts never share a file (`sub/overlay.ts` gives `sub%2Foverlay.ts.png`).
+  This is the same for a run and a regrade.
+- **Hashing:** the script name is part of the criterion in the grading hash, and every file in `grader/` is
+  hashed, so a new name or new script content gives a new grading version. Criteria that name the same script
+  share the image and, when their other evidence is the same, one judge call.
+
+### Engine revision
+
+A task gets a new grading version only when something that can move its score changed. That includes Static
+Fire's own grading code, the **engine revision**, which is a set of parts. A grading hash holds a part only
+when the task uses it:
+
+| Part | In a task's grading hash when |
+|---|---|
+| Judge instructions (a hash of the wording and layout) | The task has a judge criterion. The text for derived evidence counts only with derived evidence. |
+| Check, one revision per type (`regex`, `words`, `frontmatter`, `command`) | A gate or criterion uses the type. `command` always: the output gate is one. |
+| Derived evidence | A criterion names a derived evidence script |
+| Judge call, one revision per judge harness | The task has a judge criterion; its own judge's harness only |
+| Judging (how evidence is shown, how criteria share calls, how votes are counted) | The task has a judge criterion |
+| Scoring (how verdicts become a score) | Always |
+
+So a change to the `words` check leaves a thumbnail task as it is, and a change to the Codex judge call leaves
+every grading by a Claude Code judge as it is. A grading version name says which part changed, for example
+"Check changed: words" or "Judge instructions changed".
+
+For a change to the grading code: `bun test` fails on the engine pin (`tests/identity.test.ts`) and names the
+piece of code that changed and the revision to raise in `src/core/engine.ts`. Raise it only when the change can
+move a score. The judge instructions need no revision: they are hashed.
+
+An update to 0.38.0 changes the record store once (schema version 3) and an older Static Fire cannot open it
+after that. Before the update, stop every running Static Fire process (the dashboard, a batch) and copy the
+store: `sqlite3 runs/static-fire.db ".backup '<dest>'"`.
+
+## Files
+
+Every path here is in the project folder, the folder that holds `static-fire.yaml`. `static-fire init` makes
+one, and `static-fire guide` gives the format of each file.
+
+| Thing | Where | What |
+|---|---|---|
+| Settings | `static-fire.yaml` | Marks the project folder. The tools that trials may not use, per harness |
+| Agent config | `configs/<id>.yaml` | `harness` (claude · codex · agy · pi), `model`, `effort`, optional `profile` (apkit); the label is built from these |
+| Human config | `configs/<id>.yaml` | `harness: human` and `outputs` (task id → the file you wrote): compete with the agents |
+| Judge config | `judges.yaml` | Judges to choose from per run or regrade (first = default) |
+| Suite | `suites/<id>.yaml` | Title, description (shown with the suite's charts) + list of task IDs |
+| Task | `evals/<id>/task.yaml` | Prompt, inputs, project files, references and sources (graders only), gates, measures, criteria |
+| Records | `runs/static-fire.db` (`STATIC_FIRE_DB` overrides it) | The SQLite record store, the only copy of the records: batches, trials with their run conditions, gradings, verdicts, task versions, grading versions, user setups |
+| Trial artifacts | `runs/trials/<trial-id>/` | `workspace/` (the agent's output and files), `agent.log`, and `gradings/<grading-id>/` with the judge logs |
+| Profile cache | `.cache/` | The apkit Profiles of a batch and their lock file. Made again for each batch |
+| Prices | `pricing.yaml` (optional) | Your own token prices. Without it, the prices that ship with the engine are used |
+| Old experiments | `runs/<id>/experiment.json` | The format before 0.25.0. Only `static-fire migrate` reads it; delete the folders by hand once the dashboard shows the imported data |
+
+## How a trial runs
+
+1. Fresh workspace in the system temp folder: the task's `inputs/` (notes, screenshots), its `project/`
+   files (what the agent would find in the real project, such as past issues), plus a snapshot of the apkit
+   Profile if the config has one (installed once per batch, behind the project's `.cache/apkit.lock`, with `apkit install … --agents <agent> --yes`,
+   then `apkit remove`). It is outside any repo, so the task's answer key is not in reach on disk.
+2. The harness runs the way you run it: headless, in its no-prompt mode, with **your own user-level setup**
+   (instructions, settings, skills, MCP servers) and every tool, including the web:
+   `claude -p --dangerously-skip-permissions`, `codex exec --dangerously-bypass-approvals-and-sandbox`,
+   `agy -p --dangerously-skip-permissions`, `pi -p`. The only limit: tools that act on outside services
+   (email, Slack, your newsletter platform, paid infrastructure) are blocked, because in real work you
+   would be asked first. Edit the list in the project's `static-fire.yaml`. Each trial records its run conditions: the
+   user setup hash (the file list is stored once per setup), Profile hash, blocked tools, harness version
+   and start time. They are shown, never part of comparability.
+3. If the task is built from published work (`heldOut`), an output that copies the original is invalid.
+4. Gates and checks run; judge criteria are voted on by the run's judge, and the grading is saved under its
+   grading hash. Judges do not use your setup: they run with user-level config kept out, so grading stays
+   the same across runs.
+5. The workspace and logs are copied into the project's `runs/trials/<trial-id>/`.
+
+A human config skips steps 1–2: its file for the task is copied in and graded the same way. To grade a
+draft while you write it, run `static-fire run --tasks <task> --configs <human config>` (judge calls only).
+
+## Known gaps (pre-release)
+
+These are gaps of the engine. A gap of one task, such as inputs that are cleaner than the real ones, goes in
+its project's README.
+
+- **Pi** has no user-config switch and no JSON Schema output, so it can be a candidate but not a judge.
+- **Costs** for Codex and Antigravity are estimated from tokens × `pricing.yaml` (Gemini 4 Argon's rate
+  doubles on 2026-12-01). Claude Code and Pi report their own. Judge cost is not part of a config's cost.
+  Cost that is not in the harness's own report, such as image generation by an outside service, is not counted.
+- **Human grader** (for taste criteria) is planned but not built yet.
+- **Copy check** (`heldOut`) compares text only. An image output that copies a published original is not
+  marked invalid.
+- **Realism gaps** (see the rule at the top): a trial is one shot with no feedback and no one to answer
+  questions. Agents can reach the web, and no harness can block one site at run time, so a copied original
+  is caught after the run instead.

package/bin/static-fire.js

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+// Starts the engine (src/cli.ts) with Bun, so `npx static-fire` works with Node alone (docs/adr/0008).
+// Under Bun it uses the Bun that runs it. Under Node it uses the Bun the `bun` dependency installed.
+// The arguments and the current folder pass through unchanged.
+import { spawn } from "node:child_process";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const cli = join(here, "..", "src", "cli.ts");
+
+// The `bun` package's bin is a placeholder until its postinstall runs (not with --ignore-scripts).
+// Its platform package holds the same binary either way.
+function bunFromDependency() {
+  const require = createRequire(import.meta.url);
+  let dir;
+  try { dir = dirname(require.resolve("bun/package.json")); } catch { return null; }
+  const exe = process.platform === "win32" ? "bun.exe" : "bun";
+  const installed = join(dir, "bin", "bun.exe");
+  if (existsSync(installed) && !(statSync(installed).size < 4096 && readFileSync(installed, "utf8").includes("postinstall"))) return installed;
+  const arch = process.arch === "arm64" ? "aarch64" : process.arch;
+  try {
+    const platform = dirname(createRequire(join(dir, "package.json")).resolve(`@oven/bun-${process.platform}-${arch}/package.json`));
+    if (existsSync(join(platform, "bin", exe))) return join(platform, "bin", exe);
+  } catch {}
+  return null;
+}
+
+const bun = process.versions.bun ? process.execPath : bunFromDependency();
+if (!bun) {
+  console.error("static-fire: Bun was not found. The `bun` package should bring it: run `npm install` again without --ignore-scripts, or install Bun from https://bun.sh.");
+  process.exit(1);
+}
+
+const child = spawn(bun, [cli, ...process.argv.slice(2)], { stdio: "inherit" });
+const signals = ["SIGINT", "SIGTERM", "SIGHUP"];
+for (const s of signals) process.on(s, () => child.kill(s));
+child.on("error", (e) => { console.error(`static-fire: could not start Bun (${bun}): ${e.message}`); process.exit(1); });
+child.on("exit", (code, signal) => {
+  if (signal) {
+    for (const s of signals) process.removeAllListeners(s);
+    process.kill(process.pid, signal);
+  } else process.exit(code ?? 1);
+});

package/package.json

@@ -1,6 +1,31 @@
 {
   "name": "static-fire",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.44.0",
+  "type": "module",
+  "description": "Benchmark agent configurations (harness + model + effort + profile) on your real workflows.",
+  "license": "MIT",
+  "bin": {
+    "static-fire": "bin/static-fire.js"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "web/",
+    "pricing.yaml"
+  ],
+  "scripts": {
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "bun": "^1.4.4",
+    "marked": "^15.0.0",
+    "react": "^19.1.0",
+    "react-dom": "^19.1.0"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.2.0",
+    "@types/react": "^19.1.0",
+    "@types/react-dom": "^19.1.0",
+    "typescript": "^5.8.0"
+  }
+}

package/pricing.yaml

@@ -0,0 +1,20 @@
+# API-equivalent rates (USD per million tokens) for harnesses that don't report their own cost.
+# Claude Code and Pi report cost themselves; Codex and agy only report tokens, so they are priced here.
+# Verify before relying on them.
+codex:
+  # Third-party trackers citing OpenAI's pricing page, checked 2026-10-28.
+  gpt-6-luna:   { input: 0.10, cachedInput: 0.01, output: 0.50 }
+  gpt-6.1-sol:  { input: 2.00, cachedInput: 0.10, output: 10.00 }
+  gpt-5.6-luna: { input: 0.20, cachedInput: 0.02, output: 1.20 }
+  gpt-5.6-sol:  { input: 5.00, cachedInput: 0.50, output: 30.00 }
+agy:
+  # Gemini API standard tier, checked 2026-11-04 (therundown.ai, pricepertoken.com, lmspedia.org).
+  # Introductory rate through 2026-11-30; from 2026-12-01 it is input 4.00, cachedInput 0.20, output 20.00.
+  gemini-4-argon-low:    { input: 2.00, cachedInput: 0.10, output: 10.00 }
+  gemini-4-argon-medium: { input: 2.00, cachedInput: 0.10, output: 10.00 }
+  gemini-4-argon-high:   { input: 2.00, cachedInput: 0.10, output: 10.00 }
+  # Launched 2026-11-12; third-party trackers (benchleader.com, aipricing.guru, datastudios.org), checked 2026-11-13.
+  gemini-4-hydrogen-low:    { input: 1.50, cachedInput: 0.075, output: 7.50 }
+  gemini-4-hydrogen-medium: { input: 1.50, cachedInput: 0.075, output: 7.50 }
+  gemini-4-hydrogen-high:   { input: 1.50, cachedInput: 0.075, output: 7.50 }
+  gemini-4-hydrogen-xhigh:  { input: 1.50, cachedInput: 0.075, output: 7.50 }

package/src/adapters/index.ts

@@ -0,0 +1,153 @@
+// Adapter contract: run one task prompt headless in `cwd`, return usage + cost.
+// Adding a harness = one function here. Realism first (CONTEXT.md, ADR-0004): each harness runs the way
+// the user runs it, with their own user-level setup and every tool, in its no-prompt (yolo) mode. The only
+// limit is `guard`: outside-service tools the user would have to approve (static-fire.yaml). An agent that
+// finds and copies a held-out original on the web is caught after the run (runner, copying.ts).
+import { existsSync, mkdirSync, readdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { join } from "node:path";
+import type { AgentConfig, AgentHarness, AgentResult, Harness, Usage } from "../core/types";
+import { findProject, pricingPath } from "../core/project";
+
+export interface Guard { blockedTools: string[] }
+export interface RunArgs { config: AgentConfig; prompt: string; cwd: string; logPath: string; timeoutMs: number; guard?: Guard }
+export type Adapter = (a: RunArgs) => Promise<AgentResult>;
+
+const zero = (): Usage => ({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 });
+
+// engine:judgeCall:begin  (judges run through this code: tests/identity.test.ts pins it, see ENGINE.judgeCall in core/engine.ts)
+export async function exec(cmd: string[], cwd: string, logPath: string, timeoutMs: number, env?: Record<string, string>, input?: string) {
+  const proc = Bun.spawn(cmd, { cwd, stdin: input === undefined ? "ignore" : new Blob([input]), stdout: "pipe", stderr: "pipe", env: env ? { ...process.env, ...env } : undefined });
+  let timedOut = false;
+  const timer = setTimeout(() => { timedOut = true; proc.kill(); }, timeoutMs);
+  const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
+  clearTimeout(timer);
+  writeFileSync(logPath, stdout);
+  if (stderr.trim()) writeFileSync(`${logPath}.stderr`, stderr);
+  return { stdout, stderr, code, timedOut };
+}
+
+/** HOME for agy judges: everything in ~/.gemini (so the login works) except the user's GEMINI.md. Trials use the real HOME. */
+let cleanHome: string | null = null;
+export function agyHome() {
+  if (cleanHome) return cleanHome;
+  const home = join(tmpdir(), "static-fire-agy-home");
+  const gem = join(home, ".gemini");
+  mkdirSync(gem, { recursive: true });
+  const src = join(homedir(), ".gemini");
+  for (const f of readdirSync(src)) if (f !== "GEMINI.md" && !existsSync(join(gem, f))) symlinkSync(join(src, f), join(gem, f));
+  return (cleanHome = home);
+}
+
+// engine:judgeCall:end
+
+const jsonl = (s: string) => s.split("\n").flatMap((l) => { try { return [JSON.parse(l)]; } catch { return []; } });
+// The rates: the pricing.yaml of the project found from the current folder when it has one, else the engine's.
+let pricing: any;
+const rates = () => (pricing ??= Bun.YAML.parse(readFileSync(pricingPath(findProject()), "utf8")));
+
+export function estimate(harness: Harness, model: string, u: Usage) {
+  const rate = rates()[harness]?.[model];
+  if (!rate) return null;
+  return (u.inputTokens * rate.input + u.cacheReadTokens * (rate.cachedInput ?? rate.input) + u.outputTokens * rate.output) / 1e6;
+}
+
+const claude: Adapter = async ({ config, prompt, cwd, logPath, timeoutMs, guard }) => {
+  const cmd = ["claude", "-p", prompt, "--model", config.model, "--output-format", "json", "--no-session-persistence",
+    "--dangerously-skip-permissions"];
+  if (config.effort) cmd.push("--effort", config.effort);
+  if (guard?.blockedTools.length) cmd.push("--disallowedTools", ...guard.blockedTools);   // variadic: keep it last
+  const r = await exec(cmd, cwd, logPath, timeoutMs);
+  let parsed: any;
+  try { parsed = JSON.parse(r.stdout); } catch { /* fallthrough */ }
+  const result = Array.isArray(parsed) ? parsed.find((x: any) => x.type === "result") : parsed;
+  if (!result) return { ok: false, error: r.timedOut ? "timed out" : (r.stderr.slice(-400) || "no result"), usage: zero(), costUsd: null, costSource: "unknown" };
+  const u = result.usage ?? {};
+  return {
+    ok: !result.is_error,
+    error: result.is_error ? String(result.result).slice(0, 400) : undefined,
+    usage: { inputTokens: u.input_tokens ?? 0, outputTokens: u.output_tokens ?? 0, cacheReadTokens: u.cache_read_input_tokens ?? 0, cacheWriteTokens: u.cache_creation_input_tokens ?? 0 },
+    costUsd: result.total_cost_usd ?? null,
+    costSource: result.total_cost_usd != null ? "reported" : "unknown",
+    turns: result.num_turns,
+  };
+};
+
+/** Codex block list → config overrides. "name@marketplace" is a plugin (its MCP servers are not in config.toml,
+ * and overriding one of them as a server leaves Codex unable to start); any other name is a config.toml server. */
+export function codexBlockArgs(names: string[]): string[] {
+  return names.flatMap((n) => ["-c", n.includes("@") ? `plugins.${n}.enabled=false` : `mcp_servers.${n}.enabled=false`]);
+}
+
+const codex: Adapter = async ({ config, prompt, cwd, logPath, timeoutMs, guard }) => {
+  const cmd = ["codex", "exec", "--json", "--skip-git-repo-check", "--ephemeral",
+    "-m", config.model, "--dangerously-bypass-approvals-and-sandbox", "-C", cwd];
+  if (config.effort) cmd.push("-c", `model_reasoning_effort=${config.effort}`);
+  cmd.push(...codexBlockArgs(guard?.blockedTools ?? []));
+  cmd.push(prompt);
+  const r = await exec(cmd, cwd, logPath, timeoutMs);
+  const events = jsonl(r.stdout);
+  const usage = zero();
+  let turns = 0;
+  for (const e of events) {
+    if (e.type !== "turn.completed") continue;
+    turns++;
+    const cached = e.usage?.cached_input_tokens ?? 0;
+    usage.inputTokens += (e.usage?.input_tokens ?? 0) - cached;
+    usage.cacheReadTokens += cached;
+    usage.outputTokens += e.usage?.output_tokens ?? 0;   // includes reasoning tokens (OpenAI usage semantics)
+  }
+  const failed = events.find((e) => e.type === "turn.failed" || e.type === "error");
+  const costUsd = estimate("codex", config.model, usage);
+  return {
+    ok: turns > 0 && !failed,
+    error: failed ? String(failed.message ?? failed.error?.message).slice(0, 400) : turns ? undefined : (r.timedOut ? "timed out" : r.stderr.slice(-400) || "no turns"),
+    usage, costUsd, costSource: costUsd != null ? "estimated" : "unknown", turns,
+  };
+};
+
+const agy: Adapter = async ({ config, prompt, cwd, logPath, timeoutMs }) => {
+  const cmd = ["agy", "-p", prompt, "--model", config.model, "--output-format", "json", "--dangerously-skip-permissions"];
+  if (config.effort) cmd.push("--effort", config.effort);
+  const r = await exec(cmd, cwd, logPath, timeoutMs);
+  let res: any;
+  try { res = JSON.parse(r.stdout); } catch { /* fallthrough */ }
+  if (!res) return { ok: false, error: r.timedOut ? "timed out" : (r.stderr.slice(-400) || "no result"), usage: zero(), costUsd: null, costSource: "unknown" };
+  const u = res.usage ?? {};
+  const cached = u.cache_read_tokens ?? 0;
+  const usage: Usage = {
+    inputTokens: Math.max(0, (u.input_tokens ?? 0) - cached),
+    outputTokens: (u.output_tokens ?? 0) + (u.thinking_tokens ?? 0),   // thinking is billed as output
+    cacheReadTokens: cached, cacheWriteTokens: 0,
+  };
+  const costUsd = estimate("agy", config.model, usage);
+  return {
+    ok: res.status === "SUCCESS",
+    error: res.status === "SUCCESS" ? undefined : String(res.error ?? res.response ?? res.status).slice(0, 400),
+    usage, costUsd, costSource: costUsd != null ? "estimated" : "unknown", turns: res.num_turns,
+  };
+};
+
+const pi: Adapter = async ({ config, prompt, cwd, logPath, timeoutMs }) => {
+  const cmd = ["pi", "-p", "--mode", "json", "--no-session", "--model", config.model];
+  if (config.provider) cmd.push("--provider", config.provider);
+  if (config.effort) cmd.push("--thinking", config.effort);
+  cmd.push(prompt);
+  const r = await exec(cmd, cwd, logPath, timeoutMs);
+  const usage = zero();
+  let cost = 0, turns = 0, error: string | undefined;
+  for (const e of jsonl(r.stdout)) {
+    if (e.type !== "turn_end") continue;
+    const m = e.message ?? {};
+    if (m.stopReason === "error") { error = String(m.errorMessage).slice(0, 400); continue; }
+    turns++;
+    usage.inputTokens += m.usage?.input ?? 0;
+    usage.outputTokens += m.usage?.output ?? 0;
+    usage.cacheReadTokens += m.usage?.cacheRead ?? 0;
+    usage.cacheWriteTokens += m.usage?.cacheWrite ?? 0;
+    cost += m.usage?.cost?.total ?? 0;
+  }
+  return { ok: turns > 0 && !error, error: error ?? (turns ? undefined : "no turns"), usage, costUsd: turns ? cost : null, costSource: turns ? "reported" : "unknown", turns };
+};
+
+export const adapters: Record<AgentHarness, Adapter> = { claude, codex, pi, agy };