static-fire 0.0.0-stage → 0.54.1

package/src/guide.md

@@ -0,0 +1,624 @@
+# Static Fire: the guide for agents
+
+Static Fire compares agent configurations (harness, model, effort) on tasks from the user's own work. It
+runs each configuration on the same task, grades every output the same way, and shows the scores against
+the cost on a dashboard.
+
+You are the agent that takes the user from a goal, such as "I want to test which agent is the best at making
+thumbnails", to a dashboard with data. The user gives the goal, examples, what good output looks like, and
+approvals. You do the rest: you make the files, run the commands and read the results. Ask the user only for
+what you cannot find out yourself. This guide is the whole manual. You do not need the engine source.
+
+The first part is the path, one step at a time. Each step says what it is for, the facts you need, and the
+safe default. How you talk with the user is yours to decide. The second part is the reference: every file
+format and command.
+
+Run every command as `static-fire <command>`, or `npx static-fire <command>` when `static-fire` is not
+installed.
+
+**Which project a command uses.** A command uses the nearest project above the current folder (a folder
+with `static-fire.yaml`, the current one included). Outside any project it uses the home, the one project
+the user keeps all their tasks and results in, and prints `using the home project: <folder>` on stderr.
+`static-fire home` prints the home; `static-fire home <folder>` sets it.
+
+## The rule above all: realism
+
+A trial must be as close as possible to the user's real job: the same inputs, tools, context and
+instructions they would give their agent, and graders that read the output as its real readers would. A
+score is worth something only when it predicts how a configuration does on the user's real work. Each
+difference from the real job is a known gap. Write each gap of a task in the project's `README.md` (make
+the file when the first gap comes up), so the user can see what the score does not cover.
+
+## What spends
+
+These start agent runs or judge calls, which cost money or use up the user's plan:
+
+- `static-fire run`: agent runs, and judge calls to grade each trial.
+- `static-fire rerun <batch>`: the failed trials of the batch again, with their judge calls.
+- `static-fire regrade`: judge calls only, for each trial it grades again.
+- On the dashboard: New run on the Runs page, and Regrade on the Results page.
+
+The rule: get the user's approval before each one starts, every time, with what it will run (step 6). An
+approval covers the one command it was given for. `run --plan` and `regrade --plan` print the plan and
+spend nothing.
+
+## The path
+
+### 1. Learn the goal, the examples and what good looks like
+
+**Why.** A task is only as good as its link to real work. The user knows the job, its inputs and its readers.
+You do not.
+
+**What you need from the user:**
+
+- What job they want to test, and why it matters to them.
+- One or more real examples of the inputs: the notes, brief, files or links they would give their agent.
+- What good output looks like, and what makes an output fail. A past output they liked, or one they did not
+  like, helps a lot.
+- Who reads the output in real life, and how. A viewer who sees a thumbnail at 320 px, an editor who skims
+  the headings, a client who checks every number.
+- Which configurations they want to compare, if they know. `init` finds the harnesses on the machine.
+
+**Facts.** Tasks from the user's real work are the point of Static Fire and its value: the scores then say
+how agents do on the work the user cares about. Any source is allowed. A public benchmark item or a made-up
+job is fine when the user wants it. The user and you own the quality of the task. Static Fire does not judge
+it.
+
+**Safe default.** Ask for real examples first. When the user has none at hand, offer to look for material
+on the machine with their permission, or to build the task from a description and record that as a gap.
+Do not invent the user's taste: ask what good looks like.
+
+### 2. Set up the project and settle the block list
+
+**Why.** A project is the folder that holds the tasks, the configurations and the records. Most users need
+one: the home. A trial runs with no permission prompts, so nobody is there to approve a tool that sends
+mail, posts a message, spends money or deploys. The block list stops those tools in trials.
+
+**Do.** Run `static-fire init --json`. It never waits for input and never changes a file that exists.
+
+- With no home, outside any project, it makes the project in `~/static-fire` and saves it as the home.
+  `static-fire init <folder> --json` makes it in that folder instead, and saves that as the home.
+- With a home, outside any project, it writes nothing and reports the home. `static-fire init <folder>`
+  then makes a second project in that folder, with its own records and dashboard. The home does not change.
+- Inside a project, it writes nothing and reports that project.
+
+Tell the user where the home is, or will be, before you add their task to it. `static-fire home` prints it,
+or says none is set, and writes nothing; with none set, the home will be `~/static-fire` unless the user
+names another folder. The report's `project` is the project's folder, `created` says if this run made it,
+`home` is the home's folder (or null), `isHome` says if `project` is the home, and `homeSaved` says if this
+run saved the home.
+
+**Safe default.** Put the task in the home. Make a second project only when the user asks for a separate
+one.
+
+A new project has:
+
+- `configs/<harness>.yaml`: one agent configuration for each harness it found on the `PATH` (`claude`,
+  `codex`, `agy`, `pi`), with a default model.
+- `judges.yaml`: one judge, from the first harness found that can judge (`claude`, then `codex`, then `agy`).
+  Pi cannot be a judge. When no harness that can judge is found, `judges.yaml` lists none, and no batch can
+  run until one is added (see the reference).
+- `evals/`, `suites/`, `static-fire.yaml`, a `.gitignore` for `runs/` and `.cache/`, and `AGENTS.md`.
+
+The JSON report also has, for each harness: `found` and `path`, `blockable`, `tools` (each with `name`,
+`kind`, `foundIn` and `block`, the entries that block it) and `notes`. It also has `judge`, `written`, `kept`
+and `notes`. Run `static-fire init --json` again in the project at any time: it changes nothing and lists the
+tools again.
+
+**Settle the block list.** `init` finds the tools; it does not choose and does not ask. For each tool in the
+report, decide if it acts on an outside service: sends or posts something, changes data in an account,
+spends money, or deploys. Then the user decides. Before they decide, they need to see:
+
+- Every tool found, for each harness, with its kind and where it was found. They cannot judge a list they
+  have not seen.
+- Which tools you propose to block, each with its reason.
+- Which tools stay on.
+- Their choices: approve your list, change it, or block nothing.
+
+Blocking nothing is a valid choice. Its facts: a trial then runs with every tool of the user's setup and no
+permission prompts, which is the closest to how their agents really run. And a tool that acts on an outside
+service can then act in a trial, with nobody there to stop it.
+
+Write the entries of the user's choice under `trials.blockedTools.<harness>` in `static-fire.yaml`, copied
+from each tool's `block` list.
+
+**Facts.**
+
+- Static Fire ships no list of outside services and blocks nothing by default. Your proposal is the list.
+- Blocking works for Claude Code and Codex trials. Antigravity (`agy`) and Pi trials cannot block a tool
+  yet: they run with every tool of the user's setup. Tell the user this before they choose those harnesses.
+- A plugin with no MCP server has nothing to block (its `block` list is empty).
+- The claude.ai connectors in the list are the ones Claude Code last saw, so the list can be out of date.
+
+When the report lists no tools (no harness found, or none with outside tools), there is nothing to block yet.
+Run `static-fire init --json` again after the user installs a harness.
+
+**Safe default.** When you are not sure if a tool acts on an outside service, propose to block it. The
+result of a fast "yes" from the user must be safe. Tools that only read (search, docs, the local file
+system) can stay on, because the real job has them.
+
+**Configurations.** Show the user the configurations in the project, and change them to what they want to
+compare: other models, other effort levels, more than one configuration per harness. A configuration is a
+small YAML file (see the reference). `init` writes a current model id for each harness it found. For another
+model, take its id from the harness itself (its model list or its documentation). Static Fire does not check
+a model id: a wrong one shows up as a failed trial in step 5.
+
+A trial runs with the user's own setup for that harness: their instructions, settings, skills and MCP
+servers, minus the block list. That is on purpose: it is how their agents really run.
+
+### 3. Write the task
+
+**Why.** The task is the job the configurations do and the questions their output is graded on. A score is
+only as good as its questions.
+
+**Do.** Run `static-fire new task <id>`, with a short kebab-case id. It makes `evals/<id>/task.yaml`, with a
+comment on each part, and an empty `evals/<id>/inputs/`. Put the example inputs in `inputs/`. Fill in
+`task.yaml` (see the reference for every field).
+
+**Facts.**
+
+- The prompt is the only message the agent gets, as the user would give it. The agent's working folder holds
+  `inputs/` (at the same path as in the task folder), the files of `project/` at its top level, and an empty
+  `out/`. Name the input files and the output path in the prompt, as the user would.
+- The output is one file. Text or an image both work. A judge sees a PNG or JPEG output as an image.
+- Material that only graders may see goes in `references/`, `sources/` or `heldOut`, never in `inputs/`.
+- Each criterion is one yes or no question about the output, where yes is a pass. A good criterion names
+  what to look at and where the line between pass and fail is, so two careful graders give the same answer.
+  "Does it sound human?" is not a criterion. "Is every paragraph three sentences or fewer?" is.
+- A criterion may limit what its grader sees only to act like a real reader, never to make grading easier.
+- Use a check (regex, word count, frontmatter field, shell command) when a rule can decide it. Use a judge
+  when it takes reading. Do not use the `human` grader: it is not built yet, so its verdict is always an
+  error and every grading of the task is invalid.
+- A check is cheap to test before any trial: write a sample output by hand and run the check's command or
+  pattern on it yourself. A `regex` is a JavaScript regular expression.
+- Test each derived evidence script the same way before the first trial: run it on a sample output, see
+  that it ends well inside its 60-second limit, and open the PNG it writes. Every file in the task's
+  `grader/` folder is part of the grading version, so a fix after a trial costs a regrade (step 5).
+- Give each criterion a level: `floor` (most configurations should pass), `strong` (good ones pass) or
+  `ceiling` (only the best pass). A mix makes the scores spread out. These three names are the ones the
+  dashboard shows.
+- `static-fire tasks` warns under 15 criteria in all, under 3 in a measure, or when a level is missing.
+  These are advice. With 15 criteria, one criterion moves the score by about 7 points.
+- A gate is a criterion outside the measures: when one fails, the task score is 0. "The output file exists
+  and is not empty" is always a gate, added by the engine.
+
+**Confirm the criteria with the user.** The criteria decide every score, so the user approves them as their
+own step, apart from any approval to run. Show each criterion in plain words with its level, and explain the
+levels in plain words with their names: floor, strong, ceiling. Offer to change any criterion, and get the
+user's yes on the criteria before you go on.
+
+**Safe default.** Turn each thing the user said about good and bad output into one or more criteria. Write
+each gap between the task and the real job in the project's `README.md`.
+
+### 4. Check the task and put it in a suite
+
+**Why.** A task with an error cannot run. A task counts toward the Agent Performance Index only when it is in a
+suite.
+
+**Do.** Run `static-fire tasks`. Each error stops the task from running: fix every one. Warnings are advice.
+Then add the task to a suite, a file `suites/<id>.yaml` (see the reference), and run `static-fire suites`,
+`static-fire configs` and `static-fire judges` to see that each file loads.
+
+**Safe default.** One suite per kind of work, for example "Thumbnails". Put the task in a new suite when no
+suite fits.
+
+### 5. Run one trial and look at the result
+
+**Why.** The first trial shows that the task works: the agent finds its inputs, writes the output where the
+task expects it, and each verdict is right for the right reason. It is the least spend that shows this.
+
+**Before you run.** A trial spends: the agent run, and the judge calls that grade it. Show the user its plan
+(add `--plan` to the command below, as in step 6) and get their approval first.
+
+**Do.** Run one trial of one configuration, usually the one the user works with:
+`static-fire run --tasks <task> --configs <config> --trials 1`. A trial can take several minutes (the limit
+is 20 minutes). The command prints a line for each trial, with its score and the agent's cost, and ends with
+`batch <id> done`.
+
+**Look at the result:**
+
+- The output: `runs/trials/<trial-id>/workspace/`, at the task's output path. The agent's log is
+  `runs/trials/<trial-id>/agent.log`.
+- The verdicts: `static-fire show --tasks <task> --json`. Each trial in `trials` has `grades.verdicts`
+  (each criterion's score, each vote's pass and reason, or an `error`), `result.costUsd` with
+  `result.costSource`, and `grades.judgeCostUsd`.
+- A trial that `failed` did not run (the harness could not start, a login expired, it timed out); its
+  `result.error` says why. A grading that is `invalid` had a grader error; a regrade grades it again.
+
+When a verdict is wrong because of the task (the question is unclear, a check has a bug, the prompt names
+the wrong path), fix the task and run one trial again. When a verdict is wrong because the output is bad,
+the task works. Show the user the output and the verdicts, so they can say whether the grading matches
+their own view.
+
+**Facts.** A change to the prompt, the output path, or a file in `inputs/` or `project/` makes a new task
+hash: earlier trials no longer share a chart with new ones. A change to a criterion, `context`, a reference,
+any file in `grader/` (a check's helper script or a derived evidence script) or the judge makes a new
+grading version: earlier trials are left out of the view until a `static-fire regrade` grades them again.
+That regrade spends judge calls and needs the user's approval like any other spend. So settle the task, its
+grader scripts included, before the batch.
+
+### 6. Work out the batch cost, get approval, run the batch
+
+**Why.** The batch is where most of the spend goes. The user approves it knowing what it uses.
+
+**The plan in runs comes first.** Add `--plan` to the `run` command you mean to start, for example
+`static-fire run --suites <suite> --configs <a>,<b> --trials 3 --plan --json`. It starts nothing and writes
+nothing. It gives the trials, the agent runs for each configuration (a human configuration needs none),
+the judge calls for the judge as a range, and what the range leaves out (`judgeCalls.notCounted`). These
+counts are what a user on a subscription can judge: their plan counts runs and usage, not dollars. `static-fire regrade <flags> --plan --json` gives the plan of a
+regrade the same way. `rerun` has no plan: it runs the failed trials of the batch again, each with its
+grading.
+
+**Facts.**
+
+- Judge calls are often the larger part of a batch, in count and in cost. Their number per trial comes
+  from the judge criteria, put in groups: criteria with the same evidence and the same derived evidence
+  script are one group; each `isolate` criterion, each other set of evidence, each other derived script and
+  the judged gates make more groups. Each group gets one judge call per vote. With 3 votes (the default,
+  `--votes`), 2 calls run first and the third only when they disagree. That is the range: the low end when
+  every first pair agrees, the high end when none does.
+- Outside the range: a judge answer that does not parse is tried once more, and the plan says that this
+  call is not counted. A trial with no output, or with a derived evidence script that fails, gets fewer calls.
+- Dollar figures are a second fact, and they are API prices, not what a subscription is charged. The cost
+  of a configuration is known only once it has run, and configurations on other models can cost very
+  different amounts for the same task. A trial's judge cost is in `grades.judgeCostUsd`.
+- Claude Code and Pi report their own cost (`costSource: reported`). Codex and Antigravity costs are
+  estimated from tokens and the engine's price list (`estimated`). A model that the price list does not have
+  gives no cost (`unknown`, shown as `?` in the `run` output). A project can have its own `pricing.yaml`.
+- Cost that is not in the harness's own report, such as image generation by an outside service, is not
+  counted. Static Fire enforces no spending limit.
+
+**Safe default.** The user sees the plan in runs before the batch, with the dollar figures you have as API
+prices, what is measured made clear from what is scaled up, and approves it. Nothing runs before that
+approval. A batch that starts prints the same plan first.
+
+**Do.** Run the batch, for example `static-fire run --suites <suite> --configs <a>,<b> --trials 3`, or
+`--configs all` for every configuration. It runs 4 trials at a time by default (`--concurrency`). A batch can
+take a long time, so start it in the background and follow it with `static-fire list` or on the dashboard's
+Runs page. When trials fail for a reason outside the task (a usage limit, a login),
+`static-fire rerun <batch>` runs the failed ones again in the same batch, after the user approves it.
+
+### 7. Open the dashboard
+
+**Why.** The dashboard is where the user sees which configuration does the work best and what it costs, and
+why each score is what it is.
+
+**Do.** Run `static-fire serve` in the project and keep it running in the background: it is a server and does
+not exit. It prints `Static Fire → http://localhost:4477 (records: <path>)`. Give the user that address.
+`--port <n>` picks another port. You can start it at any time; trials show up as they are graded.
+
+**What the user sees.** The Results page shows the score of each configuration against its cost per task, as
+a chart or a table, for the Agent Performance Index or one task. The index covers the suites the user turns on with
+the suite buttons. With none picked, it covers the best set the data supports, and one line says which suites it
+covers and why each other suite is left out. `static-fire show` uses the same set and prints the same line. With one
+task in one suite, the index and the task show the same score. Each point is the mean of the valid trials with its 95% confidence
+interval. Opening a trial shows its output, with each verdict and its reason in the Verdicts tab. The Runs page lists each batch and its trials,
+and can start a new run. The Compare page shows what changed between two grading versions of a task.
+
+Before any trial is graded, the Results page says "No trials in this view". The Task section of the Filter menu
+lists every task of the project, also a task that is in no suite; `static-fire show --tasks <task>` shows one
+task as text. The Tasks page shows each suite and task: its description, what the agent gets and how it is
+graded.
+
+## Reference
+
+### The project folder
+
+| Path | What |
+|---|---|
+| `static-fire.yaml` | Marks the project. The tools that trials may not use |
+| `configs/<id>.yaml` | One agent configuration, or a human configuration |
+| `judges.yaml` | The judges. The first is the default |
+| `suites/<id>.yaml` | A suite: a named list of tasks |
+| `evals/<id>/task.yaml` | A task, with its folders next to it |
+| `pricing.yaml` | Optional. Token prices that replace the engine's |
+| `runs/static-fire.db` | The record store: every batch, trial, grading and verdict. `STATIC_FIRE_DB` moves it |
+| `runs/trials/<trial-id>/` | Each trial's `workspace/`, `agent.log` and `gradings/<grading-id>/` |
+| `.cache/` | Profiles installed for a batch |
+
+The record store is the only copy of the records. Do not edit or delete it. To copy it, use
+`sqlite3 runs/static-fire.db ".backup '<dest>'"`, or `static-fire export <path>` for JSON.
+
+In the examples below, a code block that starts with `yaml file=<path>` is a whole file, and the path is
+where it goes in the project. The examples are one small, valid project.
+
+### Task: `evals/<id>/task.yaml`
+
+```yaml file=evals/thumbnail-text/task.yaml
+id: thumbnail-text
+title: Thumbnail text for a video
+description: Writes the words on a YouTube thumbnail from the video's outline.
+prompt: |
+  Read the outline of my next video in inputs/outline.md. Write the text for its
+  thumbnail: at most 4 words, in capitals, that make a viewer want to click.
+  Save it to out/thumbnail.md with YAML frontmatter that has a `title` field
+  (the video title you would pair with it), then the thumbnail text.
+inputs: ./inputs
+output: ./out/thumbnail.md
+context: The viewer scrolls a YouTube home page and sees the thumbnail for about one second.
+references: ./references
+gates:
+  - id: has-title
+    question: Does the file have frontmatter with a non-empty title?
+    level: floor
+    grader: check
+    check: { type: frontmatter, field: title, minChars: 1 }
+measures:
+  - id: readable
+    label: Readable at a glance
+    criteria:
+      - id: four-words-max
+        question: Does the text after the frontmatter have 4 words or fewer?
+        level: floor
+        grader: check
+        check: { type: words, max: 4 }
+      - id: no-hashtags
+        question: Is the file free of hashtags?
+        level: floor
+        grader: check
+        check: { type: regex, pattern: "#\\w", max: 0 }
+      - id: no-filler
+        question: Is every word needed, with no filler such as "the", "a" or "very"?
+        level: strong
+        grader: judge
+  - id: gets-clicked
+    label: Gets clicked
+    criteria:
+      - id: names-the-payoff
+        question: Does the text name the specific result the video shows, not a vague promise?
+        level: strong
+        grader: judge
+        evidence: [output, inputs]
+      - id: not-the-title
+        question: Does the text add something the title does not already say?
+        level: strong
+        grader: judge
+      - id: beats-past-work
+        question: Is it at least as specific as the best past thumbnail in the references?
+        level: ceiling
+        grader: judge
+        evidence: [output, references]
+```
+
+Here `inputs/outline.md` and one or more files in `references/` sit next to `task.yaml`. This example has 6
+criteria, so `static-fire tasks` warns that it has fewer than 15; a real task would have more.
+
+| Field | Required | What |
+|---|---|---|
+| `id` | yes | The task's folder name under `evals/` |
+| `title` | yes | A short name, shown on the dashboard |
+| `description` | no | One line on what the task tests |
+| `version` | no | A whole number, a readable label only. It decides nothing |
+| `prompt` | yes | The only message the agent gets, as the user would give it |
+| `inputs` | yes | A folder in the task folder (for example `./inputs`). Copied into the agent's working folder at the same path |
+| `output` | yes | The file the agent must write, relative to its working folder (for example `./out/answer.md`). `out/` exists at the start |
+| `project` | no | A folder whose files the agent would find in the real project. Its content is copied to the top of the working folder. Judges can see it as evidence |
+| `context` | no | One paragraph every judge reads before the criteria |
+| `references` | no | A folder of past work that judges can compare with. The agent never sees it |
+| `sources` | no | A folder of saved copies of the pages the inputs link to, for fact checks. The agent never sees it |
+| `heldOut` | no | A file: the real original the task was built from. An output that shares more than 30% of its runs of 8 words with it makes the trial invalid. The agent never sees it |
+| `gates` | no | A list of criteria outside the measures. When one fails, the task score is 0 |
+| `measures` | yes | At least one measure |
+
+A measure has `id` (kebab-case, unique), `label` (shown on the dashboard), `weight` (optional, a positive
+number, 1 by default) and `criteria` (at least one). The task score is 0 when a gate failed, else the
+weighted mean of the measure scores, from 0 to 100. A measure's score is the share of its criteria passed.
+
+The engine adds the gate `output-exists` (the output file exists and is not empty) to every task, so no
+criterion can use that id.
+
+Files whose names start with `.` are left out of every task folder. Each trial and grading uses a copy of the
+task folder made when it starts, so editing the task while a batch runs does not change that batch's trials;
+a trial whose files changed after the batch started fails and says so.
+
+### Criterion
+
+| Field | For | What |
+|---|---|---|
+| `id` | all | Kebab-case, unique in the task |
+| `question` | all | A yes or no question, where yes means pass. Names what to look at and where the line is |
+| `level` | all | `floor`, `strong` or `ceiling` |
+| `grader` | all | `check` or `judge`. (`human` is accepted but not built: its verdict is always an error) |
+| `check` | check | One check (see below) |
+| `evidence` | judge | What the judge reads: a list of `output`, `inputs`, `project`, `references`, `sources`. Default `[output]`. `project`, `references` and `sources` need that field on the task |
+| `isolate` | judge | `true` gives the criterion its own judge call. By default, judge criteria with the same evidence share one call |
+| `derived` | judge | A script in the task's `grader/` folder that makes an image of the output for the judge (see "Derived evidence") |
+
+A judge always sees the task's prompt and `context`, then the evidence, then the questions. It gives each
+criterion several votes (3 by default, `--votes` on `run`); the criterion's score is the share of votes that
+passed. A text file is shown inline. An image is attached for a Claude Code or Codex judge, and given as a
+file to open for an Antigravity judge. A judge runs without the user's setup, so grading is the same every
+time.
+
+### Checks
+
+Each check gives one pass or fail. A check that cannot run (a bad pattern, a command that times out) gives
+an error, and the grading is invalid. The output file is read as text.
+
+| Type | Fields | Passes when |
+|---|---|---|
+| `regex` | `pattern`, `flags` (optional), `min`, `max` | The JavaScript regular expression matches at least `min` and at most `max` times in the whole file. With neither, at least once. `g` is always on; add `m` for line anchors, `i` for any case |
+| `words` | `min`, `max` | The word count of the text after any YAML frontmatter is in range. A word is a run of characters with a letter or digit |
+| `frontmatter` | `field`, `minChars`, `maxChars` | The file starts with YAML frontmatter (`---` lines) whose `field` is a non-empty string, with its length in range |
+| `command` | `run`, `timeoutSec` (default 60) | The shell command (run with `sh -c`) exits with 0 |
+
+A `command` check runs in the trial's working folder, with these environment variables: `OUTPUT` (the
+output file), `WORKSPACE` (the working folder) and `TASK_DIR` (a copy of the task folder, so it can read a
+script or data you put there). Its last line of output becomes the reason shown with the verdict. For
+example: `check: { type: command, run: 'python3 "$TASK_DIR/grader/check_links.py" "$OUTPUT"' }`. The command
+runs with the user's own `PATH`: a program it calls, such as `python3` or `bun`, must be installed on the
+machine that grades. The Bun that the package brings is not on that `PATH`.
+
+### Derived evidence
+
+A judge criterion can name a script that makes an image from the output. The judge then sees the work as
+its real reader does (a thumbnail at 320 px wide), or with what a reader cannot see marked on it.
+
+```yaml
+- id: readable-small
+  question: Can every word on the thumbnail be read at this size?
+  level: floor
+  grader: judge
+  derived: scale-320.py   # evals/<task>/grader/scale-320.py
+  evidence: []            # the judge sees only the derived image, not the full-size output
+```
+
+The contract for a script:
+
+- **Name.** `derived:` is a path relative to the task's `grader/` folder, one script per criterion. Only a
+  judge criterion can have one. `static-fire tasks` reports a script that does not exist.
+- **How it runs.** By its extension: `.ts` and `.js` with the Bun that runs static-fire (the package brings
+  it, so no `bun` on the `PATH` is needed), `.py` with `uv` as a single-file script (declare packages such as
+  Pillow in the script's inline `# /// script` header), `.sh` with `sh`. `uv` must be on the `PATH` of the
+  machine that grades.
+- **Input.** Argument 1 and `$OUTPUT` are the output file. `$TASK_DIR` is the task folder, so the script can
+  read other task files. `$WORKSPACE` is the working folder, and the script runs in it. The time limit is 60
+  seconds.
+- **At the same time.** Gradings run at the same time: up to `--concurrency` (4 by default) in a batch or a
+  regrade, each in its own trial folder. In one grading, every script it needs runs at once. So a script
+  writes only to its argument 2 path and keeps nothing in a fixed path that another run could share.
+- **Output.** Write one PNG to the path in argument 2 (also `$DERIVED`). A non-zero exit, a time-out, no file
+  or a file that is not a PNG makes the verdict an error, so the grading is invalid.
+- **Where it stays.** `runs/trials/<trial-id>/gradings/<grading-id>/_derived/<script>.png`, with the
+  script's output next to it as `.log`.
+- **Sharing.** Criteria that name the same script share one image, and one judge call when their other
+  evidence is the same. A new script name or new script content gives a new grading version: test it before
+  the first trial (step 3).
+
+An example script, for an image output:
+
+```python
+# /// script
+# dependencies = ["pillow"]
+# ///
+import sys
+from PIL import Image
+
+src, dest = sys.argv[1], sys.argv[2]
+img = Image.open(src)
+img.thumbnail((320, 320))
+img.save(dest, "PNG")
+```
+
+### Agent configuration: `configs/<id>.yaml`
+
+```yaml file=configs/claude-opus.yaml
+harness: claude
+model: claude-opus-5-5
+effort: medium
+```
+
+| Field | What |
+|---|---|
+| `harness` | `claude` (Claude Code), `codex` (Codex), `agy` (Antigravity) or `pi` (Pi) |
+| `model` | The model id as the harness takes it. Antigravity model ids carry the effort (`gemini-4-hydrogen-medium`) |
+| `effort` | Optional. The effort level as the harness takes it (`low`, `medium`, `high`) |
+| `provider` | Pi only: the provider the user is logged in to in Pi (for example `openai-codex`) |
+| `profile` | Optional. An apkit Profile name; needs `apkit` on the machine |
+
+The file name is the id that `--configs` takes. Do not add a `label`: the label is built from the fields
+("Claude Code · Opus 5.5 · medium"). A configuration with the same fields under another file name keeps its
+records.
+
+The user can also compete as a **human configuration**: the file they wrote for a task is graded like an
+agent's output, with no agent cost.
+
+```yaml file=configs/me.yaml
+harness: human
+outputs:
+  thumbnail-text: ~/Documents/thumbnail-text.md
+```
+
+`outputs` maps a task id to a file path (absolute, `~/`, or relative to the project). Each run copies the file
+as it is then, and only the latest trial of a human configuration counts.
+
+### Judges: `judges.yaml`
+
+```yaml file=judges.yaml
+judges:
+  - id: claude-sonnet-5-5
+    label: Claude Code · Sonnet 5.5 · medium
+    harness: claude
+    model: claude-sonnet-5-5
+    effort: medium
+```
+
+`harness` is `claude`, `codex` or `agy`. Pi cannot judge. The first judge is the default; `--judge <id>` on
+`run` or `regrade` picks another. A batch needs a judge even when every criterion is a check. Another judge
+gives another grading version, so the scores of two judges are never mixed. With no judge in the file,
+`static-fire judges` prints nothing.
+
+### Suite: `suites/<id>.yaml`
+
+```yaml file=suites/thumbnails.yaml
+title: Thumbnails
+description: Packaging for my YouTube videos.
+tasks:
+  - thumbnail-text
+```
+
+| Field | What |
+|---|---|
+| `title` | Shown on the dashboard ("Thumbnails Index") |
+| `description` | Optional. Shown on the Tasks page |
+| `tasks` | The task ids |
+| `weights` | Optional. Task id to weight; 1 by default |
+
+The suite's index is the mean task score over its tasks. A configuration gets an index only with a valid
+score on every task of the suite.
+
+### Settings: `static-fire.yaml`
+
+```yaml file=static-fire.yaml
+trials:
+  blockedTools:
+    claude:
+      - mcp__example-mail
+    codex:
+      - example-mail
+```
+
+Copy each entry from the `block` list of a tool in `static-fire init --json`. For Claude Code an entry is a
+tool rule, and `mcp__<server>` blocks every tool of that server. For Codex it is an MCP server name from its
+`config.toml`, or a plugin as `name@marketplace`. Entries for `agy` and `pi` do nothing yet. Each trial
+records the list it ran with.
+
+### Commands
+
+| Command | What |
+|---|---|
+| `static-fire init [<folder>] [--json]` | Make the home (`~/static-fire`, or `<folder>`), or report the home or the current project; with a home, `<folder>` makes a second project. Lists the harnesses and outside tools |
+| `static-fire guide` | This guide |
+| `static-fire home [<folder>]` | Show the home, the project a command uses outside any project, or save a project folder as the home |
+| `static-fire new task <id>` | A new task folder with a commented `task.yaml` |
+| `static-fire tasks` | Each task, with its errors and warnings |
+| `static-fire configs` | Each configuration and its label |
+| `static-fire judges` | Each judge; the default is marked |
+| `static-fire suites` | Each suite and its tasks |
+| `static-fire run` | One batch. `--tasks <a,b>` or `--suites <a,b>`, `--configs <a,b>` or `all` (all by default), `--trials <n>` (3), `--judge <id>`, `--votes <n>` (3), `--concurrency <n>` (4). `--plan` prints the plan and starts nothing; `--plan --json` for JSON |
+| `static-fire rerun <batch>` | The failed trials of a batch again, as new trials. `--concurrency <n>` |
+| `static-fire regrade` | Grade stored outputs again under the current grading, with judge calls only. `--tasks`, `--configs`, `--batch <id>`, `--judge`, `--votes`, `--only-invalid`, `--concurrency`, `--plan` (with `--json`) |
+| `static-fire list` | Each batch, newest first, with its trial counts |
+| `static-fire show` | The scores as text. `--tasks` or `--suites`, `--judge`, `--grading <hash>`, `--since <date>`, `--until <date>`, `--json` |
+| `static-fire serve [--port <n>]` | The dashboard, at port 4477 by default |
+| `static-fire export [path]` | Every record as JSON |
+| `static-fire migrate [--from <folder>]` | One-off import of an experiment folder from before version 0.25.0 |
+| `static-fire --version` | The installed version |
+
+A flag that a command does not take is an error. A list is comma-separated: `--configs a,b`.
+
+### Limits to know
+
+- **Trials run as the user really works.** Each trial runs headless with no permission prompts, with the
+  user's own setup for that harness and every tool, including the web, except the block list. It runs in a
+  new folder in the system temp folder, outside any repo.
+- **No blocking for Antigravity and Pi trials** yet: they can use every tool of the user's setup.
+- **Pi cannot be a judge.** A judge runs on Claude Code, Codex or Antigravity.
+- **No judge, no batch.** When `init` finds no harness that can judge, add one to `judges.yaml` first.
+- **No human grader yet.** Use checks and judges.
+- **A trial is one shot.** The agent gets the prompt once, with no feedback and nobody to answer its
+  questions.
+- **The copy check reads text only.** An image output that copies a published original is not caught.
+- **Cost** is reported by Claude Code and Pi, estimated for Codex and Antigravity, and unknown for a model
+  that has no price. Judge cost is not part of a configuration's cost. Before a batch there is the plan in
+  runs (`--plan`), not a dollar estimate. There is no spending limit.

package/src/server.ts

@@ -0,0 +1,28 @@
+// Local server: the dashboard page and the JSON API (src/api.ts) over the record store. Trials written by
+// CLI processes show up live, because the event stream watches the database rather than this process.
+import index from "../web/index.html";
+import { apiRoutes } from "./api";
+import { openStore } from "./core/db";
+import type { ProjectPaths } from "./core/project";
+import { projectCatalog } from "./core/view";
+import { PAGES } from "../web/pages";
+
+/**
+ * Serves the dashboard of `project`: its store, and its configs, judges, suites and tasks. The page is bundled
+ * once, minified and cached, unless `development` asks for Bun's development mode (bundled on request, with hot
+ * reload), which is for work on the engine (`static-fire serve --dev`, `bun run dev`).
+ */
+export function serve(port: number, project: ProjectPaths, { development = false }: { development?: boolean } = {}) {
+  const { root, dbPath, cache } = project;
+  const store = openStore(dbPath);
+  const api = apiRoutes({ store, dbPath, catalog: () => projectCatalog(root), root, cacheDir: cache });
+  const server = Bun.serve({
+    port,
+    development,
+    idleTimeout: 0,
+    // Every page path serves the one page; it shows the page its path names (web/pages.ts).
+    routes: { ...Object.fromEntries(PAGES.map((p) => [p.path, index])), ...api.routes },
+  });
+  console.log(`Static Fire → http://localhost:${server.port} (records: ${dbPath})`);
+  return server;
+}

package/web/api.ts

@@ -0,0 +1,6 @@
+/** GET JSON; rejects with the server's error message on a failed request, so no caller reads `{ error }` as data. */
+export const api = <T,>(p: string): Promise<T> => fetch(p).then(async (r) => {
+  const body = await r.json().catch(() => null);
+  if (!r.ok || (body && typeof body === "object" && "error" in body)) throw new Error(body?.error ?? `${r.status} ${r.statusText}`);
+  return body as T;
+});