static-fire 0.44.0 → 0.54.1

package/README.md

@@ -2,7 +2,7 @@
 
 Find out which **agent configuration** (harness + model + effort + apkit Profile) does your real work best,
 and what it costs. Same tasks, same graders, same judge for every config; the result is an
-Artificial Analysis-style score-vs-cost chart, the **Workflow Index**, for *your* workflows.
+Artificial Analysis-style score-vs-cost chart, the **Agent Performance Index**, for *your* workflows.
 
 **The whole point is real-world performance.** A trial reproduces the real job as closely as it can: the
 inputs, tools, context and instructions you would actually give the agent, and graders that read the
@@ -15,29 +15,32 @@ The engine's gaps are listed below; the gaps of one task go in its project's REA
 You need Node or Bun. The package brings its own Bun ([ADR-0008](docs/adr/0008-npm-package-brings-bun.md)).
 
 ```sh
-mkdir my-evals && cd my-evals
-npx static-fire init                       # make a project in this folder
+npx static-fire init                       # make your home project in ~/static-fire
 ```
 
-Then open your agent in that folder and tell it what you want to test, for example "I want to find out which
+Then open your agent in `~/static-fire` and tell it what you want to test, for example "I want to find out which
 agent is the best at making thumbnails". `init` writes an `AGENTS.md` that sends the agent to
 `static-fire guide`, which takes it from your goal to a dashboard with data. The guide also holds the file
 formats, so neither of you needs to read the engine source.
 
-A **project** is the folder that holds `static-fire.yaml`. Every command works on the project it finds by
-walking up from the current folder, so you can run it from any subfolder. The commands, with
+A **project** is the folder that holds `static-fire.yaml`. A command works on the nearest project at or above
+the current folder, so you can run it from any subfolder. Outside any project it uses your **home** project, and
+says so in one line. The commands, with
 `npx static-fire` in place of `static-fire` when it is not installed:
 
 ```sh
-static-fire init [--json]                  # make a project in the current folder; never overwrites a file
+static-fire init [<folder>] [--json]       # make the home (~/static-fire) or a project in <folder>; never overwrites a file
 static-fire guide                          # the guide for agents: from a goal to a dashboard, and the file formats
+static-fire home [<folder>]                # show the home project, or set it to a project folder
 static-fire new task <id>                  # a task folder with a commented task.yaml
 static-fire configs | judges | suites | tasks
 static-fire run --suites <suite> | --tasks <task>  --configs <config>,<config>|all \
                 [--trials 3] [--judge <judge>] [--votes 3] [--concurrency 4]   # one batch
+static-fire run ... --plan [--json]        # the plan in runs: agent runs per config and judge calls; starts nothing
 static-fire rerun <batch>                  # its failed trials again, as new trials (after a usage limit, for example)
 static-fire regrade [--tasks t] [--configs a,b] [--batch id] [--judge id] [--votes n] [--only-invalid]
                                            # judge calls only: trials with no valid grading under the current grading hash
+static-fire regrade ... --plan [--json]    # the trials it would grade and their judge calls; starts nothing
 static-fire show [--suites s | --tasks t] [--judge id] [--grading hash] [--since date] [--until date] [--json]
                                            # the comparable trials, pooled across batches (JSON for agents)
 static-fire serve                          # dashboard at http://localhost:4477
@@ -50,15 +53,17 @@ static-fire migrate                        # one-off import of an experiment fol
 
 Every trial is a record; a chart is a query over them. Running one new config on one task adds its point to
 the existing chart, next to every earlier trial that is comparable: the same task hash, and a valid grading
-under the same grading hash. The dashboard has three pages. **Results** (`/`) shows the chart, or the same
-numbers as a table. Its scope chips and **Filter** menu (harness, model, effort, Profile, judge, grading
-version, date range) are in the page URL, so a view can be bookmarked. Harness, model, effort and Profile
+under the same grading hash. The dashboard has five pages. **Results** (`/`) shows the chart, or the same
+numbers as a table. Its index buttons (the suites in view) and **Filter** menu (task, harness, model, effort,
+Profile, judge, grading version, date range) are in the page URL, so a view can be bookmarked. The per-task
+table and the trials are one card each, with a menu to pick the task. Harness, model, effort and Profile
 each take several values, for example `?model=opus&model=haiku`. When a question or the judge changes,
 earlier trials are left out of the view, and the notice under the chart offers to regrade them with judge
 calls only. **Runs** (`/runs`) lists every batch with its trials; start one from **New run** there. Runs
 started from the CLI appear live, and the run status in the top bar shows what is running. The **Compare** page (`/compare`) shows what changed between two
-grading versions of a task, and what that did to the scores and to each criterion. A suite's description is shown with
-every chart that draws the suite.
+grading versions of a task, and what that did to the scores and to each criterion. Suite and task descriptions,
+what the agent gets and how each task is graded are on the **Tasks** page (`/tasks`). **How it works**
+(`/how-it-works`) explains the scores.
 Terms are defined in [`CONTEXT.md`](CONTEXT.md); the scoring design is
 [ADR-0001](docs/adr/0001-pass-fail-criteria-in-measures.md), and trials as records is
 [ADR-0006](docs/adr/0006-trials-as-records-charts-as-queries.md).
@@ -71,9 +76,28 @@ This repo is the engine. It holds no project: tests use the made-up project in `
 bun install
 bun test
 bun run typecheck
+bun run dev                                              # the dashboard on a temp copy of the sample project, with hot reload
 cd <a project> && bun <this repo>/src/cli.ts <command>   # the engine from this checkout, on that project
+bun scripts/check-drawer.ts                              # the trial drawer's security check, in a headless browser
+bun scripts/check-results.ts                             # the Results page's layout check, in a headless browser
+bun scripts/check-notices.ts                             # the Results page's one alert and its details, in a headless browser
 ```
 
+`static-fire serve` bundles the dashboard once, minified. `bun run dev` serves it in development mode instead:
+bundled on each request, with hot reload of `web/` and Bun's error overlay. The temp copy is removed when it
+stops, so it never changes a real project or record store. To use development mode on a real project, run
+`bun <this repo>/src/cli.ts serve --dev` in it.
+
+`scripts/check-drawer.ts` shows made-up HTML and Markdown outputs in the trial drawer and checks that they cannot reach the
+dashboard's page, style or API. It needs a Chromium or Chrome on the machine (set `CHROME` to its path when it is not
+found), runs it headless, and writes only in a temporary folder. It is not part of `bun test`.
+`scripts/check-results.ts` checks the Results page the same way on a made-up project with five tasks in three suites:
+one card with a task menu for the per-task table and one for the trials, and a page that does not grow with the
+number of tasks apart from the rows of its tables.
+`scripts/check-notices.ts` checks the same way that the notices are one alert with a short count, with details on a
+click or the keyboard (one item per kind, the configs not plotted grouped by reason, Regrade, the earlier version link
+and the How it works link), and that the chart card does not grow with the number of notices.
+
 ## Planned work
 
 Planned work is tracked in [GitHub issues](https://github.com/kenneth-liao/static-fire/issues). A design
@@ -90,7 +114,7 @@ Suite (e.g. Writing)                  Suite Index    = mean task score
     ├ Gates                           (output exists, frontmatter has a title, …)
     └ Measure (e.g. "Gets opened")    Measure score  = % of its criteria passed
        └ Criterion (yes/no, floor | strong | ceiling)
-Workflow Index = mean Suite Index over the Suites in view
+Agent Performance Index = mean Suite Index over the Suites in view
 ```
 
 - Judge criteria get several **votes** (3 by default). A criterion's score is its share of passing votes.
@@ -124,26 +148,8 @@ example a thumbnail scaled to 320 px wide.
   evidence: []            # optional: the judge sees only the derived image, not the full-size output
 ```
 
-The contract for a script:
-
-- **Name:** `derived:` is a path relative to the task's `grader/` folder, one script per criterion. Task
-  validation reports a script that does not exist, and `derived` on a check or human criterion.
-- **How it runs:** by extension. `.ts` and `.js` with the Bun that runs the engine (so a Node-only install
-  needs no `bun` on the `PATH`), `.py` with `uv run --script` (declare packages such as Pillow in the script's
-  inline `# /// script` header), `.sh` with `sh`.
-- **Input:** argument 1 and `$OUTPUT` are the trial's output file. `$TASK_DIR` is the task folder, so a script
-  can read other task files, for example `$TASK_DIR/project/youtube/thumbnail-regions.json`. `$WORKSPACE` is
-  the trial workspace, which is also the working folder. The time limit is 60 s.
-- **Output:** write one PNG to the path in argument 2 (also `$DERIVED`). A non-zero exit, a time-out, no file
-  or a file that is not a PNG makes the criterion's verdict an error, so the grading is invalid.
-- **Where it stays:** `runs/trials/<trial-id>/gradings/<grading-id>/_derived/<script>.png`, with the script's
-  output next to it as `.log`. A script at the top of `grader/` keeps its name (`scale-320.py.png`); in the file
-  name, `/` is written as `%2F` and any other character outside letters, digits and `+ = . _ -` as `%` and its
-  hex code, so two scripts never share a file (`sub/overlay.ts` gives `sub%2Foverlay.ts.png`).
-  This is the same for a run and a regrade.
-- **Hashing:** the script name is part of the criterion in the grading hash, and every file in `grader/` is
-  hashed, so a new name or new script content gives a new grading version. Criteria that name the same script
-  share the image and, when their other evidence is the same, one judge call.
+`static-fire guide` gives the contract for a script: its name, how it runs, its input and output, where
+the image stays, and how scripts share an image.
 
 ### Engine revision
 
@@ -183,7 +189,7 @@ one, and `static-fire guide` gives the format of each file.
 | Agent config | `configs/<id>.yaml` | `harness` (claude · codex · agy · pi), `model`, `effort`, optional `profile` (apkit); the label is built from these |
 | Human config | `configs/<id>.yaml` | `harness: human` and `outputs` (task id → the file you wrote): compete with the agents |
 | Judge config | `judges.yaml` | Judges to choose from per run or regrade (first = default) |
-| Suite | `suites/<id>.yaml` | Title, description (shown with the suite's charts) + list of task IDs |
+| Suite | `suites/<id>.yaml` | Title, description (shown on the Tasks page) + list of task IDs |
 | Task | `evals/<id>/task.yaml` | Prompt, inputs, project files, references and sources (graders only), gates, measures, criteria |
 | Records | `runs/static-fire.db` (`STATIC_FIRE_DB` overrides it) | The SQLite record store, the only copy of the records: batches, trials with their run conditions, gradings, verdicts, task versions, grading versions, user setups |
 | Trial artifacts | `runs/trials/<trial-id>/` | `workspace/` (the agent's output and files), `agent.log`, and `gradings/<grading-id>/` with the judge logs |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "static-fire",
-  "version": "0.44.0",
+  "version": "0.54.1",
   "type": "module",
   "description": "Benchmark agent configurations (harness + model + effort + profile) on your real workflows.",
   "license": "MIT",
@@ -14,6 +14,7 @@
     "pricing.yaml"
   ],
   "scripts": {
+    "dev": "bun scripts/dev.ts",
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {

package/src/adapters/index.ts

@@ -7,7 +7,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, symlinkSync, writeFil
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import type { AgentConfig, AgentHarness, AgentResult, Harness, Usage } from "../core/types";
-import { findProject, pricingPath } from "../core/project";
+import { currentProject, pricingPath } from "../core/project";
 
 export interface Guard { blockedTools: string[] }
 export interface RunArgs { config: AgentConfig; prompt: string; cwd: string; logPath: string; timeoutMs: number; guard?: Guard }
@@ -42,9 +42,9 @@ export function agyHome() {
 // engine:judgeCall:end
 
 const jsonl = (s: string) => s.split("\n").flatMap((l) => { try { return [JSON.parse(l)]; } catch { return []; } });
-// The rates: the pricing.yaml of the project found from the current folder when it has one, else the engine's.
+// The rates: the pricing.yaml of the project this command uses (the home, outside any project) when it has one, else the engine's.
 let pricing: any;
-const rates = () => (pricing ??= Bun.YAML.parse(readFileSync(pricingPath(findProject()), "utf8")));
+const rates = () => (pricing ??= Bun.YAML.parse(readFileSync(pricingPath(currentProject()), "utf8")));
 
 export function estimate(harness: Harness, model: string, u: Usage) {
   const rate = rates()[harness]?.[model];

package/src/api.ts

@@ -5,9 +5,10 @@ import { batchState } from "./core/batch-state";
 import { listBatchSummaries, summarizeBatch, type BatchSummary } from "./core/batches";
 import { openStore } from "./core/db";
 import type { Store } from "./core/db";
-import { listTasks, loadConfigs, loadJudges } from "./core/authored";
+import { listTasks, loadConfigs, loadJudges, loadTask } from "./core/authored";
 import { projectRoot } from "./core/project";
 import { readTrialOutputFile, trialDetail } from "./core/trial-detail";
+import { taskPage } from "./core/task-page";
 import type { Task } from "./core/types";
 import { compareGradings } from "./core/compare";
 import { buildFacets } from "./core/facets";
@@ -65,6 +66,28 @@ export interface RegradeEnded extends RegradeRun {
 
 const bad = (e: unknown) => Response.json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 });
 
+/**
+ * A request that may change something (any method but GET and HEAD) from another site: its `Origin` is not the
+ * page's own (the Host it was sent to), or `Sec-Fetch-Site` says cross-site. A trial's HTML output runs in a sandboxed
+ * frame whose requests carry `Origin: null`, and any web site open in the browser can post to localhost: neither may
+ * start a run or a regrade (#68). The CLI and tests send no Origin and pass.
+ */
+export function fromAnotherSite(req: Request): boolean {
+  if (req.method === "GET" || req.method === "HEAD") return false;
+  if (req.headers.get("sec-fetch-site") === "cross-site") return true;
+  const origin = req.headers.get("origin"), host = req.headers.get("host");
+  return origin !== null && (!host || (origin !== `http://${host}` && origin !== `https://${host}`));
+}
+const ANOTHER_SITE = "This request came from another site. The dashboard's API takes changes only from its own page.";
+
+/** Every handler of `routes`, refusing a request from another site (403) before it runs. */
+function ownPageOnly<R extends Record<string, unknown>>(routes: R): R {
+  type Handler = (req: Request, ...rest: unknown[]) => unknown;
+  const wrap = (h: Handler): Handler => (req, ...rest) => (fromAnotherSite(req) ? Response.json({ error: ANOTHER_SITE }, { status: 403 }) : h(req, ...rest));
+  return Object.fromEntries(Object.entries(routes).map(([path, r]) => [path, typeof r === "function" ? wrap(r as Handler)
+    : Object.fromEntries(Object.entries(r as Record<string, Handler>).map(([method, h]) => [method, wrap(h)]))])) as R;
+}
+
 /** The API routes, and `close` to release the event stream's database connection. */
 export function apiRoutes({ store, dbPath, catalog, pollMs = 1000, root: givenRoot, cacheDir, checkPlan, now = () => new Date(), checkRegrade, runRegrade = regrade }: ApiOptions) {
   const root = () => givenRoot ?? projectRoot();   // looked up per request, so a server built with only a catalog needs no project
@@ -82,6 +105,12 @@ export function apiRoutes({ store, dbPath, catalog, pollMs = 1000, root: givenRo
     "/api/judges": () => Response.json(loadJudges(root())),
     "/api/suites": () => Response.json(catalog().suites),   // the suites the view reads, so the scope selector matches it
     "/api/tasks": () => Response.json(listTasks(root())),
+    /** One task as the Tasks page shows it (core/task-page.ts): never the graders' own material. 404 for a task the project does not have, 400 for an invalid task file. */
+    "/api/tasks/:id": (req: Request & { params: { id: string } }) => {
+      const r = root(), id = req.params.id;
+      if (!listTasks(r).some((t) => t.id === id)) return Response.json({ error: `no task ${id}` }, { status: 404 });
+      try { return Response.json(taskPage(loadTask(id, r))); } catch (e) { return bad(e); }
+    },
     "/api/view": (req: Request) => {
       try { return Response.json(buildView(store, toViewFilters(parseViewQuery(new URL(req.url).searchParams)), catalog())); } catch (e) { return bad(e); }
     },
@@ -188,12 +217,15 @@ export function apiRoutes({ store, dbPath, catalog, pollMs = 1000, root: givenRo
       const d = trialDetail(store, req.params.id);
       return d ? Response.json(d) : Response.json({ error: `no trial ${req.params.id}` }, { status: 404 });
     },
-    /** The trial's output file from its saved workspace: an image with its media type, else Markdown text; 404 with the reason when there is none. */
+    /** The trial's output file from its saved workspace, with its output path in `x-output-path`: an image with its media type, else text (served as Markdown, never as a page); 404 with the reason when there is none. */
     "/api/trials/:id/output": (req: Request & { params: { id: string } }) => {
       const out = readTrialOutputFile(store, dbPath, req.params.id);
-      return "missing" in out ? new Response(out.missing, { status: 404, headers: { "content-type": "text/plain; charset=utf-8" } })
-        : "image" in out ? new Response(out.image, { headers: { "content-type": out.mediaType } })
-        : new Response(out.text, { headers: { "content-type": "text/markdown; charset=utf-8" } });
+      if ("missing" in out) return new Response(out.missing, { status: 404, headers: { "content-type": "text/plain; charset=utf-8" } });
+      // The output path (URI-encoded) tells the drawer how to show the text: an HTML page, Markdown or other text (#68).
+      const path = { "x-output-path": encodeURIComponent(out.path) };
+      return "image" in out ? new Response(out.image, { headers: { "content-type": out.mediaType, ...path } })
+        // Text a browser never runs as a page, even when this URL is opened on its own: the drawer frames it.
+        : new Response(out.text, { headers: { "content-type": "text/markdown; charset=utf-8", "x-content-type-options": "nosniff", "content-security-policy": "sandbox", ...path } });
     },
     /** Server-sent events: one on connect, then one each time the database changes. The data is the data_version. */
     "/api/events": () => {
@@ -214,5 +246,5 @@ export function apiRoutes({ store, dbPath, catalog, pollMs = 1000, root: givenRo
       return new Response(stream, { headers: { "content-type": "text/event-stream", "cache-control": "no-cache" } });
     },
   };
-  return { routes, close: () => watcher.close() };
+  return { routes: ownPageOnly(routes), close: () => watcher.close() };
 }

package/src/cli.ts

@@ -1,15 +1,21 @@
 #!/usr/bin/env bun
 // static-fire — the same primitives the dashboard uses, for people and agents.
 //   static-fire --version                   the installed version
-//   static-fire init [--json]               make a project in the current folder (core/init.ts); never overwrites a file
+//   static-fire init [<folder>] [--json]    make the home (~/static-fire, or <folder>), report the home that is set, or
+//                                           make a second project in <folder> (core/init.ts); never overwrites a file
 //   static-fire guide                       the guide for agents (src/guide.md); needs no project
-//   static-fire serve [--port 4477]
+//   static-fire home [<folder>]             show the home, or save a project folder as the home; needs no project
+//   static-fire serve [--port 4477] [--dev]
+//                                           the dashboard, in production mode; --dev for development mode (hot reload)
 //   static-fire configs | judges | suites | tasks
 //   static-fire new task <id>               a new task folder: evals/<id>/task.yaml (commented, to fill in) and inputs/
 //   static-fire run --suites writing[,visual] | --tasks a[,b]  [--configs a,b|all] [--trials 3] [--judge id] [--votes 3] [--concurrency 4]
 //                                           one batch in the store; trial artifacts in the project's runs/trials/<trial-id>/
 //   static-fire regrade [--tasks t] [--configs a,b] [--batch id] [--judge id] [--votes n] [--only-invalid] [--concurrency 4]
 //                                           grade stored outputs again (judge calls only) where the current grading is missing
+//   static-fire run|regrade ... --plan [--json]
+//                                           the plan in runs (core/plan.ts): agent runs per config and judge calls; starts
+//                                           nothing and writes nothing. Without --plan, run and regrade print it first.
 //   static-fire rerun <batch>               run its failed trials again, as new trials in the same batch
 //   static-fire list                        every batch in the store, newest first, with its trial counts
 //   static-fire show [--suites s | --tasks t] [--judge id] [--grading hash] [--since date] [--until date] [--json]
@@ -18,17 +24,20 @@
 //   static-fire migrate [--from <folder>]   one-off import of the 2026-11-06 experiment into the store
 //
 // The commands and the flags each takes are listed in core/commands.ts; a flag a command does not take is an error.
-// The project is the folder found by walking up from the current folder to static-fire.yaml (core/project.ts).
-// Each command that needs one calls needProject() before it reads or writes anything; outside a project it
-// stops with an error that names `static-fire init`. The record store is the project's runs/static-fire.db,
+// The project is the nearest folder at or above the current folder with static-fire.yaml; outside any project it
+// is the saved home (core/project.ts, ADR-0010). Each command that needs one calls needProject() before it reads
+// or writes anything. When it uses the home, it says so in one line on stderr, so stdout (and --json) is
+// unchanged. With no home it stops with an error that names `static-fire init`, and when the saved home is not a
+// project any more, with an error that says how to set it again. The record store is the project's runs/static-fire.db,
 // or STATIC_FIRE_DB when set; trial artifacts go next to the store, and the Profile cache in the project's .cache/.
-import { readFileSync, renameSync, writeFileSync } from "node:fs";
+import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { parseArgs } from "node:util";
 import { configLabel } from "./core/label";
+import { coverageLine } from "./core/coverage";
 import { listTasks, loadConfigs, loadJudges, loadSuites } from "./core/authored";
 import { COMMANDS, OPTIONS, unknownFlags, USAGE } from "./core/commands";
-import { ENGINE_ROOT, NoProjectError, projectPaths, projectRoot, type ProjectPaths } from "./core/project";
+import { ENGINE_ROOT, HomeNotProjectError, NoProjectError, projectPaths, resolveProject, type ProjectPaths } from "./core/project";
 
 const LABEL_WIDTH = 52;   // fits the longest built config label, so `show` columns line up
 
@@ -47,25 +56,57 @@ if (cmd && COMMANDS[cmd]) {
     process.exit(1);
   }
 }
+// run and regrade print the plan as JSON only on its own: their progress lines are not JSON.
+if ((cmd === "run" || cmd === "regrade") && values.json && !values.plan) {
+  console.error(`${cmd}: --json needs --plan (static-fire ${cmd} ... --plan --json prints the plan as JSON and starts nothing)`);
+  process.exit(1);
+}
 
 const fmt = (n: number | null | undefined, d = 1) => (n == null ? "—" : n.toFixed(d));
 const log = (m: string) => console.log(m);
 
-/** The project of this command, or exit 1 with an error that names `static-fire init`, before anything is written. */
+let saidHome = false;
+/**
+ * The project of this command, or exit 1 before anything is written: with an error that names `static-fire init`
+ * when there is no project and no home, or that says how to set the home again when it is not a project any more.
+ * A command that uses the home from outside it says so, once, on stderr.
+ */
 function needProject(): ProjectPaths {
+  let found;
   try {
-    return projectPaths(projectRoot());
+    found = resolveProject();
   } catch (e) {
-    if (!(e instanceof NoProjectError)) throw e;
+    if (!(e instanceof NoProjectError || e instanceof HomeNotProjectError)) throw e;
     console.error(`${cmd}: ${e.message}`);
     process.exit(1);
   }
+  if (found.home && !saidHome) {
+    saidHome = true;
+    console.error(`using the home project: ${found.root}`);
+  }
+  return projectPaths(found.root);
 }
 
 switch (cmd) {
   case "init": {
-    const { formatReport, initProject } = await import("./core/init");
-    const report = initProject();
+    // Needs no project: it makes one, or reports the home or the project it runs in.
+    const { formatReport, NotAFolderError, runInit } = await import("./core/init");
+    if (positionals.length > 1) {
+      console.error("usage: static-fire init [<folder>] [--json]");
+      process.exit(1);
+    }
+    let report;
+    try {
+      report = runInit({ folder: positionals[0] });
+    } catch (e) {
+      if (e instanceof HomeNotProjectError) {
+        console.error(`init: ${e.message} Or run \`static-fire init <folder>\` to make a new project there, then \`static-fire home <folder>\`.`);
+        process.exit(1);
+      }
+      if (!(e instanceof NotAFolderError)) throw e;
+      console.error(`init: ${e.message}`);
+      process.exit(1);
+    }
     process.stdout.write(values.json ? `${JSON.stringify(report, null, 2)}\n` : formatReport(report));
     break;
   }
@@ -73,10 +114,41 @@ switch (cmd) {
     // An engine file, so it matches the installed version. It needs no project.
     process.stdout.write(readFileSync(join(ENGINE_ROOT, "src", "guide.md"), "utf8"));
     break;
+  case "home": {
+    // Needs no project. With a folder, saves it as the home when it is a project; the home changes only here.
+    const { NotAProjectError, readHome, saveHome, PROJECT_FILE } = await import("./core/project");
+    if (positionals.length > 1) {
+      console.error("usage: static-fire home [<folder>]");
+      process.exit(1);
+    }
+    const folder = positionals[0];
+    if (folder !== undefined) {
+      try {
+        log(`the home is now ${saveHome(folder)}`);
+      } catch (e) {
+        if (!(e instanceof NotAProjectError)) throw e;
+        console.error(`home: ${e.message}`);
+        process.exit(1);
+      }
+      break;
+    }
+    const home = readHome();
+    if (!home) {
+      log("no home is set. Run `static-fire home <folder>` with a project folder to set one, or `static-fire init` to make a project.");
+      break;
+    }
+    log(home);
+    const { existsSync } = await import("node:fs");
+    if (!existsSync(join(home, PROJECT_FILE))) {
+      console.error(`home: ${new HomeNotProjectError(home).message}`);
+      process.exitCode = 1;
+    }
+    break;
+  }
   case "serve": {
     const project = needProject();
     const { serve } = await import("./server");
-    serve(Number(values.port), project);
+    serve(Number(values.port), project, { development: values.dev === true });
     break;
   }
   case "configs":
@@ -123,6 +195,10 @@ switch (cmd) {
       tasks: list(values.tasks), suites: values.suites === "all" ? undefined : list(values.suites), configs: list(values.configs),
       trials: Number(values.trials), judge: values.judge, votes: values.votes ? Number(values.votes) : undefined,
     }, root);
+    const { batchPlanInRuns, formatPlan } = await import("./core/plan");
+    const inRuns = batchPlanInRuns(plan);
+    if (values.plan) { log(values.json ? JSON.stringify(inRuns, null, 2) : formatPlan(inRuns)); break; }
+    log(formatPlan(inRuns));
     const store = openStore(dbPath);
     try {
       const started = await startBatch(store, plan, { dbPath, root, cacheDir: cache });
@@ -135,16 +211,24 @@ switch (cmd) {
   }
   case "regrade": {
     const { root, dbPath } = needProject();
-    const { formatCounts, regrade } = await import("./core/regrade");
-    const { openStore } = await import("./core/db");
+    const { formatCounts, planRegrade, regrade } = await import("./core/regrade");
+    const { emptyStore, openStore, snapshotStore } = await import("./core/db");
     const { projectCatalog } = await import("./core/view");
+    const { formatPlan, regradePlanInRuns } = await import("./core/plan");
     const list = (v?: string) => (v === undefined ? undefined : v.split(",").map((x) => x.trim()).filter(Boolean));
-    const store = openStore(dbPath);
+    const filters = { tasks: list(values.tasks), configs: list(values.configs), batch: values.batch };
+    const opts = {
+      dbPath, catalog: projectCatalog(root), judge: values.judge, votes: values.votes ? Number(values.votes) : undefined, onlyInvalid: values["only-invalid"],
+      concurrency: Number(values.concurrency), onEvent: log,
+    };
+    // The plan writes nothing: it is planned on a copy of the store in memory (an empty one when there is no store
+    // yet), so no file is made and an existing store keeps its schema and content.
+    const store = !values.plan ? openStore(dbPath) : existsSync(dbPath) ? snapshotStore(dbPath) : emptyStore();
     try {
-      const counts = await regrade(store, { tasks: list(values.tasks), configs: list(values.configs), batch: values.batch }, {
-        dbPath, catalog: projectCatalog(root), judge: values.judge, votes: values.votes ? Number(values.votes) : undefined, onlyInvalid: values["only-invalid"],
-        concurrency: Number(values.concurrency), onEvent: log,
-      });
+      const inRuns = regradePlanInRuns(planRegrade(store, filters, opts));
+      if (values.plan) { log(values.json ? JSON.stringify(inRuns, null, 2) : formatPlan(inRuns)); break; }
+      log(formatPlan(inRuns));
+      const counts = await regrade(store, filters, opts);
       log(formatCounts(counts));
       if (counts.failed) {
         console.error(`regrade: ${counts.failed} trial(s) could not be graded (listed above)`);
@@ -255,6 +339,8 @@ function printView(v: View) {
   const { filters: f } = v;
   const range = f.since || f.until ? ` · ${f.since?.slice(0, 10) ?? "…"} to ${f.until?.slice(0, 10) ?? "…"}` : "";
   log(`${f.suiteIds.length ? `suites ${f.suiteIds.join(", ")}` : `tasks ${f.taskIds.join(", ")}`} · judge ${f.judge.label}${range}`);
+  const covers = coverageLine(v.coverage, f.judge.label);   // the line the dashboard shows under its index buttons
+  if (covers) log(covers);
   for (const [id, t] of Object.entries(v.tasks)) {
     const grading = t.gradingName ? `${t.gradingName} (${t.gradingHash.slice(0, 8)})` : t.gradingHash.slice(0, 8);
     log(`  ${id}${t.label != null ? ` v${t.label}` : ""} · task ${t.taskHash.slice(0, 8)} · grading ${grading}${t.current ? "" : " (earlier)"}`);

package/src/core/batch.ts

@@ -164,6 +164,19 @@ export async function harnessVersions(harnesses: AgentHarness[], read = readHarn
 /** A human wrote one file per task, so it gets one trial, and only for tasks it has a file for. */
 const trialCount = (c: AgentConfig, taskId: string, n: number) => (c.harness !== "human" ? n : c.outputs?.[taskId] ? 1 : 0);
 
+/** One trial a batch starts: its task, config and number. */
+export interface PlannedTrial { task: Task; config: AgentConfig; n: number }
+
+/**
+ * Every trial of a planned batch, in the order startBatch queues them: each task, each config, n = 1..trials (one
+ * for a human config with a file for the task, none without). startBatch inserts this list, and the plan in runs
+ * (plan.ts) counts it, so the counts are what starts.
+ */
+export function plannedTrials(plan: Pick<BatchPlan, "tasks" | "configs" | "trials">): PlannedTrial[] {
+  return plan.tasks.flatMap((task) => plan.configs.flatMap((config) =>
+    Array.from({ length: trialCount(config, task.id, plan.trials) }, (_, i) => ({ task, config, n: i + 1 }))));
+}
+
 /**
  * Records a new batch in one transaction: the batch (id = UTC timestamp, with -2, -3, ... when a batch
  * started in the same second has it), each task version and grading setup when first seen, the user setup,
@@ -198,7 +211,9 @@ export async function startBatch(store: Store, plan: BatchPlan, o: StartOptions)
     }
     if (files && setupHash) store.putUserSetup({ hash: setupHash, files });
     const trials: TrialRecord[] = [];
-    for (const { task, t } of ids) for (const c of plan.configs) for (let n = 1; n <= trialCount(c, task.id, plan.trials); n++) {
+    const hashes = new Map(ids.map(({ task, t }) => [task.id, t]));
+    for (const { task, config: c, n } of plannedTrials(plan)) {
+      const t = hashes.get(task.id)!;
       const tid = `${id}--${task.id}--${c.id}--${n}`;
       const r: TrialRecord = {
         id: tid, batchId: id, taskId: task.id, taskHash: t.hash, configKey: configKey(c), configId: c.id,

package/src/core/commands.ts

@@ -7,24 +7,25 @@ export const OPTIONS = {
   suites: { type: "string" }, tasks: { type: "string" }, grading: { type: "string" }, since: { type: "string" }, until: { type: "string" },
   configs: { type: "string" }, trials: { type: "string", default: "3" },
   judge: { type: "string" }, votes: { type: "string" }, "only-invalid": { type: "boolean" },
-  concurrency: { type: "string", default: "4" }, port: { type: "string", default: "4477" }, json: { type: "boolean" },
-  from: { type: "string" }, batch: { type: "string" },
+  concurrency: { type: "string", default: "4" }, port: { type: "string", default: "4477" }, dev: { type: "boolean" }, json: { type: "boolean" },
+  from: { type: "string" }, batch: { type: "string" }, plan: { type: "boolean" },
 } as const;
 
 export type Flag = keyof typeof OPTIONS;
 
 /** Each command: the flags its code reads, and its arguments for the usage text. */
 export const COMMANDS: Record<string, { flags: readonly Flag[]; args?: string }> = {
-  init: { flags: ["json"] },
+  init: { flags: ["json"], args: "[<folder>]" },
   guide: { flags: [] },
-  serve: { flags: ["port"] },
+  home: { flags: [], args: "[<folder>]" },
+  serve: { flags: ["port", "dev"] },
   configs: { flags: [] },
   judges: { flags: [] },
   suites: { flags: [] },
   tasks: { flags: [] },
   new: { flags: [], args: "task <id>" },
-  run: { flags: ["suites", "tasks", "configs", "trials", "judge", "votes", "concurrency"] },
-  regrade: { flags: ["tasks", "configs", "batch", "judge", "votes", "only-invalid", "concurrency"] },
+  run: { flags: ["suites", "tasks", "configs", "trials", "judge", "votes", "concurrency", "plan", "json"] },
+  regrade: { flags: ["tasks", "configs", "batch", "judge", "votes", "only-invalid", "concurrency", "plan", "json"] },
   rerun: { flags: ["concurrency"], args: "<batch>" },
   list: { flags: [] },
   show: { flags: ["suites", "tasks", "judge", "grading", "since", "until", "json"] },

package/src/core/compare.ts

@@ -117,7 +117,10 @@ export function compareGradings(store: Store, filters: ViewFilters, a: string, b
   if (sa.taskId !== sb.taskId) throw new Error(`grading versions ${sa.hash.slice(0, 8)} and ${sb.hash.slice(0, 8)} are of two tasks (${sa.taskId}, ${sb.taskId}); compare two grading versions of one task`);
   const taskId = sa.taskId;
   // Each setup brings its own judge (the view refuses another judge with a picked setup).
-  const { judge: _judge, ...rest } = filters;
+  // With no scope given, the scope is the task: only its trials are compared, and the default set of suites
+  // (DEC-019) may leave its suite out.
+  const { judge: _judge, ...given } = filters;
+  const rest = given.suites?.length || given.tasks?.length ? given : { ...given, tasks: [taskId] };
   const va = buildView(store, { ...rest, grading: sa.hash }, catalog);   // throws when the task is not in scope
   const vb = buildView(store, { ...rest, grading: sb.hash }, catalog);
   const ofTask = (v: View) => new Map(v.trials.filter((t) => t.taskId === taskId && valid(t)).map((t) => [t.id, t]));