static-fire 0.0.0-stage → 0.54.1

package/src/core/project.ts

@@ -0,0 +1,140 @@
+// The two roots. The engine root holds the engine's own files: its source, the dashboard and pricing.yaml. It
+// comes from this file's location, never from the current folder, so it holds when the engine runs from
+// node_modules. The project root is the user's folder: static-fire.yaml, configs/, judges.yaml, suites/,
+// evals/, the record store and trial artifacts in runs/, and the Profile cache in .cache/. A command uses the
+// nearest project at or above the current folder; outside any project it uses the home, the one project saved in
+// the user's config folder (ADR-0010). Nothing of a project is written to the engine root.
+//
+// This module is the one place that finds the project a command uses and that reads and writes the saved home.
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+
+export const ENGINE_ROOT = resolve(import.meta.dir, "../..");
+export const PROJECT_FILE = "static-fire.yaml";
+
+/** A command that needs a project was run where no static-fire.yaml is in the folder or any folder above it. */
+export class NoProjectError extends Error {
+  constructor(from: string) {
+    super(`no Static Fire project here: there is no ${PROJECT_FILE} in ${from} or in a folder above it. Run \`static-fire init\` to make a project.`);
+    this.name = "NoProjectError";
+  }
+}
+
+/** The saved home is not a project any more: its folder is gone, or has no static-fire.yaml. */
+export class HomeNotProjectError extends Error {
+  constructor(readonly home: string) {
+    super(`the home, ${home}, is not a Static Fire project any more: there is no ${PROJECT_FILE} in it. Run \`static-fire home <folder>\` with a project folder to set the home again.`);
+    this.name = "HomeNotProjectError";
+  }
+}
+
+/** A folder given as the home holds no static-fire.yaml. */
+export class NotAProjectError extends Error {
+  constructor(folder: string) {
+    super(`${folder} is not a Static Fire project: there is no ${PROJECT_FILE} in it. The home is not changed.`);
+    this.name = "NotAProjectError";
+  }
+}
+
+export type Env = Record<string, string | undefined>;
+
+/**
+ * The file that holds the saved home: XDG_CONFIG_HOME/static-fire/home when XDG_CONFIG_HOME is an absolute path,
+ * else HOME/.config/static-fire/home. It holds one line, the home's absolute folder, and nothing else. HOME is read
+ * from `env`, because Bun's os.homedir() does not see a HOME changed while the process runs (tests move it).
+ */
+export function homeFile(env: Env = process.env): string {
+  const xdg = env.XDG_CONFIG_HOME;
+  const config = xdg && isAbsolute(xdg) ? xdg : join(userFolder(env), ".config");
+  return join(config, "static-fire", "home");
+}
+
+/** The user's own folder: HOME from `env`, for the reason homeFile gives. */
+function userFolder(env: Env): string {
+  return env.HOME || homedir();
+}
+
+/** Where the first `init` outside any project makes the home: HOME/static-fire. XDG_CONFIG_HOME does not move it. */
+export function defaultHomeFolder(env: Env = process.env): string {
+  return join(userFolder(env), "static-fire");
+}
+
+/** The saved home's folder, or null when none is saved (no file, or an empty one). */
+export function readHome(env: Env = process.env): string | null {
+  const file = homeFile(env);
+  if (!existsSync(file)) return null;
+  return readFileSync(file, "utf8").trim() || null;
+}
+
+/** Saves `folder` as the home when it holds static-fire.yaml; else throws NotAProjectError and changes nothing. */
+export function saveHome(folder: string, env: Env = process.env): string {
+  const root = resolve(folder);
+  if (!existsSync(join(root, PROJECT_FILE))) throw new NotAProjectError(root);
+  const file = homeFile(env);
+  mkdirSync(dirname(file), { recursive: true });
+  const tmp = `${file}.${process.pid}.tmp`;   // a reader never sees half a file
+  writeFileSync(tmp, `${root}\n`);
+  renameSync(tmp, file);
+  return root;
+}
+
+/** The project a command uses: `home` is true when it is the saved home, used from outside any project. */
+export interface ResolvedProject { root: string; home: boolean }
+
+/**
+ * The nearest project from `from` upward; outside any project, the saved home. Throws NoProjectError when there
+ * is neither, and HomeNotProjectError when the saved home is not a project any more. Reads only.
+ */
+export function resolveProject(from = process.cwd(), env: Env = process.env): ResolvedProject {
+  const near = findProject(from);
+  if (near) return { root: near, home: false };
+  const home = readHome(env);
+  if (!home) throw new NoProjectError(resolve(from));
+  if (!existsSync(join(home, PROJECT_FILE))) throw new HomeNotProjectError(home);
+  return { root: home, home: true };
+}
+
+/** The project resolveProject finds, or null where it would throw. */
+export function currentProject(from = process.cwd(), env: Env = process.env): string | null {
+  try {
+    return resolveProject(from, env).root;
+  } catch (e) {
+    if (e instanceof NoProjectError || e instanceof HomeNotProjectError) return null;
+    throw e;
+  }
+}
+
+/** The first folder from `from` upward that holds static-fire.yaml, or null. Does not read the home. */
+export function findProject(from = process.cwd()): string | null {
+  for (let dir = resolve(from); ; dir = dirname(dir)) {
+    if (existsSync(join(dir, PROJECT_FILE))) return dir;
+    if (dirname(dir) === dir) return null;
+  }
+}
+
+/** The root of the project resolveProject finds from `from`. Throws as resolveProject does. */
+export function projectRoot(from = process.cwd()): string {
+  return resolveProject(from).root;
+}
+
+export interface ProjectPaths {
+  root: string;
+  /** The record store's default folder and the trial artifacts' folder (runs/trials/<trial-id>). */
+  runs: string;
+  /** The Profile cache and its lock file. */
+  cache: string;
+  /** The record store: STATIC_FIRE_DB when set, else runs/static-fire.db. Trial artifacts go next to it. */
+  dbPath: string;
+}
+
+export function projectPaths(root: string): ProjectPaths {
+  const runs = join(root, "runs");
+  return { root, runs, cache: join(root, ".cache"), dbPath: process.env.STATIC_FIRE_DB || join(runs, "static-fire.db") };
+}
+
+/** The pricing file: the project's own pricing.yaml when it has one, else the engine's. */
+export function pricingPath(project: string | null): string {
+  const own = project ? join(project, "pricing.yaml") : null;
+  return own && existsSync(own) ? own : join(ENGINE_ROOT, "pricing.yaml");
+}

package/src/core/regrade.ts

@@ -0,0 +1,242 @@
+// Regrade: grade saved outputs again, with judge calls only. No agent runs, and no trial or artifact is copied.
+//
+// It covers every matching trial that has the current task hash and no valid grading under the current grading
+// hash for the chosen judge. A different judge is just another grading hash on the same trials. A regrade adds a
+// grading row and never changes an earlier one; the latest finished one counts (db.ts currentGrading).
+//
+// A grading left in status `grading` (for example by a crashed process) is not finished, so it is not a valid
+// grading and the trial is graded again. The stale row is left as it is: the store cannot tell it from a grading
+// another process is doing right now.
+import { DEFAULT_VOTES, gradeTrial } from "./batch";
+import { judgeCallRange } from "./plan";
+import type { Store, TrialRecord } from "./db";
+import { gradingIdentity, taskIdentity, configKey } from "./identity";
+import { loadConfigs } from "./authored";
+import type { AgentConfig, JudgeConfig, Task } from "./types";
+import { projectCatalog, type Catalog } from "./view";
+
+/** Which trials to look at. Every field is optional; an empty object is every trial in the store. */
+export interface RegradeFilters {
+  /** Task ids. */
+  tasks?: string[];
+  /** Config ids (files in configs/, resolved to their config key) or config keys. Trials match by config key. "all": every config. */
+  configs?: string[];
+  /** One batch id. */
+  batch?: string;
+  /** Trial ids. A view's regrade passes the ids it counts as gradable (view.ts LeftOut.gradable). */
+  trials?: string[];
+}
+
+export interface RegradeOptions {
+  /** The store's database path: trial folders are beside it. */
+  dbPath: string;
+  /** Today's task and judge definitions. Default: the YAML files in the repo. */
+  catalog?: Catalog;
+  /** Today's config files, to resolve config ids in `filters.configs`. Default: configs/ in the repo. */
+  configs?: AgentConfig[];
+  /** Judge id. Default: the first in judges.yaml. */
+  judge?: string;
+  /** Votes per judge criterion. Default DEFAULT_VOTES. */
+  votes?: number;
+  /** Only trials whose latest finished grading under the current grading hash is invalid. */
+  onlyInvalid?: boolean;
+  /** Gradings at once. Default 4. */
+  concurrency?: number;
+  onEvent?: (msg: string) => void;
+  /** Called once per grading added, with how it ended. The server counts a running regrade's progress with it. */
+  onGraded?: (trialId: string, status: "done" | "invalid") => void;
+}
+
+/** Matching trials that are not graded, by reason. */
+export interface RegradeSkipped {
+  /** Ran on another task hash: a different job. Run the task again instead. */
+  otherTask: number;
+  /** Did not end done (failed, copied the held-out original, or not ended): there is no output to grade. */
+  notDone: number;
+  /** Already has a valid (done) grading under the current grading hash for the judge. */
+  alreadyGraded: number;
+  /** With onlyInvalid: its latest finished grading under the current grading hash is not invalid (or there is none). */
+  notInvalid: number;
+  /** Its task no longer loads from today's files. */
+  unknownTask: number;
+}
+
+/** What a regrade would do. Plain data: no write happens to make it. */
+export interface RegradePlan {
+  judge: JudgeConfig;
+  votes: number;
+  /** Trials that passed the filters. */
+  matched: number;
+  /** The trials to grade, in id order, with the grading hash each gets. */
+  todo: { trialId: string; taskId: string; gradingHash: string }[];
+  skipped: RegradeSkipped;
+  /** The judge calls grading `todo` makes, fewest and most (plan.ts judgeCallRange). */
+  judgeCalls: { min: number; max: number };
+}
+
+export interface RegradeCounts {
+  matched: number;
+  /** Gradings added, and how they ended. */
+  graded: number;
+  done: number;
+  invalid: number;
+  /** Trials that could not be graded (a store error); the other trials were graded. */
+  failed: number;
+  failures: { trialId: string; error: string }[];
+  skipped: RegradeSkipped;
+}
+
+/** The failures of a regrade as one message, or null when every trial was graded. */
+export function failureMessage(c: Pick<RegradeCounts, "failed" | "failures">): string | null {
+  if (!c.failed) return null;
+  return `${c.failed} trial${c.failed === 1 ? "" : "s"} could not be graded: ${c.failures.map((f) => `${f.trialId}: ${f.error}`).join("; ")}`;
+}
+
+const positiveInt = (n: number, what: string) => {
+  if (!Number.isInteger(n) || n < 1) throw new Error(`${what} must be a whole number of 1 or more (got ${n})`);
+  return n;
+};
+
+interface Resolved { plan: RegradePlan; trials: Map<string, TrialRecord>; tasks: Map<string, Task> }
+
+function resolve(store: Store, f: RegradeFilters, o: RegradeOptions): Resolved {
+  const catalog = o.catalog ?? projectCatalog();
+  const judge = o.judge ? catalog.judges.find((j) => j.id === o.judge) : catalog.judges[0];
+  if (!judge) throw new Error(`unknown judge ${o.judge ?? "(judges.yaml lists none)"}`);
+  const votes = positiveInt(o.votes ?? DEFAULT_VOTES, "votes");
+  if (f.batch !== undefined && !store.getBatch(f.batch)) throw new Error(`unknown batch ${f.batch}`);
+
+  let trials = store.listTrials(f.batch === undefined ? {} : { batchId: f.batch });
+  if (f.tasks) {
+    const want = new Set(f.tasks);
+    const known = new Set(store.listTrials().map((t) => t.taskId));
+    const unknown = [...want].filter((id) => !known.has(id) && !canLoad(catalog, id));
+    if (!want.size || unknown.length) throw new Error(`unknown or no task(s): ${unknown.join(", ") || "none given"}`);
+    trials = trials.filter((t) => want.has(t.taskId));
+  }
+  if (f.trials) {
+    const want = new Set(f.trials);
+    const known = new Set(store.listTrials().map((t) => t.id));
+    const unknown = [...want].filter((id) => !known.has(id));
+    if (!want.size || unknown.length) throw new Error(`unknown or no trial(s): ${unknown.join(", ") || "none given"}`);
+    trials = trials.filter((t) => want.has(t.id));
+  }
+  if (f.configs && !f.configs.includes("all")) {
+    const byId = new Map((o.configs ?? loadConfigs()).map((c) => [c.id, configKey(c)]));
+    const seenKeys = new Set(store.listTrials().map((t) => t.configKey));
+    const keys = new Set<string>(), unknown: string[] = [];
+    for (const v of f.configs) {
+      const key = byId.get(v) ?? (seenKeys.has(v) ? v : undefined);
+      if (key) keys.add(key); else unknown.push(v);
+    }
+    if (!f.configs.length || unknown.length) throw new Error(`unknown or no config(s): ${unknown.join(", ") || "none given"}`);
+    trials = trials.filter((t) => keys.has(t.configKey));
+  }
+
+  const skipped: RegradeSkipped = { otherTask: 0, notDone: 0, alreadyGraded: 0, notInvalid: 0, unknownTask: 0 };
+  const current = new Map<string, { task: Task; taskHash: string; gradingHash: string } | null>();
+  const hashesOf = (taskId: string) => {
+    if (!current.has(taskId)) {
+      try {
+        const task = catalog.loadTask(taskId);
+        current.set(taskId, { task, taskHash: taskIdentity(task).hash, gradingHash: gradingIdentity(task, judge).hash });
+      } catch {
+        current.set(taskId, null);
+      }
+    }
+    return current.get(taskId)!;
+  };
+
+  const todo: RegradePlan["todo"] = [];
+  for (const t of trials) {
+    const h = hashesOf(t.taskId);
+    if (!h) { skipped.unknownTask++; continue; }
+    if (t.taskHash !== h.taskHash) { skipped.otherTask++; continue; }
+    if (t.status !== "done") { skipped.notDone++; continue; }
+    const latest = store.currentGrading(t.id, h.gradingHash);
+    if (o.onlyInvalid) {
+      if (latest?.status !== "invalid") { skipped.notInvalid++; continue; }
+    } else if (latest?.status === "done") { skipped.alreadyGraded++; continue; }
+    todo.push({ trialId: t.id, taskId: t.taskId, gradingHash: h.gradingHash });
+  }
+  const tasks = new Map([...current].flatMap(([id, h]) => (h ? [[id, h.task] as const] : [])));
+  const judgeCalls = { min: 0, max: 0 };
+  for (const x of todo) {
+    const r = judgeCallRange(tasks.get(x.taskId)!, votes);
+    judgeCalls.min += r.min;
+    judgeCalls.max += r.max;
+  }
+  return { plan: { judge, votes, matched: trials.length, todo, skipped, judgeCalls }, trials: new Map(trials.map((t) => [t.id, t])), tasks };
+}
+
+function canLoad(catalog: Catalog, id: string) {
+  try { catalog.loadTask(id); return true; } catch { return false; }
+}
+
+/**
+ * Which trials a regrade would grade, and why the others are skipped. Writes nothing. Throws on an unknown
+ * task, config, batch or judge, or votes below 1.
+ */
+export function planRegrade(store: Store, filters: RegradeFilters, opts: RegradeOptions): RegradePlan {
+  return resolve(store, filters, opts).plan;
+}
+
+/**
+ * Grades every trial planRegrade selects with gradeTrial (no agent run), and returns the counts. Records the
+ * task version and grading setup first when they are new (an edited criterion, or another judge). A trial that
+ * cannot be graded (a store error) does not stop the others: it is counted in `failed` with its error.
+ */
+export async function regrade(store: Store, filters: RegradeFilters, opts: RegradeOptions): Promise<RegradeCounts> {
+  const log = opts.onEvent ?? (() => {});
+  const { plan, trials, tasks } = resolve(store, filters, opts);
+  const counts: RegradeCounts = { matched: plan.matched, graded: 0, done: 0, invalid: 0, failed: 0, failures: [], skipped: plan.skipped };
+  const at = new Date().toISOString();
+  store.transaction(() => {
+    for (const taskId of new Set(plan.todo.map((x) => x.taskId))) {
+      const task = tasks.get(taskId)!;
+      const t = taskIdentity(task), g = gradingIdentity(task, plan.judge);
+      store.putTaskVersion({ hash: t.hash, taskId, label: task.version ?? null, snapshot: t.snapshot as unknown as Record<string, unknown>, firstSeen: at });
+      store.putGradingSetup({
+        hash: g.hash, taskId, taskHash: t.hash,
+        judgeHarness: plan.judge.harness, judgeModel: plan.judge.model, judgeEffort: plan.judge.effort ?? null, judgeLabel: plan.judge.label,
+        snapshot: g.snapshot as unknown as Record<string, unknown>, firstSeen: at,
+      });
+    }
+  });
+
+  const queue = [...plan.todo];
+  const worker = async () => {
+    for (let x = queue.shift(); x; x = queue.shift()) {
+      try {
+        const g = await gradeTrial(store, opts.dbPath, trials.get(x.trialId)!, { task: tasks.get(x.taskId)!, judge: plan.judge, votes: plan.votes, gradingHash: x.gradingHash });
+        counts.graded++;
+        if (g.status === "done") counts.done++; else counts.invalid++;
+        opts.onGraded?.(x.trialId, g.status === "done" ? "done" : "invalid");
+        log(`${g.status === "done" ? "✓" : "!"} ${x.trialId} score ${g.score?.toFixed(1) ?? "—"}${g.error ? ` (${g.error})` : ""}`);
+      } catch (e) {
+        // A store error: the other trials go on, and the failure is returned with the counts.
+        const error = e instanceof Error ? e.message : String(e);
+        counts.failed++;
+        counts.failures.push({ trialId: x.trialId, error });
+        log(`✗ ${x.trialId}: ${error}`);
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.max(1, opts.concurrency ?? 4) }, worker));
+  return counts;
+}
+
+/** What the CLI prints: one line of what was graded and skipped, by reason, then one line per trial that could not be graded. */
+export function formatCounts(c: RegradeCounts): string {
+  const s = c.skipped;
+  const parts = [
+    s.otherTask && `${s.otherTask} on another task version`,
+    s.alreadyGraded && `${s.alreadyGraded} already graded`,
+    s.notInvalid && `${s.notInvalid} not invalid`,
+    s.notDone && `${s.notDone} not done`,
+    s.unknownTask && `${s.unknownTask} with an unknown task`,
+  ].filter(Boolean);
+  const failed = c.failed ? ` · ${c.failed} could not be graded` : "";
+  return `regraded ${c.graded} trial(s) (${c.done} done, ${c.invalid} invalid) of ${c.matched} matching${failed} · skipped: ${parts.join(", ") || "none"}`
+    + c.failures.map((f) => `\n  ${f.trialId}: ${f.error}`).join("");
+}

package/src/core/score.ts

@@ -0,0 +1,148 @@
+// Pure scoring (ADR-0001). Verdicts → measure % → task score → Suite Index → Agent Performance Index.
+// No I/O here, so every rule is covered by tests/score.test.ts.
+import { configLabel, EFFORTS } from "./label";
+import type { ConfigSummary, Experiment, ExperimentSummary, Grades, Stat, Task, TaskSummary, Trial, Verdict, Vote } from "./types";
+
+// engine:judging:begin
+export function verdictFromVotes(votes: Vote[], error?: string): Verdict {
+  if (!votes.length) return { score: null, votes, error: error ?? "no votes" };
+  return { score: votes.filter((x) => x.pass).length / votes.length, votes };
+}
+
+/** Adaptive votes: run the smallest majority first; run the rest only if it did not settle every criterion. */
+export const firstRoundVotes = (planned: number) => Math.floor(planned / 2) + 1;
+
+export function needsMoreVotes(votes: Record<string, Vote[]>, planned: number): boolean {
+  const need = firstRoundVotes(planned);
+  if (need >= planned) return false;
+  return Object.values(votes).some((vs) => vs.length < need || vs.some((v) => v.pass !== vs[0].pass));
+}
+
+// engine:judging:end
+
+const passed = (v: Verdict | undefined) => v?.score != null && v.score > 0.5;
+
+export function scoreTrial(task: Task, verdicts: Record<string, Verdict>, judgeCostUsd = 0): Grades {
+  const errors: string[] = [];
+  const val = (id: string) => {
+    const v = verdicts[id];
+    if (!v || v.score == null) { errors.push(id); return null; }
+    return v.score;
+  };
+  const gatesPassed = task.gates.every((g) => { val(g.id); return passed(verdicts[g.id]); });
+
+  const measures: Record<string, number> = {};
+  let sum = 0, wsum = 0;
+  for (const m of task.measures) {
+    const scores = m.criteria.map((c) => val(c.id)).filter((s): s is number => s != null);
+    if (!scores.length) continue;
+    measures[m.id] = (scores.reduce((a, b) => a + b, 0) / scores.length) * 100;
+    const w = m.weight ?? 1;
+    sum += measures[m.id] * w;
+    wsum += w;
+  }
+  const score = gatesPassed && wsum ? sum / wsum : 0;
+  return { verdicts, gatesPassed, measures, score, errors: [...new Set(errors)], judgeCostUsd };
+}
+
+// ---- statistics ----
+
+const T95: Record<number, number> = { 1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228, 15: 2.131, 20: 2.086, 30: 2.042 };
+const tCrit = (df: number) => {
+  if (T95[df]) return T95[df];
+  const keys = Object.keys(T95).map(Number).filter((k) => k <= df);
+  return df > 30 ? 1.96 : T95[Math.max(...keys)];
+};
+const mean = (xs: number[]) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
+
+/** Half-width of the 95% confidence interval of the mean. */
+export function ci95(xs: number[]): number | null {
+  if (xs.length < 2) return null;
+  const m = mean(xs)!;
+  const sd = Math.sqrt(xs.reduce((a, x) => a + (x - m) ** 2, 0) / (xs.length - 1));
+  return (tCrit(xs.length - 1) * sd) / Math.sqrt(xs.length);
+}
+
+/** Mean of independent means; the half-width combines in quadrature. */
+function combine(stats: Stat[], weights?: number[]): Stat {
+  if (!stats.length || stats.some((s) => s.mean == null)) return { mean: null, ci: null, n: 0 };
+  const w = weights ?? stats.map(() => 1);
+  const W = w.reduce((a, b) => a + b, 0);
+  const m = stats.reduce((a, s, i) => a + s.mean! * w[i], 0) / W;
+  const ci = stats.every((s) => s.ci != null) ? Math.sqrt(stats.reduce((a, s, i) => a + (s.ci! * w[i]) ** 2, 0)) / W : null;
+  return { mean: m, ci, n: Math.min(...stats.map((s) => s.n)) };
+}
+
+// ---- summaries ----
+
+export function familyOf(model: string) {
+  const parts = model.toLowerCase().split(/[-_/:]/).filter((p) => p && !/\d/.test(p) && !EFFORTS.has(p));
+  return parts.slice(0, 2).join("-") || model;
+}
+
+function taskSummary(trials: Trial[]): TaskSummary {
+  const valid = trials.filter((t) => t.status === "done" && t.grades);
+  const scores = valid.map((t) => t.grades!.score);
+  const measures: Record<string, number> = {};
+  const criteria: Record<string, number> = {};
+  for (const t of valid) {
+    for (const [k, s] of Object.entries(t.grades!.measures)) measures[k] = (measures[k] ?? 0) + s / valid.length;
+    for (const [k, v] of Object.entries(t.grades!.verdicts)) if (v.score != null) criteria[k] = (criteria[k] ?? 0) + v.score / valid.length;
+  }
+  const costs = valid.map((t) => t.result?.costUsd).filter((x): x is number => x != null);
+  return {
+    score: { mean: mean(scores), ci: ci95(scores), n: scores.length },
+    measures, criteria,
+    costUsd: mean(costs),
+    costSource: valid.find((t) => t.result)?.result?.costSource ?? "unknown",
+    durationMs: mean(valid.map((t) => t.durationMs).filter((x): x is number => x != null)),
+    valid: valid.length,
+    invalid: trials.filter((t) => t.status === "invalid").length,
+    failed: trials.filter((t) => t.status === "failed").length,
+    total: trials.length,
+  };
+}
+
+export function summarize(exp: Experiment, trials: Trial[], tasks: Task[] = []): ExperimentSummary {
+  const taskIds = Object.keys(exp.tasks);
+  const configs: ConfigSummary[] = exp.configIds.map((configId) => {
+    const c = exp.configs[configId];
+    const per: Record<string, TaskSummary> = {};
+    for (const tid of taskIds) per[tid] = taskSummary(trials.filter((t) => t.configId === configId && t.taskId === tid));
+    const suites: Record<string, Stat> = {};
+    for (const sid of exp.suiteIds) {
+      const s = exp.suites[sid];
+      suites[sid] = combine(s.tasks.map((tid) => per[tid]?.score ?? { mean: null, ci: null, n: 0 }), s.tasks.map((tid) => s.weights?.[tid] ?? 1));
+    }
+    const costs = taskIds.map((t) => per[t].costUsd).filter((x): x is number => x != null);
+    const times = taskIds.filter((t) => per[t].valid > 0).map((t) => per[t].durationMs).filter((x): x is number => x != null);
+    return {
+      configId, label: configLabel(c), harness: c.harness, model: c.model, effort: c.effort, profile: c.profile,
+      family: c.family ?? familyOf(c.model),
+      tasks: per, suites,
+      index: combine(exp.suiteIds.map((s) => suites[s])),
+      costUsd: mean(costs),
+      durationMs: mean(times),
+    };
+  });
+  return { configs, deadCriteria: deadCriteria(tasks, exp.configIds, trials) };
+}
+
+/**
+ * Criteria that carry no signal between these configs: a strong or ceiling criterion every config
+ * passed (pass rate ≥ 0.95), or any criterion every config failed (≤ 0.05). A floor criterion that
+ * every config passes is a guard working as designed, so it is not flagged.
+ */
+export function deadCriteria(tasks: Task[], configIds: string[], trials: Trial[]): Record<string, string[]> {
+  const out: Record<string, string[]> = {};
+  for (const task of tasks) {
+    const crits = task.measures.flatMap((m) => m.criteria);
+    const rates = configIds.map((cid) => taskSummary(trials.filter((t) => t.taskId === task.id && t.configId === cid)))
+      .filter((s) => s.valid > 0).map((s) => s.criteria);
+    if (rates.length < 2) continue;
+    const all = (id: string, ok: (r: number) => boolean) => rates.every((r) => r[id] != null && ok(r[id]));
+    const dead = crits.filter((c) => (c.level !== "floor" && all(c.id, (r) => r >= 0.95)) || all(c.id, (r) => r <= 0.05)).map((c) => c.id);
+    if (dead.length) out[task.id] = dead;
+  }
+  return out;
+}

package/src/core/settings.ts

@@ -0,0 +1,44 @@
+// static-fire.yaml: the user's own settings for trials, and a record of the user-level agent setup.
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { projectRoot } from "./project";
+import type { AgentHarness } from "./types";
+
+export interface Settings {
+  blockedTools: Partial<Record<AgentHarness, string[]>>;      // claude: tool rules; codex: MCP server names
+}
+
+export function loadSettings(root = projectRoot()): Settings {
+  const p = join(root, "static-fire.yaml");
+  const raw = existsSync(p) ? (Bun.YAML.parse(readFileSync(p, "utf8")) as any) : {};
+  return { blockedTools: raw?.trials?.blockedTools ?? {} };
+}
+
+const hash = (p: string) => createHash("sha256").update(readFileSync(p)).digest("hex").slice(0, 12);
+
+/** Path → short content hash for each file (folders are walked), so a result can be traced to the setup it ran with. */
+export function snapshotFiles(paths: string[]): Record<string, string> {
+  const out: Record<string, string> = {};
+  const walk = (p: string) => {
+    if (!existsSync(p)) return;
+    if (statSync(p).isDirectory()) {
+      for (const f of readdirSync(p).sort()) if (f !== ".DS_Store") walk(join(p, f));
+    } else {
+      try { out[p] = hash(p); } catch { /* changed while we read it (a tool updating its files): skip */ }
+    }
+  };
+  for (const p of paths) walk(p);
+  return out;
+}
+
+/** The user-level files each harness reads (instructions, settings, skills). Login secrets are not included. */
+export function userSetupFiles() {
+  const h = homedir();
+  return snapshotFiles([
+    join(h, ".claude", "CLAUDE.md"), join(h, ".claude", "settings.json"), join(h, ".claude", "skills"),
+    join(h, ".codex", "AGENTS.md"), join(h, ".codex", "config.toml"), join(h, ".codex", "skills"),
+    join(h, ".gemini", "GEMINI.md"), join(h, ".gemini", "settings.json"),
+  ]);
+}

package/src/core/task-file.ts

@@ -0,0 +1,95 @@
+// Task file validation. Errors stop a task from running; warnings are the authoring guide's
+// advice (enough criteria, a mix of levels) and are shown wherever the task is listed.
+import { existsSync, statSync } from "node:fs";
+import { isAbsolute, join, normalize } from "node:path";
+import { DERIVED_RUNNERS, derivedExt } from "./derived";
+import type { Criterion } from "./types";
+
+const LEVELS = ["floor", "strong", "ceiling"];
+const GRADERS = ["check", "judge", "human"];
+const EVIDENCE = ["output", "inputs", "references", "project", "sources"];
+export const MIN_CRITERIA = 15;
+export const MIN_PER_MEASURE = 3;
+
+const filled = (v: unknown) => typeof v === "string" && v.trim() !== "";
+
+/** What to fill in for each required top-level part, for the error when it is empty (`static-fire tasks`). */
+const REQUIRED: Record<string, string> = {
+  id: "the task's folder name under evals/",
+  title: "a short name for the task",
+  prompt: "the instructions the agent gets, as the user would give them",
+  inputs: "the folder of files copied into the agent's workspace, for example ./inputs",
+  output: "the path the agent writes, relative to its workspace, for example ./out/answer.md",
+};
+
+function checkCriterion(c: any, where: string, errors: string[], task: any, dir?: string) {
+  const id = filled(c?.id) ? c.id : where;
+  if (c?.id == null || c.id === "") errors.push(`${where}: id is required: a kebab-case name, unique in the task`);
+  else if (typeof c.id !== "string" || !/^[a-z0-9][a-z0-9-]*$/.test(c.id)) errors.push(`${where}: criterion id must be kebab-case`);
+  if (!filled(c?.question)) errors.push(`${id}: question is required: a yes or no question, where yes means pass`);
+  if (!LEVELS.includes(c?.level)) errors.push(`${id}: level must be floor, strong or ceiling`);
+  if (!GRADERS.includes(c?.grader)) errors.push(`${id}: grader must be check, judge or human`);
+  if (c?.grader === "check") {
+    const k = c.check;
+    if (!k) errors.push(`${id}: a check criterion needs \`check\``);
+    else if (k.type === "regex") { try { new RegExp(k.pattern, k.flags); } catch { errors.push(`${id}: invalid regex`); } }
+    else if (k.type === "frontmatter") { if (!k.field) errors.push(`${id}: frontmatter check needs \`field\``); }
+    else if (k.type === "command") { if (!k.run) errors.push(`${id}: command check needs \`run\``); }
+    else if (k.type !== "words") errors.push(`${id}: unknown check type ${JSON.stringify(k.type)}`);
+  }
+  if (c?.evidence != null && (!Array.isArray(c.evidence) || c.evidence.some((e: string) => !EVIDENCE.includes(e))))
+    errors.push(`${id}: evidence must be a list of ${EVIDENCE.join(", ")}`);
+  else for (const k of ["references", "project", "sources"]) if (c?.evidence?.includes(k) && !task[k]) errors.push(`${id}: evidence ${k} needs \`${k}\` on the task`);
+  if (c?.derived != null) checkDerived(c, id, errors, dir);
+  else if (c?.grader === "judge" && Array.isArray(c?.evidence) && !c.evidence.length) errors.push(`${id}: evidence is empty, so the judge would see nothing; name evidence or \`derived\``);
+}
+
+/** `derived`: a judge criterion's script, a path under the task's grader/ folder (README, "Derived evidence"). */
+function checkDerived(c: any, id: string, errors: string[], dir?: string) {
+  const d = c.derived;
+  if (c.grader !== "judge") { errors.push(`${id}: only a judge criterion can have \`derived\` evidence`); return; }
+  if (typeof d !== "string" || !d.trim()) { errors.push(`${id}: derived must be a path under grader/`); return; }
+  if (isAbsolute(d) || normalize(d).split(/[\\/]/)[0] === "..") { errors.push(`${id}: derived ${d} must be a path inside grader/`); return; }
+  if (!derivedExt(d)) { errors.push(`${id}: derived ${d} must end in ${Object.keys(DERIVED_RUNNERS).join(", ")}`); return; }
+  if (dir) {
+    const p = join(dir, "grader", d);
+    if (!existsSync(p) || !statSync(p).isFile()) errors.push(`${id}: derived script grader/${d} does not exist`);
+  }
+}
+
+/** `dir` is the task folder; when given, a `derived` script must exist in its grader/ folder. */
+export function validateTask(raw: any, dir?: string): { errors: string[]; warnings: string[] } {
+  const errors: string[] = [], warnings: string[] = [];
+  for (const [k, what] of Object.entries(REQUIRED)) if (!filled(raw?.[k])) errors.push(`${k} is required: ${what}`);
+  // `version` is a readable label only, in neither hash (identity.ts). A task file can leave it out.
+  if (raw?.version != null && !Number.isInteger(raw.version)) errors.push("version, when given, must be an integer");
+  if (raw?.baseline) errors.push("`baseline` was replaced by human configs: list the file under `outputs` in configs/<you>.yaml");
+  const gates: Criterion[] = raw?.gates ?? [];
+  const measures: any[] = raw?.measures ?? [];
+  if (!Array.isArray(measures) || !measures.length) errors.push("at least one measure is required");
+
+  const seen = new Set<string>();
+  const note = (c: any) => {
+    if (c?.id === "output-exists") errors.push("output-exists is reserved (it is always added as a gate)");
+    else if (filled(c?.id) && seen.has(c.id)) errors.push(`duplicate criterion id ${c.id}`);
+    seen.add(c?.id);
+  };
+  gates.forEach((g, i) => { checkCriterion(g, `gates[${i}]`, errors, raw, dir); note(g); });
+  const mids = new Set<string>();
+  measures.forEach((m, i) => {
+    const mid = filled(m?.id) ? m.id : `measures[${i}]`;
+    if (!filled(m?.id)) errors.push(`measures[${i}]: id is required: a kebab-case name for the measure`);
+    else if (mids.has(m.id)) errors.push(`duplicate measure id ${m.id}`);
+    mids.add(m?.id);
+    if (!filled(m?.label)) errors.push(`${mid}: label is required: the measure's name on the dashboard`);
+    if (m?.weight != null && !(m.weight > 0)) errors.push(`${mid}: weight must be positive`);
+    if (!Array.isArray(m?.criteria) || !m.criteria.length) { errors.push(`${mid}: a measure needs criteria`); return; }
+    m.criteria.forEach((c: any, j: number) => { checkCriterion(c, `${mid}.criteria[${j}]`, errors, raw, dir); note(c); });
+    if (m.criteria.length < MIN_PER_MEASURE) warnings.push(`measure ${mid} has ${m.criteria.length} ${m.criteria.length === 1 ? "criterion" : "criteria"}; the guide asks for at least ${MIN_PER_MEASURE}`);
+  });
+
+  const crits = measures.flatMap((m) => (Array.isArray(m?.criteria) ? m.criteria : []));
+  if (crits.length < MIN_CRITERIA) warnings.push(`${crits.length} criteria in total; the guide asks for at least ${MIN_CRITERIA} so one criterion moves the score by ≤ ${Math.round(100 / MIN_CRITERIA)} points`);
+  for (const lvl of LEVELS) if (crits.length && !crits.some((c: any) => c.level === lvl)) warnings.push(`no ${lvl} criteria; mix floor, strong and ceiling so scores spread out`);
+  return { errors, warnings };
+}