static-fire 0.0.0-stage → 0.44.0

package/src/core/migrate.ts

@@ -0,0 +1,288 @@
+// `static-fire migrate`: a one-off import of an experiment in the 2026-11-06 file format
+// (runs/<experiment>/{experiment.json, trials/<trial>.json, trials/<trial>/…}) into the record store, with its
+// artifacts copied to the per-trial folders (<folder of the database>/trials/<trial-id>/). The old experiment
+// folder is only read; delete it by hand once the dashboard shows the imported data.
+//
+// The import takes today's task files as the task the experiment ran, and today's grading hash with the
+// experiment's judge as the grading its verdicts came from (inferred: the batch request says
+// `gradingInferred: true`). Before writing anything it checks that this is true enough: the task version is
+// unchanged, every trial has ended, and each trial's verdicts are today's criteria and give its score again.
+//
+// Three steps, each one safe to run again, so an interrupted migrate is finished by running it again:
+//   1. copy each trial's workspace and agent log (to a staging folder, then one rename);
+//   2. write every record in one transaction (skipped when the trials are already in the store);
+//   3. copy each grading's judge logs to gradings/<grading-id>/ (staging folder, then one rename).
+import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { OUTPUT_GATE } from "./grade";
+import { canonicalJson, configKey, gradingIdentity, sha256, taskIdentity } from "./identity";
+import { openStore, trialDir, type BatchRecord, type GradingResult, type GradingSetupRecord, type TaskVersionRecord, type TrialRecord, type UserSetupRecord } from "./db";
+import { scoreTrial } from "./score";
+import { loadTask } from "./authored";
+import { projectRoot } from "./project";
+import type { AgentConfig, Experiment, Grades, Task, Trial } from "./types";
+
+/** The experiment the spec imports. Older ones (task versions 3 to 12) are out of scope. */
+export const DEFAULT_EXPERIMENT = "20261106-071029-writing";
+const OLDEST = "2026-11-06";
+/** Config ids renamed since: the human config `kenny` is `configs/user.yaml` today. */
+const CONFIG_RENAMES: Record<string, string> = { kenny: "user" };
+const ENDED = new Set(["done", "failed", "invalid"]);
+const COPY = { recursive: true, preserveTimestamps: true, verbatimSymlinks: true } as const;
+
+export interface MigrateOptions {
+  source: string;          // the old experiment folder, runs/<experiment-id>
+  dbPath: string;          // the store; artifacts go next to it, in trials/<trial-id>/
+  root?: string;           // the project root that holds today's tasks (evals/<id>/task.yaml)
+  log?: (msg: string) => void;
+}
+
+/** What this run added. A run that finds everything in place returns empty counts. */
+export interface MigrateResult { batches: string[]; trials: number; gradings: number; copied: number }
+
+interface PlannedTrial {
+  oldDir: string;
+  record: TrialRecord;
+  grading: { hash: string; votes: number; gradedAt: string; result: GradingResult } | null;
+}
+
+interface Plan {
+  experimentId: string;
+  batches: BatchRecord[];
+  trials: PlannedTrial[];
+  taskVersions: TaskVersionRecord[];
+  gradingSetups: GradingSetupRecord[];
+  userSetup: UserSetupRecord | null;
+}
+
+const refuse = (msg: string): never => { throw new Error(msg); };
+const readJson = <T>(p: string): T => JSON.parse(readFileSync(p, "utf8"));
+/** "2026-11-06T07:10:29.725Z" → "20261106-071029": the UTC stamp batch ids use. */
+const stamp = (iso: string) => iso.replace(/[-:]/g, "").replace(/\..+/, "").replace("T", "-");
+
+export function migrateExperiment(o: MigrateOptions): MigrateResult {
+  const log = o.log ?? (() => {});
+  const plan = planImport(o.source, o.root ?? projectRoot());   // reads only; throws before anything is written
+  const result: MigrateResult = { batches: [], trials: 0, gradings: 0, copied: 0 };
+
+  result.copied += copyTrialFolders(plan, o.dbPath);
+  const store = openStore(o.dbPath);
+  try {
+    writeRecords(store, plan, result);
+    result.copied += copyJudgeLogs(plan, o.dbPath, (id) => store.listGradings(id)[0]?.id ?? null);
+  } finally {
+    store.close();
+  }
+
+  if (!result.trials && !result.copied) log(`nothing to import: the ${plan.trials.length} trials of ${plan.experimentId} are in the store and their folders are copied`);
+  else log(`${plan.experimentId}: imported ${result.trials} trial(s) and ${result.gradings} grading(s)${result.batches.length ? ` in batch(es) ${result.batches.join(", ")}` : ""}; copied ${result.copied} folder(s)`);
+  return result;
+}
+
+// ---- 1. Plan: read the old experiment, check it, build every record -------------------------------
+
+function planImport(source: string, root: string): Plan {
+  const expPath = join(source, "experiment.json");
+  if (!existsSync(expPath)) refuse(`${source} is not an experiment folder: it has no experiment.json`);
+  const exp = readJson<Experiment>(expPath);
+  if (exp.schema !== 2) refuse(`experiment ${exp.id} has schema ${exp.schema}; only the 2026-11-06 format (schema 2) can be imported`);
+  if (exp.createdAt.slice(0, 10) < OLDEST) refuse(`experiment ${exp.id} started on ${exp.createdAt.slice(0, 10)}, older than ${OLDEST}: only experiments from ${OLDEST} on can be imported`);
+  const judge = exp.judge;
+  const now = new Date().toISOString();
+
+  const tasks = new Map<string, Task>();
+  const taskVersions: TaskVersionRecord[] = [], gradingSetups: GradingSetupRecord[] = [];
+  const hashes = new Map<string, { task: string; grading: string }>();
+  for (const [id, ran] of Object.entries(exp.tasks)) {
+    let task: Task;
+    try { task = loadTask(id, root); } catch (e) { return refuse(`task ${id} cannot be loaded today: ${(e as Error).message}`); }
+    if (task.version !== ran) {
+      refuse(`the experiment ran task ${id} version ${ran}, but today's task file ${task.version == null ? "has no version" : `is version ${task.version}`}. The import takes today's task files as the task the experiment ran, so a task that changed since cannot be imported`);
+    }
+    const t = taskIdentity(task), g = gradingIdentity(task, judge);
+    tasks.set(id, task);
+    hashes.set(id, { task: t.hash, grading: g.hash });
+    taskVersions.push({ hash: t.hash, taskId: id, label: task.version ?? null, snapshot: { ...t.snapshot }, firstSeen: now });
+    gradingSetups.push({
+      hash: g.hash, taskId: id, taskHash: t.hash, judgeHarness: judge.harness, judgeModel: judge.model, judgeEffort: judge.effort ?? null,
+      judgeLabel: judge.label, snapshot: { ...g.snapshot }, firstSeen: now,
+    });
+  }
+
+  const trialsDir = join(source, "trials");
+  const old = (existsSync(trialsDir) ? readdirSync(trialsDir) : []).filter((f) => f.endsWith(".json")).sort()
+    .map((f) => readJson<Trial>(join(trialsDir, f)));
+  if (!old.length) refuse(`experiment ${exp.id} has no trials`);
+  for (const t of old) {
+    if (!ENDED.has(t.status)) refuse(`trial ${t.id} has not ended (status ${t.status}): only a finished experiment can be imported`);
+    if (!tasks.has(t.taskId)) refuse(`trial ${t.id} is for task ${t.taskId}, which the experiment does not list`);
+    if (!exp.configs[t.configId]) refuse(`trial ${t.id} is for config ${t.configId}, which the experiment does not record`);
+    if (t.grades) checkGrades(t, t.grades, tasks.get(t.taskId)!);
+  }
+
+  // One batch per UTC start date: the experiment's own run, and trials added to it later.
+  const day = (t: Trial) => (t.startedAt ?? t.endedAt ?? exp.createdAt).slice(0, 10);
+  const groups = new Map<string, Trial[]>();
+  for (const t of old) groups.set(day(t), [...(groups.get(day(t)) ?? []), t]);
+
+  // The user setup was recorded when the experiment started; it is known only for the trials of that day.
+  const userSetup: UserSetupRecord | null = exp.userSetup?.files ? { hash: sha256(canonicalJson(exp.userSetup.files)), files: exp.userSetup.files } : null;
+  const configId = (id: string) => CONFIG_RENAMES[id] ?? id;
+
+  const batches: BatchRecord[] = [], trials: PlannedTrial[] = [];
+  for (const [date, group] of [...groups].sort(([a], [b]) => a.localeCompare(b))) {
+    const starts = group.map((t) => t.startedAt ?? t.endedAt ?? exp.createdAt).sort();
+    const ends = group.map((t) => t.endedAt).filter((e): e is string => !!e).sort();
+    const batchId = stamp(starts[0]!);
+    const inBatch = new Set(group.map((t) => t.configId));
+    batches.push({
+      id: batchId, startedAt: starts[0]!, endedAt: ends.at(-1) ?? null, harnessVersions: {},
+      request: {
+        importedFrom: exp.id, suites: exp.suiteIds, configs: exp.configIds.filter((c) => inBatch.has(c)).map(configId),
+        trials: exp.trialsPerConfig, judge: judge.id, votes: judge.votes, gradingInferred: true,
+      },
+    });
+    for (const t of group) {
+      const config: AgentConfig = { ...exp.configs[t.configId]!, id: configId(t.configId) };
+      const id = `${batchId}--${t.taskId}--${config.id}--${t.index}`;
+      const oldDir = join(trialsDir, t.id);
+      const usage = t.result?.usage;
+      const grades = t.grades;
+      trials.push({
+        oldDir,
+        record: {
+          id, batchId, taskId: t.taskId, taskHash: hashes.get(t.taskId)!.task,
+          configKey: configKey(config), configId: config.id,
+          harness: config.harness, model: config.model, effort: config.effort ?? null, profile: config.profile ?? null,
+          provider: config.provider ?? null, family: config.family ?? null,
+          n: t.index,
+          // An old "invalid" with grades was a grader error: today that is the grading's status, not the trial's.
+          status: t.status === "invalid" && grades ? "done" : t.status as TrialRecord["status"],
+          startedAt: t.startedAt ?? null, endedAt: t.endedAt ?? null, durationMs: t.durationMs ?? null,
+          inputTokens: usage?.inputTokens ?? null, outputTokens: usage?.outputTokens ?? null,
+          cacheReadTokens: usage?.cacheReadTokens ?? null, cacheWriteTokens: usage?.cacheWriteTokens ?? null,
+          costUsd: t.result?.costUsd ?? null, costSource: t.result?.costSource ?? null, turns: t.result?.turns ?? null,
+          setupHash: userSetup && date === exp.createdAt.slice(0, 10) ? userSetup.hash : null,
+          profileHash: null, harnessVersion: null, copiedShare: null,
+          blockedTools: config.harness === "human" || !exp.blockedTools ? null : (exp.blockedTools[config.harness] ?? []),
+          error: t.error ?? null,
+          dir: `trials/${id}`,
+        },
+        grading: grades ? {
+          hash: hashes.get(t.taskId)!.grading, votes: judge.votes, gradedAt: gradedAt(join(oldDir, "judge")) ?? t.endedAt ?? exp.createdAt,
+          result: {
+            status: grades.errors.length ? "invalid" : "done",
+            score: grades.score, gatesPassed: grades.gatesPassed, measures: grades.measures, judgeCostUsd: grades.judgeCostUsd ?? null,
+            error: null,
+            verdicts: Object.entries(grades.verdicts).map(([criterionId, v]) => ({ criterionId, score: v.score ?? null, error: v.error ?? null, votes: v.votes ?? [] })),
+          },
+        } : null,
+      });
+    }
+  }
+  const seen = new Set<string>();
+  for (const t of trials) {
+    if (seen.has(t.record.id)) refuse(`two trials would both be imported as ${t.record.id}`);
+    seen.add(t.record.id);
+  }
+  return { experimentId: exp.id, batches, trials, taskVersions, gradingSetups, userSetup };
+}
+
+/** The verdicts must be today's criteria and give the recorded score again, or today's grading hash would be untrue. */
+function checkGrades(t: Trial, g: Grades, task: Task) {
+  const graded = { ...task, gates: [OUTPUT_GATE, ...task.gates] };
+  const today = new Set([...graded.gates, ...task.measures.flatMap((m) => m.criteria)].map((c) => c.id));
+  const had = new Set(Object.keys(g.verdicts));
+  const onlyOld = [...had].filter((id) => !today.has(id)), onlyNew = [...today].filter((id) => !had.has(id));
+  if (onlyOld.length || onlyNew.length) {
+    refuse(`trial ${t.id}: its criteria differ from today's task ${task.id} (only in the trial: ${onlyOld.join(", ") || "none"}; only in today's task: ${onlyNew.join(", ") || "none"})`);
+  }
+  const again = scoreTrial(graded, g.verdicts, g.judgeCostUsd);
+  const near = (a: number, b: number) => Math.abs(a - b) < 1e-9;
+  const measuresAgree = Object.keys(again.measures).length === Object.keys(g.measures).length
+    && Object.entries(again.measures).every(([id, v]) => g.measures[id] != null && near(v, g.measures[id]!));
+  if (!near(again.score, g.score) || again.gatesPassed !== g.gatesPassed || !measuresAgree) {
+    refuse(`trial ${t.id}: today's task ${task.id} gives its verdicts the score ${again.score}, but the trial recorded ${g.score}`);
+  }
+}
+
+/** When the judge logs were last written: the evidence for when the trial was graded. */
+function gradedAt(judgeDir: string): string | null {
+  let latest = -1;
+  const walk = (p: string) => {
+    const st = statSync(p);
+    if (st.isDirectory()) for (const f of readdirSync(p)) walk(join(p, f));
+    else latest = Math.max(latest, st.mtimeMs);
+  };
+  if (existsSync(judgeDir)) walk(judgeDir);
+  return latest < 0 ? null : new Date(latest).toISOString();
+}
+
+// ---- 2. Copy, write, copy: each step skips what is already done --------------------------------------
+
+/**
+ * Copies `fill` into a staging folder next to `dest`, then renames it to `dest`: a folder at `dest` is always
+ * complete. A staging folder left by an interrupted run is removed first. Returns false when `dest` exists.
+ */
+function copyOnce(dest: string, fill: (staging: string) => void): boolean {
+  const parent = dirname(dest), name = dest.slice(parent.length + 1);
+  if (existsSync(parent)) for (const f of readdirSync(parent)) {
+    if (f.startsWith(`.${name}.`) && f.endsWith(".partial")) rmSync(join(parent, f), { recursive: true, force: true });
+  }
+  if (existsSync(dest)) return false;
+  const staging = join(parent, `.${name}.${process.pid}.partial`);
+  mkdirSync(staging, { recursive: true });
+  fill(staging);
+  renameSync(staging, dest);
+  return true;
+}
+
+function copyTrialFolders(plan: Plan, dbPath: string): number {
+  let copied = 0;
+  for (const t of plan.trials) {
+    const done = copyOnce(trialDir(dbPath, t.record.id), (staging) => {
+      if (existsSync(join(t.oldDir, "workspace"))) cpSync(join(t.oldDir, "workspace"), join(staging, "workspace"), COPY);
+      if (existsSync(join(t.oldDir, "agent.log"))) cpSync(join(t.oldDir, "agent.log"), join(staging, "agent.log"), COPY);
+    });
+    if (done) copied++;
+  }
+  return copied;
+}
+
+function writeRecords(store: ReturnType<typeof openStore>, plan: Plan, result: MigrateResult) {
+  store.transaction(() => {
+    const present = plan.trials.filter((t) => store.getTrial(t.record.id));
+    if (present.length === plan.trials.length) return;
+    if (present.length) {
+      throw new Error(`the store already has ${present.length} of the ${plan.trials.length} trials of ${plan.experimentId} (for example ${present[0]!.record.id}), but not all: it was not left so by migrate; check it before importing`);
+    }
+    if (plan.userSetup) store.putUserSetup(plan.userSetup);
+    for (const r of plan.taskVersions) store.putTaskVersion(r);
+    for (const r of plan.gradingSetups) store.putGradingSetup(r);
+    for (const b of plan.batches) { store.createBatch(b); result.batches.push(b.id); }
+    for (const t of plan.trials) {
+      store.insertTrial(t.record);
+      result.trials++;
+      if (!t.grading) continue;
+      const id = store.startGrading({ trialId: t.record.id, gradingHash: t.grading.hash, votes: t.grading.votes, gradedAt: t.grading.gradedAt });
+      store.finishGrading(id, t.grading.result);
+      result.gradings++;
+    }
+  });
+}
+
+/** `gradingId` gives the imported grading of a trial: its first, since later regrades add rows after it. */
+function copyJudgeLogs(plan: Plan, dbPath: string, gradingId: (trialId: string) => number | null): number {
+  let copied = 0;
+  for (const t of plan.trials) {
+    const judge = join(t.oldDir, "judge");
+    if (!t.grading || !existsSync(judge)) continue;
+    const id = gradingId(t.record.id);
+    if (id == null) throw new Error(`trial ${t.record.id} has no grading in the store to put its judge logs under`);
+    const done = copyOnce(join(trialDir(dbPath, t.record.id), "gradings", String(id)), (staging) => {
+      for (const f of readdirSync(judge)) cpSync(join(judge, f), join(staging, f), COPY);
+    });
+    if (done) copied++;
+  }
+  return copied;
+}

package/src/core/new-task.ts

@@ -0,0 +1,85 @@
+// `static-fire new task <id>`: a new task folder, evals/<id>/, with a commented task.yaml and an empty inputs/
+// folder. The file has every part a task needs, empty, so `static-fire tasks` lists what is left to fill in.
+// It has no `version`: a task file can leave it out (task-file.ts).
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+export const TASK_ID = /^[a-z0-9][a-z0-9-]*$/;
+
+/** `new task` was given an id that is not kebab-case. Nothing was written. */
+export class BadTaskId extends Error {
+  constructor(id: string) {
+    super(`task id ${JSON.stringify(id)} must be kebab-case: lowercase letters, digits and hyphens, for example thumbnail-ab`);
+    this.name = "BadTaskId";
+  }
+}
+
+/** `new task` was given an id whose folder evals/<id> exists. Nothing was changed. */
+export class TaskExists extends Error {
+  constructor(id: string) {
+    super(`task ${id} exists: evals/${id} is already there. Nothing was changed. Pick another id, or edit evals/${id}/task.yaml`);
+    this.name = "TaskExists";
+  }
+}
+
+/** The commented task.yaml of a new task. Comments say what each part is for, not how to write it. */
+export function taskTemplate(id: string): string {
+  return `# A task: one job for an agent, and how its output is graded.
+# Fill in each empty part, then run \`static-fire tasks\` to check this file.
+# A line that starts with # is optional: remove the # to use it.
+
+id: ${id}
+title:                      # a short name for the task
+# description:              # one line on what the task tests
+
+# What the agent gets
+prompt:                     # the instructions the agent gets, as the user would give them
+inputs: ./inputs            # the files the agent works from; this folder is copied into its workspace
+output:                     # the path the agent writes, relative to its workspace, for example ./out/answer.md
+# project: ./project        # files the agent would find in the real project, copied into its workspace
+
+# What only the graders get
+# context:                  # one paragraph a judge reads before the criteria
+# references: ./references  # past work a judge can compare the output with
+# sources: ./sources        # saved copies of the pages the inputs link to, for fact checks
+# heldOut: ./held-out.md    # the real original; an output that copies it is invalid
+
+# How the output is graded: criteria are yes or no questions about the output, and yes means pass
+# gates:                    # criteria outside the measures; if one fails, the task score is 0
+measures:                   # groups of criteria, each scored 0 to 100
+  - id:                     # a kebab-case name, for example clarity
+    label:                  # the measure's name on the dashboard
+    # weight: 1             # its share of the task score
+    criteria:
+      - id:                 # a kebab-case name, unique in the task
+        question:           # what to look at, and where the line between pass and fail is
+        level:              # floor (most configs pass), strong (good configs pass) or ceiling (only the best pass)
+        grader:             # check (a rule run on the output), judge (a model reads it) or human
+        # check: { type: words, min: 100 }   # for grader check: regex, words, frontmatter or command
+        # evidence: [output]                 # for grader judge: what it reads: output, inputs, project, references, sources
+        # derived: thumbnail.py              # for grader judge: a script in grader/ that makes an image of the output
+`;
+}
+
+export interface NewTask { dir: string; file: string; inputs: string }
+
+/**
+ * Makes evals/<id>/task.yaml and the empty evals/<id>/inputs/ under the project `root`. Throws BadTaskId or
+ * TaskExists before it writes anything; the task folder is made with one call that fails when it exists.
+ */
+export function newTask(root: string, id: string): NewTask {
+  if (!TASK_ID.test(id)) throw new BadTaskId(id);
+  const evals = join(root, "evals"), dir = join(evals, id);
+  if (existsSync(dir)) throw new TaskExists(id);
+  mkdirSync(evals, { recursive: true });
+  try {
+    mkdirSync(dir);
+  } catch (e) {
+    if ((e as { code?: string }).code === "EEXIST") throw new TaskExists(id);
+    throw e;
+  }
+  const file = join(dir, "task.yaml"), inputs = join(dir, "inputs");
+  mkdirSync(inputs);
+  writeFileSync(file, taskTemplate(id), { flag: "wx" });
+  return { dir, file, inputs };
+}

package/src/core/outside-tools.ts

@@ -0,0 +1,129 @@
+// The outside tools in each harness's user setup: the MCP servers, plugins and claude.ai connectors a trial loads,
+// each with the exact entry that blocks it under `trials.blockedTools.<harness>` in static-fire.yaml (ADR-0009).
+// Static Fire lists them and does not choose: the user's agent proposes which ones act on outside services, and
+// the user approves the block list.
+//
+// Read only, and names only. Setup files can hold secrets (API keys in `env`, tokens in URLs, headers or command
+// arguments), so this code takes the key names of each object and never reads a value into the report. A file
+// that does not parse gets a note with its path only: a parse error message can quote the file's content.
+//
+// Which files, and what is left out on purpose:
+// - Claude Code (`$CLAUDE_CONFIG_DIR`, else `~/.claude`): user MCP servers in `.claude.json` (in `$CLAUDE_CONFIG_DIR`
+//   when set, else in the home folder; that move is assumed, not confirmed), plugins turned on in `settings.json`
+//   with the MCP servers in their install folder, and the claude.ai connectors Claude Code last saw
+//   (`claudeAiMcpEverConnected`, a cache: it can hold a removed connector or miss a new one; the live list needs a
+//   network call). Managed (admin) MCP settings are not read.
+// - Codex (`$CODEX_HOME`, else `~/.codex`): `[mcp_servers.*]` and `[plugins.*]` in `config.toml`, without the ones
+//   turned off (`enabled = false`), which a trial does not load.
+// - Project-level setup (`.mcp.json`, Claude Code local scope, Codex project config) is not read: a trial runs in a
+//   fresh workspace outside any repo, so none of it loads.
+// - Antigravity and Pi trials have no block list yet, so their setup is not read.
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join, resolve } from "node:path";
+import type { AgentHarness } from "./types";
+
+export type OutsideToolKind = "mcp-server" | "plugin" | "connector";
+
+export interface OutsideTool {
+  /** The server, plugin ("name@marketplace") or connector name as the setup file has it. */
+  name: string;
+  kind: OutsideToolKind;
+  /** The file it was found in. */
+  foundIn: string;
+  /** The entries that block it, to paste under `trials.blockedTools.<harness>`. Empty: nothing to block. */
+  block: string[];
+}
+
+export interface HarnessTools {
+  /** false: this harness's tools cannot be blocked in trials yet. */
+  blockable: boolean;
+  tools: OutsideTool[];
+  notes: string[];
+}
+
+const BLOCKABLE = new Set<AgentHarness>(["claude", "codex"]);
+const NAMES: Partial<Record<AgentHarness, string>> = { agy: "Antigravity", pi: "Pi" };
+
+type Obj = Record<string, unknown>;
+const isObj = (v: unknown): v is Obj => typeof v === "object" && v !== null && !Array.isArray(v);
+const keys = (v: unknown) => (isObj(v) ? Object.keys(v) : []);
+
+/** A JSON or TOML file as an object, or null when it is missing. A file that does not parse adds a note with its path only. */
+function readFile(path: string, notes: string[], parse: (text: string) => unknown = JSON.parse): Obj | null {
+  if (!existsSync(path)) return null;
+  try {
+    const v = parse(readFileSync(path, "utf8"));
+    if (isObj(v)) return v;
+  } catch { /* the message can quote the file: say only which file */ }
+  notes.push(`${path} could not be read, so the tools in it are not listed.`);
+  return null;
+}
+
+/** Claude Code's form of a server name in its tool names. */
+const toolName = (server: string) => `mcp__${server.replace(/[^A-Za-z0-9_-]/g, "_")}`;
+
+/** The MCP server names of a Claude Code plugin: its `.mcp.json` (with or without an `mcpServers` key) and `mcpServers` in its plugin.json. */
+function pluginServers(installPath: string, notes: string[]): string[] {
+  const fromFile = (p: string) => {
+    const f = readFile(p, notes);
+    return f ? keys(isObj(f.mcpServers) ? f.mcpServers : f) : [];
+  };
+  const servers = fromFile(join(installPath, ".mcp.json"));
+  const manifest = readFile(join(installPath, ".claude-plugin", "plugin.json"), notes);
+  const field = manifest?.mcpServers;
+  if (isObj(field)) servers.push(...keys(field));
+  for (const p of [field].flat()) if (typeof p === "string") servers.push(...fromFile(resolve(installPath, p)));
+  return [...new Set(servers)];
+}
+
+function claudeTools(home: string, env: Record<string, string | undefined>, notes: string[]): OutsideTool[] {
+  const dir = env.CLAUDE_CONFIG_DIR || join(home, ".claude");
+  const globalFile = env.CLAUDE_CONFIG_DIR ? join(dir, ".claude.json") : join(home, ".claude.json");
+  const global = readFile(globalFile, notes);
+  const tools: OutsideTool[] = keys(global?.mcpServers).map((name) => ({ name, kind: "mcp-server", foundIn: globalFile, block: [toolName(name)] }));
+
+  const settingsFile = join(dir, "settings.json");
+  const enabled = readFile(settingsFile, notes)?.enabledPlugins;
+  const on = keys(enabled).filter((k) => (enabled as Obj)[k] === true);
+  if (on.length) {
+    const installed = readFile(join(dir, "plugins", "installed_plugins.json"), notes)?.plugins;
+    for (const name of on) {
+      const entries = isObj(installed) && Array.isArray(installed[name]) ? (installed[name] as unknown[]).filter(isObj) : [];
+      const path = (entries.find((e) => e.scope === "user") ?? entries[0])?.installPath;
+      const plugin = name.split("@")[0]!;
+      const servers = typeof path === "string" ? pluginServers(path, notes) : [];
+      tools.push({ name, kind: "plugin", foundIn: settingsFile, block: servers.map((s) => toolName(`plugin_${plugin}_${s}`)) });
+    }
+  }
+
+  const seen = global?.claudeAiMcpEverConnected;
+  const connectors = (Array.isArray(seen) ? seen : keys(seen)).filter((n): n is string => typeof n === "string");
+  for (const name of connectors) tools.push({ name, kind: "connector", foundIn: globalFile, block: [toolName(name)] });
+  if (connectors.length) {
+    notes.push(`the claude.ai connectors are the ones Claude Code last saw (${globalFile}), so the list can hold a connector that was removed or miss a new one.`);
+  }
+  return tools;
+}
+
+function codexTools(home: string, env: Record<string, string | undefined>, notes: string[]): OutsideTool[] {
+  const file = join(env.CODEX_HOME || join(home, ".codex"), "config.toml");
+  const config = readFile(file, notes, (t) => Bun.TOML.parse(t));
+  const on = (table: unknown) => keys(table).filter((k) => !(isObj((table as Obj)[k]) && ((table as Obj)[k] as Obj).enabled === false));
+  return [
+    ...on(config?.mcp_servers).map((name): OutsideTool => ({ name, kind: "mcp-server", foundIn: file, block: [name] })),
+    ...on(config?.plugins).map((name): OutsideTool => ({ name, kind: "plugin", foundIn: file, block: [name] })),
+  ];
+}
+
+/** The outside tools of `harness`'s user setup, with their block entries. A harness that is not installed has none. */
+export function findOutsideTools(harness: AgentHarness, installed: boolean, home = homedir(), env: Record<string, string | undefined> = process.env): HarnessTools {
+  const notes: string[] = [];
+  if (!BLOCKABLE.has(harness)) {
+    notes.push(`${NAMES[harness] ?? harness} tools cannot be blocked in trials yet, so they are not listed.`);
+    return { blockable: false, tools: [], notes };
+  }
+  if (!installed) return { blockable: true, tools: [], notes };
+  const tools = harness === "claude" ? claudeTools(home, env, notes) : codexTools(home, env, notes);
+  return { blockable: true, tools, notes };
+}

package/src/core/profiles.ts

@@ -0,0 +1,42 @@
+// Materialize an apkit Profile once per batch and harness: install it into a scratch folder outside any
+// Git repo, snapshot the files, then remove it so the user's apkit records stay clean. Each trial
+// workspace gets a copy of the snapshot, so every config sees identical Profile material.
+// apkit keeps one record of installs, so installs never overlap: within a process they wait in line, and
+// across processes (two `run`s at once) they hold the lock file <cache>/apkit.lock. The install, copy and
+// remove run in a helper process (apkit-install.ts) that holds the lock itself, so a `run` killed mid-install
+// leaves the helper holding the lock until the sequence ends.
+// Current apkit (0.239+): context goes in AGENTS.md, skills in each agent's own skills folder.
+import { join } from "node:path";
+import type { AgentHarness } from "./types";
+
+const APKIT_AGENT: Record<AgentHarness, string> = { claude: "claude", codex: "codex", pi: "pi", agy: "antigravity" };
+const HELPER = join(import.meta.dir, "apkit-install.ts");
+const inflight = new Map<string, Promise<string>>();
+let apkitLine: Promise<unknown> = Promise.resolve();  // this process's installs, one at a time
+
+/** The lock file every process takes before it runs apkit. */
+export const apkitLockPath = (cacheDir: string) => join(cacheDir, "apkit.lock");
+
+/**
+ * The snapshot folder of `profile` for `harness` in batch `batchId`, built on first use and reused by every
+ * trial of the batch. `cacheDir` is the project's Profile cache (`.cache/`), or a temp folder in tests.
+ */
+export function profileSnapshot(profile: string, harness: AgentHarness, batchId: string, cacheDir: string): Promise<string> {
+  const key = `${batchId}/${profile}-${harness}`;
+  const id = `${cacheDir}\0${key}`;
+  if (!inflight.has(id)) {
+    const p = apkitLine.then(() => build(profile, harness, key, cacheDir));
+    apkitLine = p.catch(() => {});
+    inflight.set(id, p);
+  }
+  return inflight.get(id)!;
+}
+
+/** Runs the helper (with this process's Bun) and waits for it; it takes the lock and builds the snapshot. */
+async function build(profile: string, harness: AgentHarness, key: string, cacheDir: string) {
+  const snap = join(cacheDir, "profiles", key);
+  const p = Bun.spawn([process.execPath, HELPER, apkitLockPath(cacheDir), profile, APKIT_AGENT[harness], snap], { stdout: "pipe", stderr: "pipe", stdin: "ignore" });
+  const [err, code] = await Promise.all([new Response(p.stderr).text(), p.exited, new Response(p.stdout).text()]);
+  if (code !== 0) throw new Error(err.trim() || `the Profile install helper exited ${code}`);
+  return snap;
+}

package/src/core/project.ts

@@ -0,0 +1,54 @@
+// The two roots. The engine root holds the engine's own files: its source, the dashboard and pricing.yaml. It
+// comes from this file's location, never from the current folder, so it holds when the engine runs from
+// node_modules. The project root is the user's folder: static-fire.yaml, configs/, judges.yaml, suites/,
+// evals/, the record store and trial artifacts in runs/, and the Profile cache in .cache/. A command finds it
+// by walking up from the current folder to static-fire.yaml. Nothing of a project is written to the engine root.
+import { existsSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+
+export const ENGINE_ROOT = resolve(import.meta.dir, "../..");
+export const PROJECT_FILE = "static-fire.yaml";
+
+/** A command that needs a project was run where no static-fire.yaml is in the folder or any folder above it. */
+export class NoProjectError extends Error {
+  constructor(from: string) {
+    super(`no Static Fire project here: there is no ${PROJECT_FILE} in ${from} or in a folder above it. Run \`static-fire init\` to make a project.`);
+    this.name = "NoProjectError";
+  }
+}
+
+/** The first folder from `from` upward that holds static-fire.yaml, or null. */
+export function findProject(from = process.cwd()): string | null {
+  for (let dir = resolve(from); ; dir = dirname(dir)) {
+    if (existsSync(join(dir, PROJECT_FILE))) return dir;
+    if (dirname(dir) === dir) return null;
+  }
+}
+
+/** The project root found from `from`. Throws NoProjectError when there is none. */
+export function projectRoot(from = process.cwd()): string {
+  const root = findProject(from);
+  if (!root) throw new NoProjectError(resolve(from));
+  return root;
+}
+
+export interface ProjectPaths {
+  root: string;
+  /** The record store's default folder and the trial artifacts' folder (runs/trials/<trial-id>). */
+  runs: string;
+  /** The Profile cache and its lock file. */
+  cache: string;
+  /** The record store: STATIC_FIRE_DB when set, else runs/static-fire.db. Trial artifacts go next to it. */
+  dbPath: string;
+}
+
+export function projectPaths(root: string): ProjectPaths {
+  const runs = join(root, "runs");
+  return { root, runs, cache: join(root, ".cache"), dbPath: process.env.STATIC_FIRE_DB || join(runs, "static-fire.db") };
+}
+
+/** The pricing file: the project's own pricing.yaml when it has one, else the engine's. */
+export function pricingPath(project: string | null): string {
+  const own = project ? join(project, "pricing.yaml") : null;
+  return own && existsSync(own) ? own : join(ENGINE_ROOT, "pricing.yaml");
+}