static-fire 0.0.0-stage → 0.44.0

package/src/core/checks.ts

@@ -0,0 +1,63 @@
+// Deterministic checks. Each returns a Verdict with one vote (pass/fail) or an error.
+import type { Check, Verdict } from "./types";
+
+export interface CheckContext { output: string; outputPath: string; workspace: string; taskDir: string }
+
+export function splitFrontmatter(text: string): { fields: Record<string, unknown>; body: string } {
+  const m = text.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+  if (!m) return { fields: {}, body: text };
+  let fields: Record<string, unknown> = {};
+  try { fields = (Bun.YAML.parse(m[1]) as Record<string, unknown>) ?? {}; } catch { /* treat as none */ }
+  return { fields, body: text.slice(m[0].length).replace(/^\s+/, "") };
+}
+
+const one = (pass: boolean, reason: string): Verdict => ({ score: pass ? 1 : 0, votes: [{ pass, reason }] });
+const err = (error: string): Verdict => ({ score: null, votes: [], error });
+
+const within = (n: number, min?: number, max?: number) => (min == null || n >= min) && (max == null || n <= max);
+const bounds = (min?: number, max?: number) => [min != null && `≥ ${min}`, max != null && `≤ ${max}`].filter(Boolean).join(" and ");
+
+export async function runCheck(check: Check, ctx: CheckContext): Promise<Verdict> {
+  try {
+    switch (check.type) {
+      // engine:check.regex:begin
+      case "regex": {
+        const flags = check.flags?.includes("g") ? check.flags : `${check.flags ?? ""}g`;
+        const n = [...ctx.output.matchAll(new RegExp(check.pattern, flags))].length;
+        const min = check.min ?? (check.max == null ? 1 : undefined);
+        return one(within(n, min, check.max), `${n} match${n === 1 ? "" : "es"} of /${check.pattern}/ (needs ${bounds(min, check.max)})`);
+      }
+      // engine:check.regex:end
+      // engine:check.words:begin
+      case "words": {
+        const n = splitFrontmatter(ctx.output).body.split(/\s+/).filter((w) => /[\p{L}\p{N}]/u.test(w)).length;
+        return one(within(n, check.min, check.max), `${n} words (needs ${bounds(check.min, check.max)})`);
+      }
+      // engine:check.words:end
+      // engine:check.frontmatter:begin
+      case "frontmatter": {
+        const v = splitFrontmatter(ctx.output).fields[check.field];
+        if (typeof v !== "string" || !v.trim()) return one(false, `frontmatter field "${check.field}" is missing`);
+        return one(within(v.length, check.minChars, check.maxChars), `"${check.field}" is ${v.length} characters (needs ${bounds(check.minChars, check.maxChars)})`);
+      }
+      // engine:check.frontmatter:end
+      // engine:check.command:begin
+      case "command": {
+        const proc = Bun.spawn(["sh", "-c", check.run], {
+          cwd: ctx.workspace, stdin: "ignore", stdout: "pipe", stderr: "pipe",
+          env: { ...process.env, OUTPUT: ctx.outputPath, WORKSPACE: ctx.workspace, TASK_DIR: ctx.taskDir },
+        });
+        let timedOut = false;
+        const timer = setTimeout(() => { timedOut = true; proc.kill(); }, (check.timeoutSec ?? 60) * 1000);
+        const [out, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
+        clearTimeout(timer);
+        if (timedOut) return err(`command timed out after ${check.timeoutSec ?? 60}s`);
+        const said = (out.trim() || stderr.trim()).split("\n").slice(-1)[0]?.slice(0, 300);
+        return one(code === 0, said || `exit ${code}`);
+      }
+      // engine:check.command:end
+    }
+  } catch (e) {
+    return err(e instanceof Error ? e.message : String(e));
+  }
+}

package/src/core/commands.ts

@@ -0,0 +1,42 @@
+// The commands of `static-fire` and the flags each one takes: the one list that the CLI (src/cli.ts) parses with,
+// rejects unknown flags by and prints as its usage. The guide for agents (src/guide.md) is checked against it
+// (tests/guide.test.ts), so the guide cannot name a command or a flag that the CLI does not have.
+
+/** Every flag, as `parseArgs` takes it. A default here is only used when the command takes the flag. */
+export const OPTIONS = {
+  suites: { type: "string" }, tasks: { type: "string" }, grading: { type: "string" }, since: { type: "string" }, until: { type: "string" },
+  configs: { type: "string" }, trials: { type: "string", default: "3" },
+  judge: { type: "string" }, votes: { type: "string" }, "only-invalid": { type: "boolean" },
+  concurrency: { type: "string", default: "4" }, port: { type: "string", default: "4477" }, json: { type: "boolean" },
+  from: { type: "string" }, batch: { type: "string" },
+} as const;
+
+export type Flag = keyof typeof OPTIONS;
+
+/** Each command: the flags its code reads, and its arguments for the usage text. */
+export const COMMANDS: Record<string, { flags: readonly Flag[]; args?: string }> = {
+  init: { flags: ["json"] },
+  guide: { flags: [] },
+  serve: { flags: ["port"] },
+  configs: { flags: [] },
+  judges: { flags: [] },
+  suites: { flags: [] },
+  tasks: { flags: [] },
+  new: { flags: [], args: "task <id>" },
+  run: { flags: ["suites", "tasks", "configs", "trials", "judge", "votes", "concurrency"] },
+  regrade: { flags: ["tasks", "configs", "batch", "judge", "votes", "only-invalid", "concurrency"] },
+  rerun: { flags: ["concurrency"], args: "<batch>" },
+  list: { flags: [] },
+  show: { flags: ["suites", "tasks", "judge", "grading", "since", "until", "json"] },
+  export: { flags: [], args: "[path]" },
+  migrate: { flags: ["from"] },
+};
+
+/** The flags given on the command line that `command` does not take, as written (`--json`). */
+export function unknownFlags(command: string, given: string[]): string[] {
+  const takes = new Set<string>(COMMANDS[command]?.flags ?? []);
+  return [...new Set(given)].filter((f) => !takes.has(f)).map((f) => `--${f}`);
+}
+
+/** The usage line the CLI prints for no command or an unknown one. */
+export const USAGE = `usage: static-fire <${Object.keys(COMMANDS).join("|")}>  (static-fire --version for the version)`;

package/src/core/compare.ts

@@ -0,0 +1,203 @@
+// Compare gradings: two grading setups of one task, over the trials that have a valid grading under both.
+// GET /api/compare and the dashboard's comparison card read it through compareGradings().
+//
+// Each side is the view (buildView) with that grading setup picked, so the scope, config facets, date range and
+// the human rule are the view's own. Only the trials valid on both sides are kept, and each side is summarised
+// over exactly those trials by the view's summary, so score.ts computes every number (DEC-2). Gradings are never
+// averaged together: each side reads only its own grading hash.
+import type { GradingSetupRecord, Store, TrialRecord } from "./db";
+import { gradingDiff, versionNames } from "./grading-versions";
+import type { GradingDiff } from "./grading-versions";
+import type { GradingContent } from "./identity";
+import type { Harness, TaskSnapshot } from "./types";
+import { buildView, projectCatalog, summarizeTrials } from "./view";
+import type { Catalog, View, ViewFilters, ViewTrial } from "./view";
+
+/** One of the two grading versions compared. */
+export interface CompareSide {
+  /** Full grading hash. */
+  hash: string;
+  /** Its name, from what changed since the version before it for the same task and judge ("Original grading" for the first). */
+  name: string;
+  /** What changed since that version, in short words; empty for the original grading. */
+  nameParts: string[];
+  /** The task hash it grades. */
+  taskHash: string;
+  /** The task file's `version` stored with that task hash (a label only), or null when the task has none. */
+  label: number | null;
+  judgeLabel: string;
+  /** When the setup was first recorded (ISO time). */
+  firstSeen: string;
+  /** True when it is today's grading setup of the task for the judge in view. */
+  current: boolean;
+  /** The task's criteria as this setup graded with them. */
+  def: TaskSnapshot;
+}
+
+/** One config's task score under each setup, over the compared trials. */
+export interface CompareConfig {
+  configKey: string;
+  label: string;
+  harness: Harness;
+  family: string;
+  profile?: string;
+  /** Compared trials of this config. */
+  n: number;
+  /** Task score (0–100) under setup a and setup b, and b − a. */
+  a: number | null;
+  b: number | null;
+  change: number | null;
+}
+
+/** One criterion's pass rate (0–1) for one config under each setup. Null when that setup's task has no such criterion. */
+export interface CompareCriterion {
+  criterionId: string;
+  configKey: string;
+  a: number | null;
+  b: number | null;
+  /**
+   * True when the criterion itself differs between the two versions: it was added, removed or changed (`diff.criteria`),
+   * or it is a command check whose `run` names a grader file that differs (`diff.files.grader`). A script the named
+   * file loads in its turn is not traced.
+   */
+  changed: boolean;
+  /** True when its pass rate differs between the versions (or it has one under one version only). */
+  moved: boolean;
+}
+
+export interface GradingComparison {
+  taskId: string;
+  a: CompareSide;
+  b: CompareSide;
+  /** Everything that differs from version a to version b, from their stored snapshots. */
+  diff: GradingDiff;
+  /** True when both setups grade the same task hash. Setups of two task versions share no trials. */
+  sameTask: boolean;
+  /** Trials with a valid grading under both setups: the only ones compared. */
+  trials: number;
+  /** Trials with a valid grading under only setup a, or only setup b. They are not counted. */
+  onlyA: number;
+  onlyB: number;
+  /** Configs with a compared trial, best under setup b first. */
+  configs: CompareConfig[];
+  /** Every criterion of either setup's task, for each config in `configs`. */
+  criteria: CompareCriterion[];
+  /** What the page's Reading line states (US-007, DEC-005). Null when no trial is compared, as with two task versions. */
+  reading: CompareReading | null;
+}
+
+/**
+ * The counts behind the Reading line. They count criteria and gates, not criterion-and-config rows: a criterion
+ * moved when its pass rate moved for at least one config.
+ */
+export interface CompareReading {
+  /** Changed criteria that moved. */
+  changedMoved: number;
+  /** Changed criteria that did not move. */
+  changedSteady: number;
+  /** Unchanged criteria that moved. */
+  unchangedMoved: number;
+  /**
+   * True when the difference is the judge varying between runs, not an effect of the change: no changed criterion
+   * moved, at least one unchanged criterion moved, and nothing outside criteria and gates differs (`diff.criteriaOnly`).
+   */
+  judgeVariation: boolean;
+}
+
+const valid = (t: ViewTrial) => t.status === "done" && !!t.grades;
+
+/**
+ * Compares grading setups `a` and `b` (full hashes or unique prefixes) of one task in the scope of `filters`.
+ * The view's own `grading` filter is replaced by each setup in turn. Throws on an unknown or ambiguous hash, the
+ * same setup twice, setups of two tasks, or a task outside the scope.
+ */
+export function compareGradings(store: Store, filters: ViewFilters, a: string, b: string, catalog: Catalog = projectCatalog()): GradingComparison {
+  const sa = resolve(store, a, "a"), sb = resolve(store, b, "b");
+  if (sa.hash === sb.hash) throw new Error(`a and b are the same grading version (${sa.hash}); pick two different ones`);
+  if (sa.taskId !== sb.taskId) throw new Error(`grading versions ${sa.hash.slice(0, 8)} and ${sb.hash.slice(0, 8)} are of two tasks (${sa.taskId}, ${sb.taskId}); compare two grading versions of one task`);
+  const taskId = sa.taskId;
+  // Each setup brings its own judge (the view refuses another judge with a picked setup).
+  const { judge: _judge, ...rest } = filters;
+  const va = buildView(store, { ...rest, grading: sa.hash }, catalog);   // throws when the task is not in scope
+  const vb = buildView(store, { ...rest, grading: sb.hash }, catalog);
+  const ofTask = (v: View) => new Map(v.trials.filter((t) => t.taskId === taskId && valid(t)).map((t) => [t.id, t]));
+  const ta = ofTask(va), tb = ofTask(vb);
+  const both = [...ta.keys()].filter((id) => tb.has(id));
+
+  const records = new Map<string, TrialRecord>(both.map((id) => [id, store.getTrial(id)!]));
+  const side = (v: View, trials: Map<string, ViewTrial>) =>
+    summarizeTrials(both.map((id) => trials.get(id)!), records, { [taskId]: v.tasks[taskId]! }, []).configs;
+  const ca = side(va, ta), cb = side(vb, tb);
+
+  const scoreOf = (list: typeof ca, key: string) => list.find((c) => c.configKey === key)?.tasks[taskId]?.score.mean ?? null;
+  const configs: CompareConfig[] = cb.map((c) => {
+    const x = scoreOf(ca, c.configKey), y = c.tasks[taskId]?.score.mean ?? null;
+    return {
+      configKey: c.configKey, label: c.label, harness: c.harness, family: c.family, ...(c.profile ? { profile: c.profile } : {}),
+      n: c.pooled[taskId]?.n ?? 0, a: x, b: y, change: x == null || y == null ? null : y - x,
+    };
+  }).sort((p, q) => (q.b ?? -1) - (p.b ?? -1) || (p.configKey < q.configKey ? -1 : 1));
+
+  const ids = (d: TaskSnapshot) => [...d.gates, ...d.measures.flatMap((m) => m.criteria)].map((c) => c.id);
+  const da = va.tasks[taskId]!.def, db = vb.tasks[taskId]!.def;
+  const inA = new Set(ids(da)), inB = new Set(ids(db));
+  const rate = (list: typeof ca, key: string, id: string, has: Set<string>) =>
+    has.has(id) ? list.find((c) => c.configKey === key)?.tasks[taskId]?.criteria[id] ?? null : null;
+  const diff = gradingDiff({ setup: sa, task: store.getTaskVersion(sa.taskHash) }, { setup: sb, task: store.getTaskVersion(sb.taskHash) });
+  const changed = changedCriteria(sa, sb, diff);
+  const criteria = [...new Set([...ids(da), ...ids(db)])].flatMap((criterionId) => configs.map((c): CompareCriterion => {
+    const x = rate(ca, c.configKey, criterionId, inA), y = rate(cb, c.configKey, criterionId, inB);
+    return { criterionId, configKey: c.configKey, a: x, b: y, changed: changed.has(criterionId), moved: x == null || y == null ? x !== y : Math.abs(y - x) >= 1e-9 };
+  }));
+
+  const names = versionNames(store, taskId);
+  const sideOf = (s: GradingSetupRecord, v: View): CompareSide => ({
+    hash: s.hash, name: names.get(s.hash)?.name ?? s.hash.slice(0, 8), nameParts: names.get(s.hash)?.nameParts ?? [], taskHash: s.taskHash, label: v.tasks[taskId]!.label, judgeLabel: s.judgeLabel, firstSeen: s.firstSeen,
+    current: v.tasks[taskId]!.current, def: v.tasks[taskId]!.def,
+  });
+  return {
+    taskId, a: sideOf(sa, va), b: sideOf(sb, vb),
+    diff,
+    sameTask: sa.taskHash === sb.taskHash,
+    trials: both.length, onlyA: ta.size - both.length, onlyB: tb.size - both.length, configs, criteria,
+    reading: both.length ? reading(criteria, diff) : null,
+  };
+}
+
+function reading(rows: CompareCriterion[], diff: GradingDiff): CompareReading {
+  const moved = new Map<string, { changed: boolean; moved: boolean }>();
+  for (const r of rows) {
+    const m = moved.get(r.criterionId);
+    moved.set(r.criterionId, { changed: r.changed, moved: r.moved || !!m?.moved });
+  }
+  const list = [...moved.values()];
+  const changedMoved = list.filter((c) => c.changed && c.moved).length;
+  const changedSteady = list.filter((c) => c.changed && !c.moved).length;
+  const unchangedMoved = list.filter((c) => !c.changed && c.moved).length;
+  return { changedMoved, changedSteady, unchangedMoved, judgeVariation: changedMoved === 0 && unchangedMoved > 0 && diff.criteriaOnly };
+}
+
+/**
+ * The ids of the gates and criteria that differ between the two versions: each one in the difference, and each
+ * command check whose `run` (under either version) names a grader file, by its path in the task folder, that was
+ * added, removed or changed.
+ */
+function changedCriteria(sa: GradingSetupRecord, sb: GradingSetupRecord, diff: GradingDiff): Set<string> {
+  const out = new Set(diff.criteria.map((c) => c.id));
+  const g = diff.files.grader, files = [...g.added, ...g.removed, ...g.changed];
+  if (!files.length) return out;
+  for (const snap of [sa.snapshot, sb.snapshot] as unknown as Partial<GradingContent>[]) {
+    for (const c of [...(snap.gates ?? []), ...(snap.criteria ?? [])]) {
+      if (c.check?.type === "command" && files.some((f) => (c.check as { run: string }).run.includes(f))) out.add(c.id);
+    }
+  }
+  return out;
+}
+
+function resolve(store: Store, prefix: string, which: string): GradingSetupRecord {
+  if (!prefix) throw new Error(`${which}: pick a grading version`);
+  const p = prefix.toLowerCase();
+  const found = store.listGradingSetups().filter((g) => g.hash.startsWith(p));
+  if (found.length === 1) return found[0]!;
+  throw new Error(found.length ? `${which}: grading ${prefix} matches ${found.length} grading versions; give more characters` : `${which}: no grading version ${prefix}`);
+}

package/src/core/config.ts

@@ -0,0 +1,25 @@
+// Agent and human configs: configs/<id>.yaml → AgentConfig, validated and with paths resolved.
+import { homedir } from "node:os";
+import { isAbsolute, join } from "node:path";
+import { HARNESSES, type AgentConfig, type Harness, type HumanOutput } from "./types";
+
+const expand = (p: string, root: string) => (p.startsWith("~/") ? join(homedir(), p.slice(2)) : isAbsolute(p) ? p : join(root, p));
+
+export function normalizeConfig(id: string, raw: any, root: string): AgentConfig {
+  const harness = raw?.harness as Harness;
+  if (!HARNESSES.includes(harness)) throw new Error(`configs/${id}.yaml: harness must be one of ${HARNESSES.join(", ")}`);
+  if ("label" in (raw ?? {})) throw new Error(`configs/${id}.yaml: remove label - it is built from harness, model, effort and profile`);
+  if (harness !== "human") {
+    if (!raw.model) throw new Error(`configs/${id}.yaml: an agent config needs a model`);
+    return { id, ...raw };
+  }
+  const entries = Object.entries(raw.outputs ?? {});
+  if (!entries.length) throw new Error(`configs/${id}.yaml: a human config needs outputs (task id → file)`);
+  const outputs: Record<string, HumanOutput> = {};
+  for (const [task, v] of entries) {
+    const o = typeof v === "string" ? { path: v } : (v as HumanOutput);
+    if (!o?.path) throw new Error(`configs/${id}.yaml: outputs.${task} needs a path`);
+    outputs[task] = { path: expand(o.path, root), ...(o.note ? { note: o.note } : {}) };
+  }
+  return { id, harness, model: "by hand", family: "human", outputs };
+}

package/src/core/copying.ts

@@ -0,0 +1,18 @@
+// Did an agent copy the held-out original (for example by finding the published version on the web)?
+// Share of the output's 8-word runs that also appear in the original. Independent writing on the same
+// topic shares almost none; a copy, even lightly edited, shares most.
+const N = 8;
+const words = (s: string) => s.toLowerCase().replace(/[^a-z0-9'\s]/g, " ").split(/\s+/).filter(Boolean);
+const runs = (ws: string[]) => { const out = new Set<string>(); for (let i = 0; i + N <= ws.length; i++) out.add(ws.slice(i, i + N).join(" ")); return out; };
+
+export function copiedShare(output: string, original: string): number {
+  const mine = runs(words(output));
+  if (!mine.size) return 0;
+  const theirs = runs(words(original));
+  let shared = 0;
+  for (const r of mine) if (theirs.has(r)) shared++;
+  return shared / mine.size;
+}
+
+/** Above this share, the trial is invalid: the output is a copy, not the agent's own work. */
+export const COPY_LIMIT = 0.3;