static-fire 0.0.0-stage → 0.44.0

package/src/core/types.ts

@@ -0,0 +1,195 @@
+// The primitives (see CONTEXT.md): Suite → Task → Measure → Criterion → Grader → Verdict.
+// Trials, gradings and batches are records in the record store (src/core/db.ts).
+
+/** Agent harnesses run a model headless. `human` is a person: its output is a file they wrote. */
+export type AgentHarness = "claude" | "codex" | "pi" | "agy";
+export type Harness = AgentHarness | "human";
+export const HARNESSES: Harness[] = ["claude", "codex", "pi", "agy", "human"];
+
+export interface HumanOutput { path: string; note?: string }
+
+export interface AgentConfig {
+  id: string;            // file name in configs/ (the label is built from the fields: core/label.ts)
+  harness: Harness;
+  model: string;         // "by hand" for a human config
+  effort?: string;
+  provider?: string;     // pi only
+  profile?: string;      // apkit Profile name; omitted = no Profile
+  family?: string;       // model family for chart colour; defaults to a guess from `model`
+  outputs?: Record<string, HumanOutput>;  // human only: task id → the file they wrote (absolute path)
+}
+
+export interface JudgeConfig {
+  id: string;
+  label: string;
+  harness: Exclude<AgentHarness, "pi">;   // judge Hosts must support JSON Schema output
+  model: string;
+  effort?: string;
+}
+
+// ---- Task definition -------------------------------------------------------------------------
+
+export type Level = "floor" | "strong" | "ceiling";
+export type Evidence = "output" | "inputs" | "references" | "project" | "sources";
+
+export type Check =
+  | { type: "regex"; pattern: string; flags?: string; min?: number; max?: number }
+  | { type: "words"; min?: number; max?: number }
+  | { type: "frontmatter"; field: string; minChars?: number; maxChars?: number }
+  | { type: "command"; run: string; timeoutSec?: number };
+
+export interface Criterion {
+  id: string;
+  question: string;           // phrased so that "yes" = pass
+  level: Level;
+  grader: "check" | "judge" | "human";
+  check?: Check;              // grader: check
+  evidence?: Evidence[];      // grader: judge (default: ["output"])
+  isolate?: boolean;          // grader: judge — its own call instead of the measure's batch
+  derived?: string;           // grader: judge — a script under the task's grader/ folder that makes an image from the output; the judge sees it (README, "Derived evidence")
+}
+
+export interface Measure {
+  id: string;
+  label: string;
+  weight?: number;            // default 1
+  criteria: Criterion[];
+}
+
+export interface Task {
+  id: string;
+  version?: number;           // a readable label only, in neither hash; a task file can leave it out
+  title: string;
+  description: string;
+  prompt: string;
+  inputs: string;             // copied into the trial workspace
+  output: string;             // path the agent must write, relative to the workspace
+  references?: string;        // graders only
+  project?: string;           // files the agent would find in the real project (copied into the workspace root, not shown to graders)
+  sources?: string;           // graders only: saved copies of the pages the notes link to, for fact checks
+  heldOut?: string;           // the real original this task was built from; an agent output that copies it is invalid
+  context?: string;           // one paragraph a judge reads before the criteria
+  gates: Criterion[];
+  measures: Measure[];
+  dir: string;                // absolute task directory (set by the loader)
+}
+
+export interface Suite {
+  id: string;
+  title: string;
+  description?: string;
+  tasks: string[];
+  weights?: Record<string, number>;
+}
+
+// ---- Results -----------------------------------------------------------------------------------
+
+export interface Usage {
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadTokens: number;
+  cacheWriteTokens: number;
+}
+
+export interface AgentResult {
+  ok: boolean;
+  error?: string;
+  usage: Usage;
+  costUsd: number | null;
+  costSource: "reported" | "estimated" | "unknown";
+  turns?: number;
+}
+
+export interface Vote { pass: boolean; reason: string }
+
+export interface Verdict {
+  score: number | null;       // 0–1 (judge: share of votes passed); null = error
+  votes: Vote[];
+  error?: string;
+}
+
+export interface Grades {
+  verdicts: Record<string, Verdict>;     // by criterion id, gates included
+  gatesPassed: boolean;
+  measures: Record<string, number>;      // 0–100 per measure
+  score: number;                         // 0–100 task score
+  errors: string[];                      // criterion ids that errored (→ trial invalid)
+  judgeCostUsd: number;
+}
+
+export type TrialStatus = "queued" | "running" | "grading" | "done" | "invalid" | "failed";
+
+export interface Trial {
+  id: string;                 // the trial id
+  experimentId: string;       // the batch id (the name stays because tests/score.test.ts sets it)
+  taskId: string;
+  configId: string;
+  index: number;
+  status: TrialStatus;
+  startedAt?: string;
+  endedAt?: string;
+  durationMs?: number;        // agent wall-clock only
+  result?: AgentResult;
+  grades?: Grades;
+  error?: string;
+}
+
+/**
+ * The input summarize() in score.ts takes: the tasks, suites and configs of one view (view.ts builds it).
+ * It is also the shape of the old `experiment.json` that migrate.ts imports. Nothing writes it any more.
+ * The name stays because score.ts and tests/score.test.ts use it, and both stay as they are.
+ */
+export interface Experiment {
+  schema: 2;
+  id: string;
+  suiteIds: string[];
+  tasks: Record<string, number | null>;   // task id → version at run time (null: the task file has none)
+  configIds: string[];
+  trialsPerConfig: number;
+  judge: JudgeConfig & { votes: number };
+  createdAt: string;
+  status: "running" | "done";
+  configs: Record<string, AgentConfig>;  // snapshot at run time
+  suites: Record<string, Suite>;         // snapshot at run time
+  userSetup?: { files: Record<string, string> };         // the user-level setup trials ran with (path → hash)
+  blockedTools?: Partial<Record<AgentHarness, string[]>>;      // outside-service tools blocked in trials
+  gradedAt?: string;                     // when the current grading started; createdAt before any regrade
+  taskDefs?: Record<string, TaskSnapshot>; // each task's criteria as the current grading used them
+}
+
+/** A task's definition as one grading setup used it, so an earlier grading setup shows its own criteria. */
+export type TaskSnapshot = Pick<Task, "id" | "title" | "description" | "version" | "gates" | "measures">;
+
+// ---- Summaries (what the dashboard and `static-fire show` read) -------------------------------------
+
+export interface Stat { mean: number | null; ci: number | null; n: number }
+
+export interface TaskSummary {
+  score: Stat;
+  measures: Record<string, number>;      // mean 0–100 per measure
+  criteria: Record<string, number>;      // mean pass rate 0–1 per criterion
+  costUsd: number | null;
+  costSource: AgentResult["costSource"];
+  durationMs: number | null;
+  valid: number; invalid: number; failed: number; total: number;
+}
+
+export interface ConfigSummary {
+  configId: string;
+  label: string;
+  harness: Harness;
+  model: string;
+  family: string;
+  effort?: string;
+  profile?: string;
+  tasks: Record<string, TaskSummary>;
+  suites: Record<string, Stat>;          // Suite Index per suite
+  index: Stat;                           // Workflow Index
+  costUsd: number | null;                // mean cost per task over all tasks
+  durationMs: number | null;             // mean time per task
+}
+
+export interface ExperimentSummary {
+  configs: ConfigSummary[];
+  deadCriteria: Record<string, string[]>; // task id → criterion ids every config passed or failed
+}

package/src/core/view-query.ts

@@ -0,0 +1,85 @@
+// The query of GET /api/view, shared by the server and the dashboard (no Node imports here).
+// One table of parameters: a filter added to the view is one more field in ViewQuery and one more entry in
+// SINGLE (one value) or CONFIG_FACETS (repeated, several values), and in PARAMS for its place in the URL. Both
+// the route and the dashboard's loading code pick it up. This file is the one home of the config facet list.
+import type { ViewFilters } from "./view";
+
+/** What the chart is about: the Workflow Index over every suite, one suite, or one task. */
+export type Scope = { kind: "index" } | { kind: "suite"; id: string } | { kind: "task"; id: string };
+
+/** The view a page asks for. Every field is optional; an empty object is the default view. */
+export interface ViewQuery {
+  scope?: Scope;
+  judge?: string;
+  grading?: string;
+  /**
+   * Config facets: the picked values of each. A trial matches a facet when its value is any of them.
+   * `none` selects configs without an effort or without a Profile.
+   */
+  harness?: string[];
+  model?: string[];
+  effort?: string[];
+  profile?: string[];
+  since?: string;
+  until?: string;
+}
+
+/** Scope in a URL: absent or `index`, `suite:<id>`, `task:<id>`. */
+export const scopeParam = (s: Scope): string => (s.kind === "index" ? "index" : `${s.kind}:${s.id}`);
+
+function parseScope(v: string): Scope {
+  if (v === "index") return { kind: "index" };
+  const m = /^(suite|task):(.+)$/.exec(v);
+  if (!m) throw new Error(`scope ${v} is not index, suite:<id> or task:<id>`);
+  return { kind: m[1] as "suite" | "task", id: m[2]! };
+}
+
+/** Parameters that take one value each. A view has one judge, one grading version and one date range. */
+type Single = "judge" | "grading" | "since" | "until";
+const SINGLE: Single[] = ["judge", "grading", "since", "until"];
+/** The config facets, each a field of the trial row: the parameter repeats for each picked value (`?model=opus&model=haiku`). */
+export const CONFIG_FACETS = ["harness", "model", "effort", "profile"] as const;
+export type ConfigFacet = (typeof CONFIG_FACETS)[number];
+/** Every parameter, in the order a URL the page writes lists them. */
+const PARAMS: readonly ("scope" | Single | ConfigFacet)[] = ["scope", "judge", "grading", "harness", "model", "effort", "profile", "since", "until"];
+const KNOWN = new Set<string>(["scope", ...SINGLE, ...CONFIG_FACETS]);
+const isConfig = (k: string): k is ConfigFacet => (CONFIG_FACETS as readonly string[]).includes(k);
+
+/** The values in one stable order, each once, with empty ones dropped: a set of picks has one URL. */
+export const valueSet = (values: Iterable<string>): string[] => [...new Set(values)].filter(Boolean).sort();
+
+/** Reads a query string. Throws on an unknown or repeated parameter or a bad scope, so a filter is never silently ignored. */
+export function parseViewQuery(params: URLSearchParams): ViewQuery {
+  const unknown = [...new Set([...params.keys()].filter((k) => !KNOWN.has(k)))];
+  if (unknown.length) throw new Error(`unknown view parameter(s): ${unknown.join(", ")} (known: ${PARAMS.join(", ")})`);
+  const repeated = (["scope", ...SINGLE] as string[]).filter((k) => params.getAll(k).length > 1);
+  if (repeated.length) throw new Error(`view parameter(s) ${repeated.join(", ")} can take only one value (repeat only ${CONFIG_FACETS.join(", ")})`);
+  const q: ViewQuery = {};
+  const scope = params.get("scope");
+  if (scope) q.scope = parseScope(scope);
+  for (const k of SINGLE) { const v = params.get(k); if (v) q[k] = v; }
+  for (const k of CONFIG_FACETS) { const v = valueSet(params.getAll(k)); if (v.length) q[k] = v; }
+  return q;
+}
+
+/** The query string for `q`: only the fields that are set; the index scope is the default, so it is left out. */
+export function viewQueryParams(q: ViewQuery): URLSearchParams {
+  const p = new URLSearchParams();
+  if (q.scope && q.scope.kind !== "index") p.set("scope", scopeParam(q.scope));
+  for (const k of PARAMS) {
+    if (k === "scope") continue;
+    if (isConfig(k)) for (const v of valueSet(q[k] ?? [])) p.append(k, v);
+    else if (q[k]) p.set(k, q[k]!);
+  }
+  return p;
+}
+
+/** The view function's filters for `q`: a suite scope is that suite, a task scope that task. */
+export function toViewFilters(q: ViewQuery): ViewFilters {
+  const f: ViewFilters = {};
+  if (q.scope?.kind === "suite") f.suites = [q.scope.id];
+  if (q.scope?.kind === "task") f.tasks = [q.scope.id];
+  for (const k of SINGLE) if (q[k]) f[k] = q[k];
+  for (const k of CONFIG_FACETS) { const v = valueSet(q[k] ?? []); if (v.length) f[k] = v; }
+  return f;
+}