static-fire 0.44.0 → 0.54.1

package/src/core/trial-detail.ts

@@ -44,8 +44,8 @@ export function trialDetail(store: Store, trialId: string): TrialDetail | null {
   };
 }
 
-/** The trial's output file as the drawer shows it: an image (by the output path's extension) or text. */
-export type TrialOutputFile = { image: Uint8Array<ArrayBuffer>; mediaType: string } | { text: string };
+/** The trial's output file as the drawer shows it: an image (by the output path's extension) or text, with the output path it was read from. */
+export type TrialOutputFile = ({ image: Uint8Array<ArrayBuffer>; mediaType: string } | { text: string }) & { path: string };
 
 /**
  * The trial's output file: the task's `output` path, as recorded with the task version the trial ran on, inside
@@ -69,5 +69,5 @@ export function readTrialOutputFile(store: Store, dbPath: string, trialId: strin
   const rel = relative(realpathSync(workspace), real);
   if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return { missing: `trial ${trialId}: its output path ${output} leaves the workspace` };
   const mediaType = IMAGE_MEDIA[extname(output).toLowerCase()];
-  try { return mediaType ? { image: readFileSync(real), mediaType } : { text: readFileSync(real, "utf8") }; } catch { return none; }   // a folder, or unreadable
+  try { return mediaType ? { image: readFileSync(real), mediaType, path: output } : { text: readFileSync(real, "utf8"), path: output }; } catch { return none; }   // a folder, or unreadable
 }

package/src/core/types.ts

@@ -184,7 +184,7 @@ export interface ConfigSummary {
   profile?: string;
   tasks: Record<string, TaskSummary>;
   suites: Record<string, Stat>;          // Suite Index per suite
-  index: Stat;                           // Workflow Index
+  index: Stat;                           // Agent Performance Index
   costUsd: number | null;                // mean cost per task over all tasks
   durationMs: number | null;             // mean time per task
 }

package/src/core/view-query.ts

@@ -4,12 +4,20 @@
 // the route and the dashboard's loading code pick it up. This file is the one home of the config facet list.
 import type { ViewFilters } from "./view";
 
-/** What the chart is about: the Workflow Index over every suite, one suite, or one task. */
-export type Scope = { kind: "index" } | { kind: "suite"; id: string } | { kind: "task"; id: string };
+/**
+ * What the chart is about: the Agent Performance Index over the suites in view (the picked suites, else the default
+ * set), or one task.
+ */
+export type Scope = { kind: "index" } | { kind: "task"; id: string };
 
 /** The view a page asks for. Every field is optional; an empty object is the default view. */
 export interface ViewQuery {
   scope?: Scope;
+  /**
+   * The picked suites of the index, as the repeated `suite` parameter (`?suite=short&suite=writing`). None picked:
+   * the view chooses the default set. Only with the index scope.
+   */
+  suites?: string[];
   judge?: string;
   grading?: string;
   /**
@@ -24,14 +32,14 @@ export interface ViewQuery {
   until?: string;
 }
 
-/** Scope in a URL: absent or `index`, `suite:<id>`, `task:<id>`. */
-export const scopeParam = (s: Scope): string => (s.kind === "index" ? "index" : `${s.kind}:${s.id}`);
+/** Scope in a URL: absent or `index`, `task:<id>`. A link from before #80 can say `suite:<id>`: that is `suite=<id>`. */
+export const scopeParam = (s: Scope): string => (s.kind === "index" ? "index" : `task:${s.id}`);
 
-function parseScope(v: string): Scope {
-  if (v === "index") return { kind: "index" };
+function parseScope(v: string): { scope: Scope; suite?: string } {
+  if (v === "index") return { scope: { kind: "index" } };
   const m = /^(suite|task):(.+)$/.exec(v);
   if (!m) throw new Error(`scope ${v} is not index, suite:<id> or task:<id>`);
-  return { kind: m[1] as "suite" | "task", id: m[2]! };
+  return m[1] === "task" ? { scope: { kind: "task", id: m[2]! } } : { scope: { kind: "index" }, suite: m[2]! };
 }
 
 /** Parameters that take one value each. A view has one judge, one grading version and one date range. */
@@ -41,8 +49,8 @@ const SINGLE: Single[] = ["judge", "grading", "since", "until"];
 export const CONFIG_FACETS = ["harness", "model", "effort", "profile"] as const;
 export type ConfigFacet = (typeof CONFIG_FACETS)[number];
 /** Every parameter, in the order a URL the page writes lists them. */
-const PARAMS: readonly ("scope" | Single | ConfigFacet)[] = ["scope", "judge", "grading", "harness", "model", "effort", "profile", "since", "until"];
-const KNOWN = new Set<string>(["scope", ...SINGLE, ...CONFIG_FACETS]);
+const PARAMS: readonly ("scope" | "suite" | Single | ConfigFacet)[] = ["scope", "suite", "judge", "grading", "harness", "model", "effort", "profile", "since", "until"];
+const KNOWN = new Set<string>(["scope", "suite", ...SINGLE, ...CONFIG_FACETS]);
 const isConfig = (k: string): k is ConfigFacet => (CONFIG_FACETS as readonly string[]).includes(k);
 
 /** The values in one stable order, each once, with empty ones dropped: a set of picks has one URL. */
@@ -56,7 +64,15 @@ export function parseViewQuery(params: URLSearchParams): ViewQuery {
   if (repeated.length) throw new Error(`view parameter(s) ${repeated.join(", ")} can take only one value (repeat only ${CONFIG_FACETS.join(", ")})`);
   const q: ViewQuery = {};
   const scope = params.get("scope");
-  if (scope) q.scope = parseScope(scope);
+  const suites = params.getAll("suite");
+  if (scope) {
+    const parsed = parseScope(scope);
+    q.scope = parsed.scope;
+    if (parsed.suite) suites.push(parsed.suite);
+  }
+  const picked = valueSet(suites);
+  if (picked.length && q.scope?.kind === "task") throw new Error(`suite cannot be picked with scope ${scope}: a task's view has no suites`);
+  if (picked.length) q.suites = picked;
   for (const k of SINGLE) { const v = params.get(k); if (v) q[k] = v; }
   for (const k of CONFIG_FACETS) { const v = valueSet(params.getAll(k)); if (v.length) q[k] = v; }
   return q;
@@ -66,19 +82,20 @@ export function parseViewQuery(params: URLSearchParams): ViewQuery {
 export function viewQueryParams(q: ViewQuery): URLSearchParams {
   const p = new URLSearchParams();
   if (q.scope && q.scope.kind !== "index") p.set("scope", scopeParam(q.scope));
+  else for (const v of valueSet(q.suites ?? [])) p.append("suite", v);
   for (const k of PARAMS) {
-    if (k === "scope") continue;
+    if (k === "scope" || k === "suite") continue;
     if (isConfig(k)) for (const v of valueSet(q[k] ?? [])) p.append(k, v);
     else if (q[k]) p.set(k, q[k]!);
   }
   return p;
 }
 
-/** The view function's filters for `q`: a suite scope is that suite, a task scope that task. */
+/** The view function's filters for `q`: the picked suites of the index, or a task scope's task. */
 export function toViewFilters(q: ViewQuery): ViewFilters {
   const f: ViewFilters = {};
-  if (q.scope?.kind === "suite") f.suites = [q.scope.id];
   if (q.scope?.kind === "task") f.tasks = [q.scope.id];
+  else if (q.suites?.length) f.suites = valueSet(q.suites);
   for (const k of SINGLE) if (q[k]) f[k] = q[k];
   for (const k of CONFIG_FACETS) { const v = valueSet(q[k] ?? []); if (v.length) f[k] = v; }
   return f;

package/src/core/view.ts

@@ -11,6 +11,8 @@ import type { NamedGradingSetup, VersionName } from "./grading-versions";
 import { gradingHash as currentGradingHash, taskHash as currentTaskHash } from "./identity";
 import type { CriterionContent, GradingContent } from "./identity";
 import { summarize } from "./score";
+import { defaultSuites } from "./coverage";
+import type { Coverage } from "./coverage";
 import { CONFIG_FACETS, valueSet } from "./view-query";
 import type { ConfigFacet } from "./view-query";
 import { loadJudges, loadSuites, loadTask } from "./authored";
@@ -21,9 +23,12 @@ import type { AgentConfig, ConfigSummary, Criterion, Experiment, Grades, JudgeCo
 
 /** What the caller asks for. Every field is optional; an empty object is the default view. */
 export interface ViewFilters {
-  /** Suite ids. Their tasks are in scope and the Workflow Index is the mean over these suites. Default: every suite. */
+  /**
+   * Suite ids. Their tasks are in scope and the Agent Performance Index is the mean over these suites. Default: the
+   * default set of the data in view (coverage.ts defaultSuites, DEC-019).
+   */
   suites?: string[];
-  /** Task ids. When set, only these tasks are in scope and no suite is (so no Suite or Workflow Index). Wins over `suites`. */
+  /** Task ids. When set, only these tasks are in scope and no suite is (so no Suite or Agent Performance Index). Wins over `suites`. */
   tasks?: string[];
   /** Judge id from judges.yaml. Default: the first one (the default judge). */
   judge?: string;
@@ -185,6 +190,8 @@ export interface View {
     /** The values of each config facet applied, sorted (absent when not set). */
     harness?: string[]; model?: string[]; effort?: string[]; profile?: string[];
   };
+  /** Which suites the index covers and why each other suite is left out (DEC-020). Null for a view of tasks. */
+  coverage: Coverage | null;
   /** Task id → the task in scope. */
   tasks: Record<string, ViewTask>;
   /** The trials in the view, by task, config key, then id. */
@@ -198,63 +205,66 @@ export interface View {
 
 /** Selects and summarises the comparable trials for `filters`. Throws on an unknown id or an unusable filter. */
 export function buildView(store: Store, filters: ViewFilters = {}, catalog: Catalog = projectCatalog()): View {
-  const { suites, taskIds } = scope(filters, catalog);
+  const { suites: candidates, taskIds: allTaskIds, picked: suitesPicked } = scope(filters, catalog);
   const since = bound(filters.since, "since"), until = bound(filters.until, "until");
   const picked = filters.grading ? pickGrading(store, filters.grading) : null;
   // A view shows one judge: a picked grading setup's judge, for every task in scope.
   const judge = picked ? setupJudge(picked, filters.judge, catalog.judges) : pickJudge(filters.judge, catalog.judges);
   const pickedTasks = new Set(picked ? store.gradingSetupTaskIds(picked.hash) : []);
-  if (picked && !taskIds.some((t) => pickedTasks.has(t))) {
-    throw new Error(`grading version ${picked.hash} is for task ${[...pickedTasks].join(", ")}, which is not in scope (${taskIds.join(", ")})`);
+  if (picked && !allTaskIds.some((t) => pickedTasks.has(t))) {
+    throw new Error(`grading version ${picked.hash} is for task ${[...pickedTasks].join(", ")}, which is not in scope (${allTaskIds.join(", ")})`);
+  }
+
+  // 1. Gather each task that can be in view (every suite's, when the suites are not picked).
+  const gathered = new Map<string, Gathered>();
+  for (const taskId of allTaskIds) gathered.set(taskId, gatherTask(store, catalog.loadTask(taskId), judge, picked, pickedTasks, filters, since, until));
+
+  // 2. The suites in view: the picked ones, else the default set (DEC-019). A view of tasks has no suites.
+  let suites = candidates;
+  let coverage: Coverage | null = null;
+  if (!filters.tasks?.length) {
+    const completedBy = new Map(catalog.suites.map((s) => [s.id, completers(s, gathered)]));
+    if (!suitesPicked) {
+      const ids = new Set(defaultSuites(candidates.map((s) => s.id), completedBy));
+      suites = candidates.filter((s) => ids.has(s.id));
+      // The default set is the one rule, with no exception for a picked grading version: a view that must show its
+      // task picks that task's suite (the dashboard puts the suite pick in the address when it sets the grading pick).
+      if (picked && !suites.some((s) => s.tasks.some((t) => pickedTasks.has(t)))) {
+        const of = candidates.filter((s) => s.tasks.some((t) => pickedTasks.has(t))).map((s) => s.id);
+        throw new Error(`grading version ${picked.hash} is for task ${[...pickedTasks].join(", ")}, which is not in the default set of suites (${suites.map((s) => s.id).join(", ")}); pick its suite (${of.join(", ")}) or its task to see it`);
+      }
+    }
+    const inView = new Set(suites.map((s) => s.id));
+    const ref = (s: Suite) => ({ id: s.id, title: s.title });
+    const both = (ids: Set<string>[]) => ids.length ? [...ids[0]!].filter((c) => ids.every((x) => x.has(c))).length : 0;
+    coverage = {
+      picked: suitesPicked,
+      covered: suites.map(ref),
+      leftOut: catalog.suites.filter((s) => !inView.has(s.id)).map((s) => {
+        if (suitesPicked) return { ...ref(s), completed: null, reason: "not-picked" as const };
+        const completed = completedBy.get(s.id)!.size;
+        return { ...ref(s), completed, reason: completed ? "too-few" as const : "none-completed" as const };
+      }),
+      complete: both(suites.map((s) => completedBy.get(s.id)!)),
+    };
   }
+  const taskIds = filters.tasks?.length ? allTaskIds : [...new Set(suites.flatMap((s) => s.tasks))];
 
+  // 3. The view of those tasks.
   const tasks: Record<string, ViewTask> = {};
   const inView: ViewTrial[] = [];
   const records = new Map<string, TrialRecord>();   // trial id → its row, for the config fields
   const left = { otherGrading: new Counter(), otherTask: new Counter(), ungraded: new Counter() };
-  const otherVersions = new Map<string, { version: OtherTaskVersion; rows: TrialRecord[] }>();   // with its trials, to count per grading version
+  const versions: OtherTaskVersion[] = [];
   const gradable: string[] = [];
-
   for (const taskId of taskIds) {
-    const task = catalog.loadTask(taskId);
-    const nowTask = currentTaskHash(task), nowGrading = currentGradingHash(task, judge);
-    const names = versionNames(store, taskId);
-    const tHash = pickedTasks.has(taskId) ? picked!.taskHash : nowTask;
-    const gHash = pickedTasks.has(taskId) ? picked!.hash : nowGrading;
-    const stored = store.getTaskVersion(tHash);
-    const label = stored ? stored.label : task.version ?? null;   // a stored task version keeps its label, even none
-    tasks[taskId] = {
-      taskHash: tHash, gradingHash: gHash, gradingName: names.get(gHash)?.name ?? null,   // null: today's grading version is not stored yet
-      label,
-      current: tHash === nowTask && gHash === nowGrading,
-      def: gHash === nowGrading ? snapshotOf(task) : taskDefinition(task, picked!, label),
-    };
-
-    const rows = store.listTrials({ taskId }).filter((r) => ENDED.has(r.status) && inRange(r.startedAt, since, until) && matchesConfig(r, filters));
-    const same: TrialRecord[] = [];
-    for (const r of rows) {
-      if (r.taskHash === tHash) { same.push(r); continue; }
-      left.otherTask.add(r.configKey);
-      const key = `${taskId}\0${r.taskHash}`;   // two tasks can share an assignment (task hash): each keeps its own entry
-      const v = otherVersions.get(key) ?? otherVersions.set(key, { version: otherVersion(store, taskId, r.taskHash, names), rows: [] }).get(key)!;
-      v.version.trials++;
-      v.rows.push(r);
-    }
-    for (const r of latestHumanOnly(same)) {
-      records.set(r.id, r);
-      if (r.status !== "done") { inView.push(viewTrial(r, null, [])); continue; }   // failed, or copied the held-out original
-      const g = store.currentGrading(r.id, gHash);
-      if (g) { inView.push(viewTrial(r, g, store.listVerdicts(g.id))); continue; }
-      const other = store.listGradings(r.id).some((x) => x.status !== "grading");
-      (other ? left.otherGrading : left.ungraded).add(r.configKey);
-      gradable.push(r.id);
-    }
-  }
-
-  for (const { version: v, rows } of otherVersions.values()) {
-    const shown = latestHumanOnly(rows).filter((r) => r.status === "done");
-    for (const g of v.gradingSetups) g.trials = shown.filter((r) => store.currentGrading(r.id, g.hash)?.status === "done").length;
-    v.link = linkTarget(v.gradingSetups, judge);
+    const g = gathered.get(taskId)!;
+    tasks[taskId] = g.task;
+    inView.push(...g.trials);
+    for (const r of g.records) records.set(r.id, r);
+    for (const k of ["otherGrading", "otherTask", "ungraded"] as const) for (const key of g.left[k]) left[k].add(key);
+    versions.push(...g.versions);
+    gradable.push(...g.gradable);
   }
 
   inView.sort((a, b) => cmp(a.taskId, b.taskId) || cmp(a.configKey, b.configKey) || cmp(a.id, b.id));
@@ -263,30 +273,98 @@ export function buildView(store: Store, filters: ViewFilters = {}, catalog: Cata
       suiteIds: suites.map((s) => s.id), taskIds, judge, grading: picked?.hash ?? null, since, until,
       ...Object.fromEntries(CONFIG_FACETS.map((k) => [k, valueSet(filters[k] ?? [])] as const).filter(([, v]) => v.length)),
     },
+    coverage,
     tasks,
     trials: inView,
     summary: summarizeTrials(inView, records, tasks, suites),
     leftOut: {
       otherGrading: left.otherGrading.count(),
-      otherTask: { ...left.otherTask.count(), versions: [...otherVersions.values()].map((x) => x.version) },
+      otherTask: { ...left.otherTask.count(), versions },
       ungraded: left.ungraded.count(),
       gradable: gradable.sort(),
     },
   };
 }
 
+/** One task's part of a view: its definition in view, its trials in view, and its left-out trials by config key. */
+interface Gathered {
+  task: ViewTask;
+  trials: ViewTrial[];
+  records: TrialRecord[];
+  left: { otherGrading: string[]; otherTask: string[]; ungraded: string[] };
+  versions: OtherTaskVersion[];
+  gradable: string[];
+}
+
+function gatherTask(store: Store, task: Task, judge: JudgeConfig, picked: GradingSetupRecord | null, pickedTasks: Set<string>,
+  filters: ViewFilters, since: string | null, until: string | null): Gathered {
+  const taskId = task.id;
+  const nowTask = currentTaskHash(task), nowGrading = currentGradingHash(task, judge);
+  const names = versionNames(store, taskId);
+  const tHash = pickedTasks.has(taskId) ? picked!.taskHash : nowTask;
+  const gHash = pickedTasks.has(taskId) ? picked!.hash : nowGrading;
+  const stored = store.getTaskVersion(tHash);
+  const label = stored ? stored.label : task.version ?? null;   // a stored task version keeps its label, even none
+  const out: Gathered = {
+    task: {
+      taskHash: tHash, gradingHash: gHash, gradingName: names.get(gHash)?.name ?? null,   // null: today's grading version is not stored yet
+      label,
+      current: tHash === nowTask && gHash === nowGrading,
+      def: gHash === nowGrading ? snapshotOf(task) : taskDefinition(task, picked!, label),
+    },
+    trials: [], records: [], left: { otherGrading: [], otherTask: [], ungraded: [] }, versions: [], gradable: [],
+  };
+
+  const rows = store.listTrials({ taskId }).filter((r) => ENDED.has(r.status) && inRange(r.startedAt, since, until) && matchesConfig(r, filters));
+  const same: TrialRecord[] = [];
+  const otherVersions = new Map<string, { version: OtherTaskVersion; rows: TrialRecord[] }>();   // with its trials, to count per grading version
+  for (const r of rows) {
+    if (r.taskHash === tHash) { same.push(r); continue; }
+    out.left.otherTask.push(r.configKey);
+    const v = otherVersions.get(r.taskHash) ?? otherVersions.set(r.taskHash, { version: otherVersion(store, taskId, r.taskHash, names), rows: [] }).get(r.taskHash)!;
+    v.version.trials++;
+    v.rows.push(r);
+  }
+  for (const r of latestHumanOnly(same)) {
+    out.records.push(r);
+    if (r.status !== "done") { out.trials.push(viewTrial(r, null, [])); continue; }   // failed, or copied the held-out original
+    const g = store.currentGrading(r.id, gHash);
+    if (g) { out.trials.push(viewTrial(r, g, store.listVerdicts(g.id))); continue; }
+    const other = store.listGradings(r.id).some((x) => x.status !== "grading");
+    (other ? out.left.otherGrading : out.left.ungraded).push(r.configKey);
+    out.gradable.push(r.id);
+  }
+
+  for (const { version: v, rows } of otherVersions.values()) {
+    const shown = latestHumanOnly(rows).filter((r) => r.status === "done");
+    for (const g of v.gradingSetups) g.trials = shown.filter((r) => store.currentGrading(r.id, g.hash)?.status === "done").length;
+    v.link = linkTarget(v.gradingSetups, judge);
+    out.versions.push(v);
+  }
+  return out;
+}
+
+/** The configs (config keys) that have a valid score on every task of `suite` in what was gathered. */
+function completers(suite: Suite, gathered: Map<string, Gathered>): Set<string> {
+  const per = suite.tasks.map((t) => new Set((gathered.get(t)?.trials ?? []).filter((x) => x.status === "done" && x.grades).map((x) => x.configKey)));
+  if (!per.length) return new Set();
+  return new Set([...per[0]!].filter((c) => per.every((p) => p.has(c))));
+}
+
 // ---- Selection ---------------------------------------------------------------------------------
 
 const ENDED = new Set(["done", "failed", "invalid"]);
 const cmp = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);
 
-function scope(f: ViewFilters, catalog: Catalog): { suites: Suite[]; taskIds: string[] } {
-  if (f.tasks?.length) return { suites: [], taskIds: [...new Set(f.tasks)] };
+function scope(f: ViewFilters, catalog: Catalog): { suites: Suite[]; taskIds: string[]; picked: boolean } {
+  if (f.tasks?.length) return { suites: [], taskIds: [...new Set(f.tasks)], picked: false };
   const byId = new Map(catalog.suites.map((s) => [s.id, s]));
-  const suites = f.suites?.length
-    ? [...new Set(f.suites)].map((id) => byId.get(id) ?? fail(`no suite ${id} (suites: ${[...byId.keys()].join(", ") || "none"})`))
-    : catalog.suites;
-  return { suites, taskIds: [...new Set(suites.flatMap((s) => s.tasks))] };
+  const picked = !!f.suites?.length;
+  // Picked suites are in catalog order, so a set of picks has one view whatever order it was given in.
+  const ids = new Set(f.suites ?? []);
+  for (const id of ids) if (!byId.has(id)) fail(`no suite ${id} (suites: ${[...byId.keys()].join(", ") || "none"})`);
+  const suites = picked ? catalog.suites.filter((s) => ids.has(s.id)) : catalog.suites;
+  return { suites, taskIds: [...new Set(suites.flatMap((s) => s.tasks))], picked };
 }
 
 function pickJudge(id: string | undefined, judges: JudgeConfig[]): JudgeConfig {