pinecall 0.1.0

package/dist/cli/testing/latency.js

@@ -0,0 +1,44 @@
+/** The latency a golden was answered with: the median of every turn's own metrics entry. */
+import { TURN_METRICS } from "../view.js";
+/**
+ * The medians of one call. The median and not the mean: one cold first turn would drag an average
+ * of four turns somewhere no caller ever waited, and the number here is meant to be the one a
+ * typical caller felt. Nothing is computed that the log does not already carry — a metric the
+ * session never measured is absent from the answer rather than reported as a zero.
+ */
+export function mediansOf(entries) {
+    const medians = {};
+    for (const name of TURN_METRICS) {
+        const measured = takenPerTurn(entries, name);
+        if (measured.length > 0)
+            medians[name] = median(measured);
+    }
+    return medians;
+}
+/** The medians on one line, in the order a person reads them: the wait first, then where it went. */
+export function latencyLine(medians) {
+    const parts = TURN_METRICS.filter((name) => medians[name] !== undefined).map((name) => `${name} ${Math.round(medians[name] * 1000)}ms`);
+    return parts.join(" · ");
+}
+// Every finished agent turn carries the block livekit measured for it under livekit's own field
+// names, which is where these three are read from — never from a second, summarised copy.
+function takenPerTurn(entries, name) {
+    const taken = [];
+    for (const entry of entries) {
+        if (entry.type !== "turn.agent")
+            continue;
+        const metrics = entry.data["metrics"];
+        const value = typeof metrics === "object" && metrics !== null
+            ? metrics[name]
+            : undefined;
+        if (typeof value === "number")
+            taken.push(value);
+    }
+    return taken;
+}
+function median(values) {
+    const sorted = [...values].sort((one, two) => one - two);
+    const middle = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
+}
+//# sourceMappingURL=latency.js.map

package/dist/cli/testing/latency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"latency.js","sourceRoot":"","sources":["../../../src/cli/testing/latency.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAE5F,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAM1C;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,OAAgB;IACxC,MAAM,OAAO,GAAY,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,WAAW,CAAC,OAAgB;IAC1C,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,GAAG,CAC1E,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAE,GAAG,IAAI,CAAC,IAAI,CAC3D,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,gGAAgG;AAChG,0FAA0F;AAC1F,SAAS,YAAY,CAAC,OAAgB,EAAE,IAAY;IAClD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY;YAAE,SAAS;QAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI;YAC3D,CAAC,CAAE,OAAmC,CAAC,IAAI,CAAC;YAC5C,CAAC,CAAC,SAAS,CAAC;QACd,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,CAAC;AACjG,CAAC"}

package/dist/cli/testing/matrix.d.ts

@@ -0,0 +1,12 @@
+/** A finished run as a person reads it: a line per golden, the evidence under the ones that broke. */
+import type { EvalRun } from "./gateway.js";
+import { type Medians } from "./latency.js";
+export declare const DECLARED = "declared";
+/** The medians of each call the run opened, by call id: what `latency.ts` read off each log. */
+export type Latencies = Record<string, Medians>;
+/**
+ * The whole report. `declaredAs` is what the tenant's class wrote — `llm = "haiku"` — so the
+ * header names the model a person recognises instead of the runner's placeholder column.
+ */
+export declare function reportOf(run: EvalRun, latencies: Latencies, declaredAs: string): string[];
+//# sourceMappingURL=matrix.d.ts.map

package/dist/cli/testing/matrix.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matrix.d.ts","sourceRoot":"","sources":["../../../src/cli/testing/matrix.ts"],"names":[],"mappings":"AAAA,sGAAsG;AAEtG,OAAO,KAAK,EAAQ,OAAO,EAAiB,MAAM,cAAc,CAAC;AACjE,OAAO,EAAe,KAAK,OAAO,EAAE,MAAM,cAAc,CAAC;AAOzD,eAAO,MAAM,QAAQ,aAAa,CAAC;AAEnC,gGAAgG;AAChG,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEhD;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAqBzF"}

package/dist/cli/testing/matrix.js

@@ -0,0 +1,103 @@
+/** A finished run as a person reads it: a line per golden, the evidence under the ones that broke. */
+import { latencyLine } from "./latency.js";
+// The two marks a verdict wears, from the one module that defines them: a judge that held and one
+// that did not mean the same thing here as they do on a call's own score.
+import { BROKEN, HELD, reasonOf } from "./score.js";
+// The column a conversation is drawn under when nobody named a model — the runner's own word for
+// "whatever the agent declared". The report puts the class's own word in its place.
+export const DECLARED = "declared";
+/**
+ * The whole report. `declaredAs` is what the tenant's class wrote — `llm = "haiku"` — so the
+ * header names the model a person recognises instead of the runner's placeholder column.
+ */
+export function reportOf(run, latencies, declaredAs) {
+    const matrix = run.matrix;
+    if (matrix === null)
+        return [`${run.agent}  ${run.status}${run.error === null ? "" : `: ${run.error}`}`];
+    const models = matrix.models.map((model) => named(model, declaredAs));
+    const counted = `${matrix.goldens.length} golden${matrix.goldens.length === 1 ? "" : "s"}`;
+    const lines = [`${run.agent} · ${counted} · ${models.join(" · ")}`];
+    // With two models each golden is drawn twice, so the model heads its own block; with one there
+    // is nothing to head and the goldens sit directly under the line that already names it.
+    const compared = matrix.models.length > 1;
+    for (const model of matrix.models) {
+        if (compared)
+            lines.push(`  ${named(model, declaredAs)}`);
+        for (const cell of matrix.runs.filter((one) => one.model === model)) {
+            lines.push(...linesOfCell(cell, callOf(run, cell), latencies, compared));
+        }
+    }
+    lines.push(...divergences(matrix, declaredAs), footer(run, matrix));
+    // A run that stopped keeps the cells it had scored, so it has both a matrix and a reason. The
+    // reason goes last, under the count: without it the footer reads as a score rather than as a
+    // fragment — `8/20 held` of a suite whose app walked out after eight.
+    if (run.error !== null)
+        lines.push(`  ${run.error}`);
+    return lines;
+}
+/** What the calls themselves cost in provider fees, summed from each one's own `call.summary`. */
+function callCost(matrix) {
+    return matrix.runs.reduce((total, cell) => total + (cell.summary?.cost?.eur ?? 0), 0);
+}
+// One golden under one model: the mark, its name, and — only when something broke — a line per
+// judge that did not hold, and the call to open. A green golden is one line, because a suite of
+// thirty green goldens must stay readable on one screen.
+function linesOfCell(cell, call, latencies, indented) {
+    const pad = indented ? "    " : "  ";
+    const broken = cell.scores.filter((score) => !score.passed);
+    const measured = latencyLine(latencies[call] ?? {});
+    if (broken.length === 0) {
+        return [`${pad}${HELD} ${cell.golden}${measured === "" ? "" : `  ${measured}`}`];
+    }
+    const width = Math.max(...broken.map((score) => score.metric.length), "log".length);
+    return [
+        `${pad}${BROKEN} ${cell.golden}`,
+        ...broken.map((score) => `${pad}      ${score.metric.padEnd(width)}  broken   ${reasonOf(score)}`),
+        `${pad}      ${"log".padEnd(width)}           ${call}`,
+    ];
+}
+// The one thing a matrix is for: a golden two models do not agree about. Same judge, same golden,
+// one model holding it and another not — printed on its own, because it is the finding.
+function divergences(matrix, declaredAs) {
+    if (matrix.models.length < 2)
+        return [];
+    const lines = [];
+    for (const golden of matrix.goldens) {
+        for (const metric of matrix.metrics) {
+            const known = matrix.models
+                .map((model) => ({ model, score: scoreAt(matrix, model, golden, metric) }))
+                .filter((one) => one.score !== undefined);
+            if (known.length < 2)
+                continue;
+            if (new Set(known.map((one) => one.score.passed)).size === 1)
+                continue;
+            const said = known
+                .map((one) => `${one.score.passed ? HELD : BROKEN} ${named(one.model, declaredAs)}`)
+                .join("   ");
+            lines.push(`      ${golden}  ${metric}  ${said}`);
+        }
+    }
+    return lines.length === 0 ? [] : ["  divergences", ...lines];
+}
+// The last line, and the one read first: how many held, what the judge was asked, what the calls
+// cost, and how long a person waited for it.
+function footer(run, matrix) {
+    const cells = matrix.runs.length;
+    const held = cells - new Set(matrix.failures.map((one) => `${one.model} ${one.golden}`)).size;
+    const seconds = Math.round((run.finished_at ?? run.started_at) - run.started_at);
+    const cost = callCost(matrix).toFixed(4);
+    const asked = matrix.judge_calls;
+    return `  ${held}/${cells} · ${asked} judge call${asked === 1 ? "" : "s"} · ${cost} EUR · ${seconds}s`;
+}
+function scoreAt(matrix, model, golden, metric) {
+    const cell = matrix.runs.find((one) => one.model === model && one.golden === golden);
+    return cell?.scores.find((one) => one.metric === metric);
+}
+function callOf(run, cell) {
+    const opened = run.calls.find((one) => one.golden === cell.golden && one.model === cell.model);
+    return opened?.call ?? "";
+}
+function named(model, declaredAs) {
+    return model === DECLARED ? declaredAs : model;
+}
+//# sourceMappingURL=matrix.js.map

package/dist/cli/testing/matrix.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matrix.js","sourceRoot":"","sources":["../../../src/cli/testing/matrix.ts"],"names":[],"mappings":"AAAA,sGAAsG;AAGtG,OAAO,EAAE,WAAW,EAAgB,MAAM,cAAc,CAAC;AACzD,kGAAkG;AAClG,0EAA0E;AAC1E,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEpD,iGAAiG;AACjG,oFAAoF;AACpF,MAAM,CAAC,MAAM,QAAQ,GAAG,UAAU,CAAC;AAKnC;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAY,EAAE,SAAoB,EAAE,UAAkB;IAC7E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC1B,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,CAAC,GAAG,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACzG,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,UAAU,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC3F,MAAM,KAAK,GAAG,CAAC,GAAG,GAAG,CAAC,KAAK,MAAM,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACpE,+FAA+F;IAC/F,wFAAwF;IACxF,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC;QAC1D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,KAAK,CAAC,EAAE,CAAC;YACpE,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IACpE,8FAA8F;IAC9F,6FAA6F;IAC7F,sEAAsE;IACtE,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kGAAkG;AAClG,SAAS,QAAQ,CAAC,MAAc;IAC9B,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxF,CAAC;AAED,+FAA+F;AAC/F,gGAAgG;AAChG,yDAAyD;AACzD,SAAS,WAAW,CAAC,IAAU,EAAE,IAAY,EAAE,SAAoB,EAAE,QAAiB;IACpF,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,GAAG,GAAG,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpF,OAAO;QACL,GAAG,GAAG,GAAG,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE;QAChC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,SAAS,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAClG,GAAG,GAAG,SAAS,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE;KACvD,CAAC;AACJ,CAAC;AAED,kGAAkG;AAClG,wFAAwF;AACxF,SAAS,WAAW,CAAC,MAAc,EAAE,UAAkB;IACrD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM;iBACxB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;iBAC1E,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAC/B,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;gBAAE,SAAS;YACxE,MAAM,IAAI,GAAG,KAAK;iBACf,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;iBACpF,IAAI,CAAC,KAAK,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,GAAG,KAAK,CAAC,CAAC;AAC/D,CAAC;AAED,iGAAiG;AACjG,6CAA6C;AAC7C,SAAS,MAAM,CAAC,GAAY,EAAE,MAAc;IAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;IACjC,MAAM,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9F,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC;IACjC,OAAO,KAAK,IAAI,IAAI,KAAK,MAAM,KAAK,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,UAAU,OAAO,GAAG,CAAC;AACzG,CAAC;AAED,SAAS,OAAO,CAAC,MAAc,EAAE,KAAa,EAAE,MAAc,EAAE,MAAc;IAC5E,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IACrF,OAAO,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,MAAM,CAAC,GAAY,EAAE,IAAU;IACtC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/F,OAAO,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,EAAE,UAAkB;IAC9C,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;AACjD,CAAC"}

package/dist/cli/testing/progress.d.ts

@@ -0,0 +1,26 @@
+/** What `pinecall test` draws while the runtime works: the header, a line per golden, a bar, in place. */
+import { type Door, type EvalRun } from "./gateway.js";
+/** What this terminal knows before the gateway has a row: whose run, its size, and its names. */
+export interface Watched {
+    agent: string;
+    goldens: number;
+    /** The models as a person says them: the class's own word, or the ones named on the command line. */
+    models: string[];
+    /** The word the runner's `declared` column is printed as — the same word `matrix.ts` is handed. */
+    declaredAs: string;
+}
+/**
+ * The run, followed until the POST that started it answers. The runner rewrites the run's row as
+ * it goes — the call before its first turn, the cell the moment it is judged — so this reads that
+ * row every half second and draws it; the answer to the POST is the run that is returned. On a
+ * stream that is not a terminal only the header is written, and under `--json` nothing at all, so
+ * a CI log never fills with redraws and a JSON reader never meets a line that is not JSON.
+ *
+ * `sharing` is every other stream that reaches the same terminal, hooked for as long as the block
+ * is live; a test hands a fake stream and an empty list, so the real process is never touched.
+ */
+export declare function followed(door: Door, watched: Watched, pending: Promise<EvalRun>, out: NodeJS.WritableStream, asJson: boolean, sharing?: NodeJS.WritableStream[]): Promise<EvalRun>;
+/** The first line, the moment the run has an id: whose suite, how big, under which model, which run. */
+export declare function header(watched: Watched, id: string): string;
+export declare function inFlight(door: Door, agent: string): Promise<string | undefined>;
+//# sourceMappingURL=progress.d.ts.map

package/dist/cli/testing/progress.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../../../src/cli/testing/progress.ts"],"names":[],"mappings":"AAAA,0GAA0G;AAE1G,OAAO,EAA8B,KAAK,IAAI,EAAE,KAAK,OAAO,EAAE,MAAM,cAAc,CAAC;AAMnF,iGAAiG;AACjG,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,qGAAqG;IACrG,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,mGAAmG;IACnG,UAAU,EAAE,MAAM,CAAC;CACpB;AAWD;;;;;;;;;GASG;AACH,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,EACzB,GAAG,EAAE,MAAM,CAAC,cAAc,EAC1B,MAAM,EAAE,OAAO,EACf,OAAO,GAAE,MAAM,CAAC,cAAc,EAAmD,GAChF,OAAO,CAAC,OAAO,CAAC,CA0BlB;AAwBD,wGAAwG;AACxG,wBAAgB,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAG3D;AAID,wBAAsB,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAGrF"}

package/dist/cli/testing/progress.js

@@ -0,0 +1,185 @@
+/** What `pinecall test` draws while the runtime works: the header, a line per golden, a bar, in place. */
+import { oneRun, theRuns } from "./gateway.js";
+import { DECLARED, reportOf } from "./matrix.js";
+// How often the run's row is read while it is going: a local gateway answers in a millisecond.
+const EVERY_MS = 500;
+const OPEN = "○";
+const READING = "reading…";
+const BAR_WIDTH = 20;
+const FILLED = "█";
+const EMPTY = "░";
+// How much of a call id the live line shows: enough to tell two apart, and the whole of it is on
+// the report the moment the golden breaks, and on `runs show` always.
+const A_CALL_SHOWN = 12;
+/**
+ * The run, followed until the POST that started it answers. The runner rewrites the run's row as
+ * it goes — the call before its first turn, the cell the moment it is judged — so this reads that
+ * row every half second and draws it; the answer to the POST is the run that is returned. On a
+ * stream that is not a terminal only the header is written, and under `--json` nothing at all, so
+ * a CI log never fills with redraws and a JSON reader never meets a line that is not JSON.
+ *
+ * `sharing` is every other stream that reaches the same terminal, hooked for as long as the block
+ * is live; a test hands a fake stream and an empty list, so the real process is never touched.
+ */
+export async function followed(door, watched, pending, out, asJson, sharing = out === process.stdout ? [process.stderr] : []) {
+    const knocked = Date.now();
+    const live = !asJson && isTerminal(out);
+    const total = watched.goldens * Math.max(watched.models.length, 1);
+    let settled = false;
+    const answered = pending.finally(() => (settled = true));
+    let id;
+    const bottom = new Bottom(out);
+    if (live)
+        bottom.own(sharing);
+    try {
+        while (!settled) {
+            await Promise.race([answered.catch(() => undefined), sleep(EVERY_MS)]);
+            if (settled)
+                break;
+            if (id === undefined) {
+                id = await inFlight(door, watched.agent);
+                if (id !== undefined && !asJson)
+                    out.write(`${header(watched, id)}\n`);
+            }
+            if (id === undefined || !live)
+                continue;
+            const soFar = await oneRun(door, id);
+            bottom.draw([...settledLines(soFar, watched), ...readingLines(soFar), bar(soFar, total, knocked)]);
+        }
+        bottom.erase();
+        return await settledRun(door, answered, id);
+    }
+    finally {
+        bottom.giveBack();
+    }
+}
+// A spoken suite is minutes of real calls, and the POST that started it is one request held open
+// the whole time: node's fetch gives up on a response that has not begun in five, and a run of
+// ten spoken goldens crosses that. The run itself is fine — the gateway is still driving it and
+// writing its row — so a connection that died is not a run that died. The row is the answer.
+async function settledRun(door, answered, id) {
+    try {
+        return await answered;
+    }
+    catch (lost) {
+        if (id === undefined)
+            throw lost;
+        return await untilItStops(door, id);
+    }
+}
+/** Poll one run's row until it is no longer running, and answer with it however it ended. */
+async function untilItStops(door, id) {
+    for (;;) {
+        const run = await oneRun(door, id);
+        if (run.status !== "running")
+            return run;
+        await sleep(EVERY_MS);
+    }
+}
+/** The first line, the moment the run has an id: whose suite, how big, under which model, which run. */
+export function header(watched, id) {
+    const counted = `${watched.goldens} golden${watched.goldens === 1 ? "" : "s"}`;
+    return `${watched.agent} · ${counted} · ${watched.models.join(" · ")} · ${id}`;
+}
+// The run that has the gateway right now, by the door that lists them: the runner admits one run
+// at a time and writes its row before the first call, so the newest running row is this one.
+export async function inFlight(door, agent) {
+    const newest = (await theRuns(door, 1, agent))[0];
+    return newest?.status === "running" ? newest.id : undefined;
+}
+// Every cell judged so far, drawn as the report will draw it: the report is asked for one cell at
+// a time, so a golden's live line and its final line come from the same code and cannot differ.
+// With two models the model heads its block, exactly as the report heads it.
+function settledLines(run, watched) {
+    const lines = [];
+    let heading;
+    for (const cell of run.matrix?.runs ?? []) {
+        if (watched.models.length > 1 && cell.model !== heading) {
+            heading = cell.model;
+            lines.push(`  ${cell.model === DECLARED ? watched.declaredAs : cell.model}`);
+        }
+        lines.push(...reportOf(narrowed(run, cell), {}, watched.declaredAs).slice(1, -1));
+    }
+    return lines;
+}
+// The same run with one cell in its matrix: what `reportOf` prints for it is that golden's lines
+// between the header and the footer, and nothing about any other golden.
+function narrowed(run, cell) {
+    const matrix = run.matrix;
+    const failures = matrix.failures.filter((one) => one.golden === cell.golden && one.model === cell.model);
+    return { ...run, matrix: { ...matrix, models: [cell.model], goldens: [cell.golden], runs: [cell], failures } };
+}
+// A call the row names and no cell answers for yet is the conversation being driven right now.
+function readingLines(run) {
+    const judged = run.matrix?.runs.length ?? 0;
+    return run.calls
+        .slice(judged)
+        .map((opened) => `  ${OPEN} ${opened.golden}  ${opened.call.slice(0, A_CALL_SHOWN)}…  ${READING}`);
+}
+function bar(run, total, knocked) {
+    const judged = run.matrix?.runs.length ?? 0;
+    const filled = total === 0 ? BAR_WIDTH : Math.round((judged / total) * BAR_WIDTH);
+    const seconds = Math.round((Date.now() - knocked) / 1000);
+    return `[${FILLED.repeat(filled)}${EMPTY.repeat(BAR_WIDTH - filled)}] ${judged}/${total} · ${seconds}s`;
+}
+// What a terminal that has not said its width is taken to be: a pty with no window size reports 0.
+const A_TERMINAL_WIDTH = 80;
+/**
+ * The bottom of the terminal while the run is going: the block is redrawn over itself by climbing
+ * back the rows it drew last time, so anything else writing to the same terminal in between would
+ * leave that climb counting rows the block no longer owns and the next redraw would eat the wrong
+ * lines. So every stream that reaches this terminal is owned for as long as the block is live: a
+ * write from anywhere else takes the block away first and scrolls above it, and the next half
+ * second draws the block again underneath, whole.
+ */
+class Bottom {
+    out;
+    #rows = 0;
+    #write;
+    #borrowed = new Map();
+    constructor(out) {
+        this.out = out;
+        this.#write = out.write.bind(out);
+    }
+    /** Take over `write` on this stream and every other one that reaches the same terminal. */
+    own(sharing) {
+        for (const stream of [this.out, ...sharing]) {
+            const owned = stream;
+            if (this.#borrowed.has(owned))
+                continue;
+            const raw = owned.write.bind(stream);
+            this.#borrowed.set(owned, raw);
+            owned.write = (...args) => {
+                this.erase();
+                return raw(...args);
+            };
+        }
+    }
+    /** Give every stream its own `write` back, whatever the run did. */
+    giveBack() {
+        for (const [owned, raw] of this.#borrowed)
+            owned.write = raw;
+        this.#borrowed.clear();
+    }
+    /** The block, drawn over the one before it: climb to where it began, clear everything below. */
+    draw(lines) {
+        // A line wider than the terminal takes more than one row, and the climb counts rows, not
+        // lines, or the second redraw would land in the middle of the first.
+        const columns = this.out.columns || A_TERMINAL_WIDTH;
+        const climb = this.#rows > 0 ? `\x1b[${this.#rows}A` : "";
+        this.#write(`${climb}\x1b[0J${lines.map((line) => `${line}\n`).join("")}`);
+        this.#rows = lines.reduce((rows, line) => rows + Math.max(1, Math.ceil([...line].length / columns)), 0);
+    }
+    /** Take the block away and leave the cursor where it began. A block already gone costs nothing. */
+    erase() {
+        if (this.#rows > 0)
+            this.draw([]);
+    }
+}
+function isTerminal(out) {
+    return out.isTTY === true;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=progress.js.map

package/dist/cli/testing/progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.js","sourceRoot":"","sources":["../../../src/cli/testing/progress.ts"],"names":[],"mappings":"AAAA,0GAA0G;AAE1G,OAAO,EAAE,MAAM,EAAE,OAAO,EAAsC,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEjD,+FAA+F;AAC/F,MAAM,QAAQ,GAAG,GAAG,CAAC;AAYrB,MAAM,IAAI,GAAG,GAAG,CAAC;AACjB,MAAM,OAAO,GAAG,UAAU,CAAC;AAC3B,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,MAAM,GAAG,GAAG,CAAC;AACnB,MAAM,KAAK,GAAG,GAAG,CAAC;AAClB,iGAAiG;AACjG,sEAAsE;AACtE,MAAM,YAAY,GAAG,EAAE,CAAC;AAExB;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAU,EACV,OAAgB,EAChB,OAAyB,EACzB,GAA0B,EAC1B,MAAe,EACf,OAAO,GAA4B,GAAG,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;IAEjF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnE,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IACzD,IAAI,EAAsB,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,IAAI;QAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC9B,IAAI,CAAC;QACH,OAAO,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvE,IAAI,OAAO;gBAAE,MAAM;YACnB,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;gBACrB,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;gBACzC,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,MAAM;oBAAE,GAAG,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YACzE,CAAC;YACD,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,IAAI;gBAAE,SAAS;YACxC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACrG,CAAC;QACD,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,OAAO,MAAM,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC9C,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,QAAQ,EAAE,CAAC;IACpB,CAAC;AACH,CAAC;AAED,iGAAiG;AACjG,+FAA+F;AAC/F,gGAAgG;AAChG,6FAA6F;AAC7F,KAAK,UAAU,UAAU,CAAC,IAAU,EAAE,QAA0B,EAAE,EAAsB;IACtF,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC;IACxB,CAAC;IAAC,OAAO,IAAI,EAAE,CAAC;QACd,IAAI,EAAE,KAAK,SAAS;YAAE,MAAM,IAAI,CAAC;QACjC,OAAO,MAAM,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACtC,CAAC;AACH,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,EAAU;IAChD,SAAS,CAAC;QACR,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QACzC,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED,wGAAwG;AACxG,MAAM,UAAU,MAAM,CAAC,OAAgB,EAAE,EAAU;IACjD,MAAM,OAAO,GAAG,GAAG,OAAO,CAAC,OAAO,UAAU,OAAO,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC/E,OAAO,GAAG,OAAO,CAAC,KAAK,MAAM,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;AACjF,CAAC;AAED,iGAAiG;AACjG,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAU,EAAE,KAAa;IACtD,MAAM,MAAM,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,OAAO,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9D,CAAC;AAED,kGAAkG;AAClG,gGAAgG;AAChG,6EAA6E;AAC7E,SAAS,YAAY,CAAC,GAAY,EAAE,OAAgB;IAClD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAA2B,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YACxD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;YACrB,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC/E,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,iGAAiG;AACjG,yEAAyE;AACzE,SAAS,QAAQ,CAAC,GAAY,EAAE,IAAU;IACxC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAO,CAAC;IAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC;IACzG,OAAO,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC;AACjH,CAAC;AAED,+FAA+F;AAC/F,SAAS,YAAY,CAAC,GAAY;IAChC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;IAC5C,OAAO,GAAG,CAAC,KAAK;SACb,KAAK,CAAC,MAAM,CAAC;SACb,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,OAAO,EAAE,CAAC,CAAC;AACvG,CAAC;AAED,SAAS,GAAG,CAAC,GAAY,EAAE,KAAa,EAAE,OAAe;IACvD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;IAClF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;IAC1D,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,KAAK,MAAM,OAAO,GAAG,CAAC;AAC1G,CAAC;AAED,mGAAmG;AACnG,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAU5B;;;;;;;GAOG;AACH,MAAM,MAAM;IAKmB,GAAG;IAJhC,KAAK,GAAG,CAAC,CAAC;IACD,MAAM,CAAQ;IACd,SAAS,GAAG,IAAI,GAAG,EAAiB,CAAC;IAE9C,YAA6B,GAA0B;mBAA1B,GAAG;QAC9B,IAAI,CAAC,MAAM,GAAI,GAAyB,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAU,CAAC;IACpE,CAAC;IAED,2FAA2F;IAC3F,GAAG,CAAC,OAAgC;QAClC,KAAK,MAAM,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,MAA2B,CAAC;YAC1C,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,SAAS;YACxC,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAU,CAAC;YAC9C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC/B,KAAK,CAAC,KAAK,GAAG,CAAC,GAAG,IAAe,EAAE,EAAE;gBACnC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACb,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;YACtB,CAAC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,QAAQ;QACN,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS;YAAE,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC;QAC7D,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,gGAAgG;IAChG,IAAI,CAAC,KAAe;QAClB,yFAAyF;QACzF,qEAAqE;QACrE,MAAM,OAAO,GAAI,IAAI,CAAC,GAA0B,CAAC,OAAO,IAAI,gBAAgB,CAAC;QAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1G,CAAC;IAED,mGAAmG;IACnG,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC;CACF;AAED,SAAS,UAAU,CAAC,GAA0B;IAC5C,OAAQ,GAA0B,CAAC,KAAK,KAAK,IAAI,CAAC;AACpD,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}

package/dist/cli/testing/reproduction.d.ts

@@ -0,0 +1,19 @@
+/** A golden that broke, written out whole: what the model was asked, and everything that happened. */
+import type { Entry, EvalRun } from "./gateway.js";
+import type { Golden } from "./goldens.js";
+/** Where a broken golden is left, under the directory the suite was run in. One folder per run. */
+export declare const REPRODUCTIONS = ".pinecall/evals";
+/** What stands in for the requests when nobody kept them: the reason, not an empty list. */
+export declare const NOT_RECORDED: string;
+/**
+ * One file per broken golden, and nothing at all when the suite is green. It carries the four
+ * things a person needs to disagree with a verdict without running anything: the golden as it was
+ * written, the requests the model answered — the region order, the tool list, the view, verbatim —
+ * the whole log of the call, and what each judge said. The prompt is here and nowhere else: the
+ * log keeps a hash of each block on purpose (protocol/schema/events/prompt.changed.json), so a
+ * run that is being reproduced is the one reader that asks the runtime for the text.
+ */
+export declare function writtenOut(run: EvalRun, goldens: Golden[], logs: Record<string, Entry[]>, under?: string): string[];
+/** The line the report prints under a broken golden, once, naming the folder and not each file. */
+export declare function whereTheyAre(paths: string[]): string[];
+//# sourceMappingURL=reproduction.d.ts.map

package/dist/cli/testing/reproduction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reproduction.d.ts","sourceRoot":"","sources":["../../../src/cli/testing/reproduction.ts"],"names":[],"mappings":"AAAA,sGAAsG;AAKtG,OAAO,KAAK,EAAQ,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C,mGAAmG;AACnG,eAAO,MAAM,aAAa,oBAAoB,CAAC;AAE/C,4FAA4F;AAC5F,eAAO,MAAM,YAAY,QAEyD,CAAC;AAEnF;;;;;;;GAOG;AACH,wBAAgB,UAAU,CACxB,GAAG,EAAE,OAAO,EACZ,OAAO,EAAE,MAAM,EAAE,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAC7B,KAAK,GAAE,MAAsB,GAC5B,MAAM,EAAE,CAWV;AAiCD,mGAAmG;AACnG,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAItD"}

package/dist/cli/testing/reproduction.js

@@ -0,0 +1,65 @@
+/** A golden that broke, written out whole: what the model was asked, and everything that happened. */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+/** Where a broken golden is left, under the directory the suite was run in. One folder per run. */
+export const REPRODUCTIONS = ".pinecall/evals";
+/** What stands in for the requests when nobody kept them: the reason, not an empty list. */
+export const NOT_RECORDED = "not recorded: this call's requests were built in the worker process, which a spoken run " +
+    "drives over LiveKit. The log below is the whole of what this call left behind.";
+/**
+ * One file per broken golden, and nothing at all when the suite is green. It carries the four
+ * things a person needs to disagree with a verdict without running anything: the golden as it was
+ * written, the requests the model answered — the region order, the tool list, the view, verbatim —
+ * the whole log of the call, and what each judge said. The prompt is here and nowhere else: the
+ * log keeps a hash of each block on purpose (protocol/schema/events/prompt.changed.json), so a
+ * run that is being reproduced is the one reader that asks the runtime for the text.
+ */
+export function writtenOut(run, goldens, logs, under = REPRODUCTIONS) {
+    const broken = (run.matrix?.runs ?? []).filter((cell) => cell.scores.some((score) => !score.passed));
+    if (broken.length === 0)
+        return [];
+    const folder = join(under, run.id);
+    mkdirSync(folder, { recursive: true });
+    return broken.map((cell) => {
+        const call = callOf(run, cell);
+        const path = join(folder, `${cell.golden}.json`);
+        writeFileSync(path, `${JSON.stringify(aReproduction(run, cell, call, goldens, logs[call] ?? []), null, 2)}\n`);
+        return path;
+    });
+}
+/** One broken cell as a file: the declaration, the requests, the log, and every verdict on it. */
+function aReproduction(run, cell, call, goldens, log) {
+    return {
+        run: run.id,
+        agent: run.agent,
+        golden: cell.golden,
+        model: cell.model,
+        call,
+        // The golden as it was written: the state it seeded, what the caller said, what it expected.
+        declared: goldens.find((one) => one.name === cell.golden) ?? null,
+        verdicts: cell.scores.map((score) => ({
+            metric: score.metric,
+            passed: score.passed,
+            criteria: score.criteria,
+            reason: score.reason,
+        })),
+        // In the order they went out: one per request, so a turn that ran a tool has more than one.
+        // The runtime says null when the run kept none — a spoken run builds its requests in the
+        // worker process — and here that is the sentence saying so, never an empty list: `tools: []`
+        // under a call whose finding was "ran no tool at all" misled this project once already.
+        asked: cell.asked ?? NOT_RECORDED,
+        log,
+    };
+}
+/** The line the report prints under a broken golden, once, naming the folder and not each file. */
+export function whereTheyAre(paths) {
+    if (paths.length === 0)
+        return [];
+    const folder = dirname(paths[0]);
+    return [`  ${paths.length} reproduction${paths.length === 1 ? "" : "s"} written to ${folder}/`];
+}
+/** Which call a cell was answered by: the run names it when it opens, before the first turn. */
+function callOf(run, cell) {
+    return run.calls.find((one) => one.golden === cell.golden && one.model === cell.model)?.call ?? "";
+}
+//# sourceMappingURL=reproduction.js.map

package/dist/cli/testing/reproduction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reproduction.js","sourceRoot":"","sources":["../../../src/cli/testing/reproduction.ts"],"names":[],"mappings":"AAAA,sGAAsG;AAEtG,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAK1C,mGAAmG;AACnG,MAAM,CAAC,MAAM,aAAa,GAAG,iBAAiB,CAAC;AAE/C,4FAA4F;AAC5F,MAAM,CAAC,MAAM,YAAY,GACvB,0FAA0F;IAC1F,gFAAgF,CAAC;AAEnF;;;;;;;GAOG;AACH,MAAM,UAAU,UAAU,CACxB,GAAY,EACZ,OAAiB,EACjB,IAA6B,EAC7B,KAAK,GAAW,aAAa;IAE7B,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACrG,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACnC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,OAAO,CAAC,CAAC;QACjD,aAAa,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/G,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,kGAAkG;AAClG,SAAS,aAAa,CACpB,GAAY,EACZ,IAAU,EACV,IAAY,EACZ,OAAiB,EACjB,GAAY;IAEZ,OAAO;QACL,GAAG,EAAE,GAAG,CAAC,EAAE;QACX,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI;QACJ,6FAA6F;QAC7F,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI;QACjE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACpC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CAAC;QACH,4FAA4F;QAC5F,yFAAyF;QACzF,6FAA6F;QAC7F,wFAAwF;QACxF,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,YAAY;QACjC,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,YAAY,CAAC,KAAe;IAC1C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;IAClC,OAAO,CAAC,KAAK,KAAK,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe,MAAM,GAAG,CAAC,CAAC;AAClG,CAAC;AAED,gGAAgG;AAChG,SAAS,MAAM,CAAC,GAAY,EAAE,IAAU;IACtC,OAAO,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;AACrG,CAAC"}

package/dist/cli/testing/score.d.ts

@@ -0,0 +1,11 @@
+/** The `call.score` entry as a person reads it: what each judge said, and what the asking cost. */
+import type { CallScore } from "@pinecall/protocol";
+export declare const HELD = "\u2713";
+export declare const BROKEN = "\u2717";
+/** The sentence a judge or a graph wrote when it decided. A hard policy writes the seqs for free. */
+export declare function reasonOf(said: {
+    reason: string;
+}): string;
+/** The score as the terminal prints it: the headline, a line per judge, then what it cost. */
+export declare function linesOfScore(score: CallScore): string[];
+//# sourceMappingURL=score.d.ts.map

package/dist/cli/testing/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../../src/cli/testing/score.ts"],"names":[],"mappings":"AAAA,mGAAmG;AAEnG,OAAO,KAAK,EAAE,SAAS,EAAY,MAAM,oBAAoB,CAAC;AAI9D,eAAO,MAAM,IAAI,WAAM,CAAC;AACxB,eAAO,MAAM,MAAM,WAAM,CAAC;AAoB1B,qGAAqG;AACrG,wBAAgB,QAAQ,CAAC,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAEzD;AAED,8FAA8F;AAC9F,wBAAgB,YAAY,CAAC,KAAK,EAAE,SAAS,GAAG,MAAM,EAAE,CAEvD"}

package/dist/cli/testing/score.js

@@ -0,0 +1,54 @@
+/** The `call.score` entry as a person reads it: what each judge said, and what the asking cost. */
+// The mark a verdict wears. `held` and `broken` are the two the matrix draws too, so they are
+// defined here once and imported there: one glyph, one meaning, wherever a verdict is printed.
+export const HELD = "✓";
+export const BROKEN = "✗";
+const DEFERRED = "?";
+const SKIPPED = "·";
+// livekit's three verdicts said in ours, plus the fourth that is ours alone. See scoring.md.
+const MARK = {
+    held: HELD,
+    broken: BROKEN,
+    deferred: DEFERRED,
+    skipped: SKIPPED,
+};
+// `passed` means no judge answered broken, which is the answer to a question somebody asked. A
+// call nobody asked about has NO answer: the field is absent, and absent is a third thing that
+// must read as neither green nor red. The runtime's docs/decisions/scoring.md says so, and this
+// line is that.
+const NOBODY_JUDGED = "nobody judged this call";
+const NO_REASON = "no reason was written down";
+/** The sentence a judge or a graph wrote when it decided. A hard policy writes the seqs for free. */
+export function reasonOf(said) {
+    return said.reason === "" ? NO_REASON : said.reason;
+}
+/** The score as the terminal prints it: the headline, a line per judge, then what it cost. */
+export function linesOfScore(score) {
+    return [headline(score), ...score.judges.map(oneJudge), costLine(score)];
+}
+// The one line read first. Three states and never two: held, broken, or nobody looked — and when
+// nobody looked, the reason the entry carries is printed rather than a verdict invented for it.
+function headline(score) {
+    if (score.passed === undefined || score.passed === null) {
+        return `${SKIPPED} ${NOBODY_JUDGED}: ${score.not_judged ?? "the entry says nothing about why"}`;
+    }
+    return `${score.passed ? HELD : BROKEN} ${score.passed ? "every judge held" : "a judge answered broken"}`;
+}
+// One judge: its mark, its name, the sentence it wrote, and the seqs that sentence named — which
+// are what a person opens the log at to disagree with it.
+function oneJudge(judgment) {
+    const seqs = judgment.evidence.seqs;
+    const at = seqs.length === 0 ? "" : `  [seq ${seqs.join(", ")}]`;
+    return `  ${MARK[judgment.verdict] ?? SKIPPED} ${judgment.name.padEnd(10)} ${reasonOf(judgment)}${at}`;
+}
+// What judging cost: the count of questions that actually reached a model, and the bill when one
+// could be priced. An unknown bill is left unsaid rather than printed as a zero — a model nobody
+// reported tokens for has no price, not a free one.
+function costLine(score) {
+    const asked = score.judge_calls;
+    const priced = score.judge_cost_eur === undefined || score.judge_cost_eur === null
+        ? ""
+        : ` · ${score.judge_cost_eur.toFixed(4)} EUR`;
+    return `  ${asked} judge call${asked === 1 ? "" : "s"}${priced}`;
+}
+//# sourceMappingURL=score.js.map

package/dist/cli/testing/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../../../src/cli/testing/score.ts"],"names":[],"mappings":"AAAA,mGAAmG;AAInG,8FAA8F;AAC9F,+FAA+F;AAC/F,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC;AACxB,MAAM,CAAC,MAAM,MAAM,GAAG,GAAG,CAAC;AAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC;AACrB,MAAM,OAAO,GAAG,GAAG,CAAC;AAEpB,6FAA6F;AAC7F,MAAM,IAAI,GAA2B;IACnC,IAAI,EAAE,IAAI;IACV,MAAM,EAAE,MAAM;IACd,QAAQ,EAAE,QAAQ;IAClB,OAAO,EAAE,OAAO;CACjB,CAAC;AAEF,+FAA+F;AAC/F,+FAA+F;AAC/F,gGAAgG;AAChG,gBAAgB;AAChB,MAAM,aAAa,GAAG,yBAAyB,CAAC;AAEhD,MAAM,SAAS,GAAG,4BAA4B,CAAC;AAE/C,qGAAqG;AACrG,MAAM,UAAU,QAAQ,CAAC,IAAwB;IAC/C,OAAO,IAAI,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;AACtD,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,YAAY,CAAC,KAAgB;IAC3C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,iGAAiG;AACjG,gGAAgG;AAChG,SAAS,QAAQ,CAAC,KAAgB;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QACxD,OAAO,GAAG,OAAO,IAAI,aAAa,KAAK,KAAK,CAAC,UAAU,IAAI,kCAAkC,EAAE,CAAC;IAClG,CAAC;IACD,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,yBAAyB,EAAE,CAAC;AAC5G,CAAC;AAED,iGAAiG;AACjG,0DAA0D;AAC1D,SAAS,QAAQ,CAAC,QAAkB;IAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;IACpC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IACjE,OAAO,KAAK,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC;AACzG,CAAC;AAED,iGAAiG;AACjG,iGAAiG;AACjG,oDAAoD;AACpD,SAAS,QAAQ,CAAC,KAAgB;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,CAAC;IAChC,MAAM,MAAM,GACV,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,IAAI;QACjE,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,MAAM,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IAClD,OAAO,KAAK,KAAK,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC;AACnE,CAAC"}

package/dist/cli/testing/seeding.d.ts

@@ -0,0 +1,32 @@
+/** Which golden's state a starting call opens in, and the check that it was the right one. */
+import type { Call as SdkCall } from "../../client/index.js";
+import type { Snapshot } from "../../agent/state.js";
+import type { EvalRun } from "./gateway.js";
+import type { Golden } from "./goldens.js";
+/** What to say when the calls did not arrive in the order the run says it opened them in. */
+export declare const OUT_OF_ORDER = "the calls did not arrive in the order the run opened them: {said} took the state of {seeded}";
+/**
+ * The states a run is about to open, handed out as its calls start.
+ *
+ * The seed cannot come off the log: `session.configure {state}` writes `state.changed` into the
+ * call and stops there — the tenant's class is another process and nothing on the wire moves it —
+ * and by the time that entry reaches this terminal the bridge is re-rendering on every field, so
+ * restoring three fields would show the model two states the call was never in. `mount`'s own
+ * `opening` seam is the moment between `onCall` and the first render, and this is what fills it.
+ *
+ * Which state belongs to which call is the runner's order: one run at a time (a second is refused
+ * with 409) and one conversation at a time inside it, models outermost. `mismatched` checks that
+ * afterwards against what the run says it opened, so a wrong seed is a loud line and never a
+ * golden that quietly tested something else.
+ */
+export declare class Openings {
+    private waiting;
+    private readonly seeded;
+    /** The goldens this run will open, once per model, in the order the runner will walk them. */
+    expects(goldens: Golden[], models: number): void;
+    /** The state this starting call opens in: the next golden's, or none when no run opened it. */
+    opening(call: SdkCall): Snapshot | undefined;
+    /** The sentence to print when a call took a state that was not its golden's, or nothing. */
+    mismatched(run: EvalRun): string | undefined;
+}
+//# sourceMappingURL=seeding.d.ts.map

package/dist/cli/testing/seeding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"seeding.d.ts","sourceRoot":"","sources":["../../../src/cli/testing/seeding.ts"],"names":[],"mappings":"AAAA,8FAA8F;AAE9F,OAAO,KAAK,EAAE,IAAI,IAAI,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C,6FAA6F;AAC7F,eAAO,MAAM,YAAY,iGACuE,CAAC;AAEjG;;;;;;;;;;;;;GAaG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IAEvC,8FAA8F;IAC9F,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAG/C;IAED,+FAA+F;IAC/F,OAAO,CAAC,IAAI,EAAE,OAAO,GAAG,QAAQ,GAAG,SAAS,CAM3C;IAED,4FAA4F;IAC5F,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAO3C;CACF"}