pinecall 0.1.0

package/dist/cli/test.js

@@ -0,0 +1,128 @@
+/** `pinecall test [paths]`: the goldens, through the app this terminal is holding, scored by the runtime. */
+import { existsSync, watch } from "node:fs";
+import { parseArgs } from "node:util";
+import { Pinecall } from "../client/index.js";
+import { modelOf } from "../runtime/connect.js";
+import { theDoor } from "./env.js";
+import { load } from "./load.js";
+import { GOLDENS, goldensIn, matching, NO_GOLDENS } from "./testing/goldens.js";
+import { mountedForASuite, ranSuite } from "./testing/suite.js";
+const USAGE = "usage: pinecall test [paths] [--file agent.tsx] [--model m]… [--grep x] [--watch] [--json]\n" +
+    "       pinecall test --voice [--background-noise dB] [--packet-loss 0.05]\n";
+export const group = {
+    purpose: "the goldens, run through the app in this terminal's own process",
+    usage: `${USAGE}
+  Ring 1: every golden of test/goldens through the class this terminal holds, scored by the
+  gateway's judges, printed as a matrix. Exits 1 when a golden did not hold, and writes every
+  broken one to .pinecall/evals/<run>/ with the requests the model answered.
+
+  [paths]             files or directories of goldens; the whole of test/goldens when none
+  --file agent.tsx    which class to mount, when the directory holds more than one
+  --model m           a column of the matrix: vendor/model, repeatable
+  --grep x            only the goldens whose name matches
+  --watch             run again whenever a file changes
+  --json              the run as one JSON document instead of the matrix
+  --voice             ring 2: the same goldens said out loud on a real line
+  --background-noise  dB under the caller, on a spoken run: a television behind them
+  --packet-loss       the share of the caller's packets that never arrive, 0 to 1`,
+    run,
+};
+// Ring 2: the same goldens, said out loud. Only the three fields a spoken run adds travel — a
+// written run must not carry a `voice: false` that reads as a decision somebody made.
+function aLine(values) {
+    if (values.voice !== true)
+        return {};
+    const noise = numberOf(values["background-noise"]);
+    const loss = numberOf(values["packet-loss"]);
+    return {
+        voice: true,
+        ...(noise === undefined ? {} : { interferer_db: noise }),
+        ...(loss === undefined ? {} : { packet_loss: loss }),
+    };
+}
+/** A flag that must be a number to mean anything: anything else is left out rather than sent as NaN. */
+function numberOf(said) {
+    if (said === undefined)
+        return undefined;
+    const value = Number(said);
+    return Number.isFinite(value) ? value : undefined;
+}
+// How long a change waits before the suite runs again: two saves of the same file in an editor
+// are one change to a person, and a run costs real calls.
+const SETTLE_MS = 150;
+/**
+ * The class is mounted HERE, in this process, exactly as `pinecall chat` mounts it: the tenant's
+ * @tool bodies run against the tenant's own database and a breakpoint in one is reachable. The
+ * gateway drives the conversations and scores them, because the judge, the keys and the log are
+ * its — see docs/decisions/pinecall-test.md. The suite itself is `testing/suite.ts`, which the
+ * console runs too.
+ */
+export async function run(argv, out = process.stdout) {
+    const { values, positionals } = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            file: { type: "string" },
+            model: { type: "string", multiple: true },
+            grep: { type: "string" },
+            watch: { type: "boolean", default: false },
+            json: { type: "boolean", default: false },
+            voice: { type: "boolean", default: false },
+            "background-noise": { type: "string" },
+            "packet-loss": { type: "string" },
+        },
+    });
+    const door = theDoor();
+    if (door === undefined)
+        return 2;
+    if (positionals.length === 0 && !existsSync(GOLDENS)) {
+        process.stderr.write(`${NO_GOLDENS}\n${USAGE}`);
+        return 2;
+    }
+    const loaded = await load(values.file);
+    const pc = new Pinecall({ url: door.url, apiKey: door.apiKey });
+    const held = mountedForASuite(loaded, pc);
+    const models = (values.model ?? []).map(modelOf).filter((model) => model !== undefined);
+    try {
+        await pc.connect();
+        const suite = async () => {
+            const goldens = matching(await goldensIn(positionals), values.grep);
+            if (goldens.length === 0) {
+                process.stderr.write(`no golden matched${values.grep === undefined ? "" : ` --grep ${values.grep}`}\n`);
+                return 2;
+            }
+            return await ranSuite({ door, loaded, held, goldens, models, line: aLine(values), out, json: values.json === true });
+        };
+        const code = await suite();
+        if (values.watch !== true)
+            return code;
+        return await watching(positionals, suite, out);
+    }
+    finally {
+        pc.close();
+    }
+}
+// A change to a golden re-runs the suite; a change to the class does not, because the class is
+// already imported into this process and node will not import it twice. The line says so, so that
+// nobody watches a stale agent answer a fresh golden.
+async function watching(paths, suite, out) {
+    const watched = paths.length > 0 ? paths : [GOLDENS];
+    out.write(`watching ${watched.join(" ")} — restart to pick up a change to the class\n`);
+    let running = false;
+    let pending;
+    for (const path of watched) {
+        watch(path, { recursive: true }, () => {
+            if (pending !== undefined)
+                clearTimeout(pending);
+            pending = setTimeout(() => {
+                if (running)
+                    return;
+                running = true;
+                void suite().finally(() => (running = false));
+            }, SETTLE_MS);
+        });
+    }
+    // The watchers hold the process; a person ends it with the same ctrl-C that ends `run`.
+    return await new Promise(() => { });
+}
+//# sourceMappingURL=test.js.map

package/dist/cli/test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.js","sourceRoot":"","sources":["../../src/cli/test.ts"],"names":[],"mappings":"AAAA,6GAA6G;AAE7G,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAEnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEhE,MAAM,KAAK,GACT,8FAA8F;IAC9F,6EAA6E,CAAC;AAEhF,MAAM,CAAC,MAAM,KAAK,GAAU;IAC1B,OAAO,EAAE,iEAAiE;IAC1E,KAAK,EAAE,GAAG,KAAK;;;;;;;;;;;;;kFAaiE;IAChF,GAAG;CACJ,CAAC;AAEF,8FAA8F;AAC9F,sFAAsF;AACtF,SAAS,KAAK,CAAC,MAAgF;IAC7F,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;IAC7C,OAAO;QACL,KAAK,EAAE,IAAI;QACX,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACxD,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC;AAED,wGAAwG;AACxG,SAAS,QAAQ,CAAC,IAAwB;IACxC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACzC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACpD,CAAC;AAED,+FAA+F;AAC/F,0DAA0D;AAC1D,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,IAAc,EAAE,GAAG,GAA0B,OAAO,CAAC,MAAM;IACnF,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;QACxC,IAAI,EAAE,IAAI;QACV,gBAAgB,EAAE,IAAI;QACtB,OAAO,EAAE;YACP,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;YACzC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC1C,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzC,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC1C,kBAAkB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACtC,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAClC;KACF,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;IACvB,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IACjC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,UAAU,KAAK,KAAK,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAExF,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,KAAK,IAAqB,EAAE;YACxC,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,SAAS,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACxG,OAAO,CAAC,CAAC;YACX,CAAC;YACD,OAAO,MAAM,QAAQ,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC;QACvH,CAAC,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,KAAK,EAAE,CAAC;QAC3B,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACvC,OAAO,MAAM,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC;AAED,+FAA+F;AAC/F,kGAAkG;AAClG,sDAAsD;AACtD,KAAK,UAAU,QAAQ,CACrB,KAAe,EACf,KAA4B,EAC5B,GAA0B;IAE1B,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACrD,GAAG,CAAC,KAAK,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IACxF,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,OAAmC,CAAC;IACxC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE;YACpC,IAAI,OAAO,KAAK,SAAS;gBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;YACjD,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACxB,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,KAAK,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC;YAChD,CAAC,EAAE,SAAS,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC;IACD,wFAAwF;IACxF,OAAO,MAAM,IAAI,OAAO,CAAS,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AAC7C,CAAC"}

package/dist/cli/testing/caller.d.ts

@@ -0,0 +1,28 @@
+/** A synthetic caller: who is on the phone, what they want, and the words they use to get it. */
+/**
+ * One persona, written by the tenant in `test/personas/<name>.ts` and default-exported.
+ *
+ * A persona is not a script: `goal` and `style` are what the model playing the caller is told it
+ * is, and `facts` is everything that caller knows about themselves and may state. The model
+ * improvises every turn from those three — see docs/decisions/simulate.md.
+ */
+export interface Persona {
+    name: string;
+    /** What this caller is after, in one line. */
+    goal: string;
+    /** How they talk: hurried, switching into English mid-sentence, hard of hearing. */
+    style: string;
+    /** Their name, their phone, the appointment they are calling about: what they may never invent. */
+    facts?: Record<string, unknown>;
+    /** The state the call starts in, when this caller is somebody the business already knows. */
+    state?: Record<string, unknown>;
+}
+/** Where the tenant's personas live: one file per caller, beside the goldens they are run with. */
+export declare const PERSONAS = "test/personas";
+/** What to say when there are none: the directory, and that a persona is one file in it. */
+export declare const NO_PERSONAS = "no personas at test/personas: one file per caller, default-exporting one";
+/** Every persona in that directory, in the order `ls` prints them. */
+export declare function personasIn(folder?: string): Promise<Persona[]>;
+/** One persona by the name of its file, or undefined when nobody wrote that caller. */
+export declare function personaNamed(wanted: string, folder?: string): Promise<Persona | undefined>;
+//# sourceMappingURL=caller.d.ts.map

package/dist/cli/testing/caller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"caller.d.ts","sourceRoot":"","sources":["../../../src/cli/testing/caller.ts"],"names":[],"mappings":"AAAA,iGAAiG;AAQjG;;;;;;GAMG;AACH,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,oFAAoF;IACpF,KAAK,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,6FAA6F;IAC7F,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,mGAAmG;AACnG,eAAO,MAAM,QAAQ,kBAAkB,CAAC;AAExC,4FAA4F;AAC5F,eAAO,MAAM,WAAW,6EAA2E,CAAC;AAEpG,sEAAsE;AACtE,wBAAsB,UAAU,CAAC,MAAM,GAAE,MAAiB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAI9E;AAED,uFAAuF;AACvF,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,MAAiB,GACxB,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC,CAE9B"}

package/dist/cli/testing/caller.js

@@ -0,0 +1,30 @@
+/** A synthetic caller: who is on the phone, what they want, and the words they use to get it. */
+import { readdir } from "node:fs/promises";
+import { basename, extname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { useTypeScript } from "../load.js";
+/** Where the tenant's personas live: one file per caller, beside the goldens they are run with. */
+export const PERSONAS = "test/personas";
+/** What to say when there are none: the directory, and that a persona is one file in it. */
+export const NO_PERSONAS = `no personas at ${PERSONAS}: one file per caller, default-exporting one`;
+/** Every persona in that directory, in the order `ls` prints them. */
+export async function personasIn(folder = PERSONAS) {
+    const names = await readdir(resolve(folder)).catch(() => []);
+    const files = names.filter((name) => extname(name) === ".ts").sort();
+    return await Promise.all(files.map((name) => personaOf(join(resolve(folder), name))));
+}
+/** One persona by the name of its file, or undefined when nobody wrote that caller. */
+export async function personaNamed(wanted, folder = PERSONAS) {
+    return (await personasIn(folder)).find((persona) => persona.name === wanted);
+}
+/** One file's persona. The file's own name is the caller's, so a directory listing is the roster. */
+async function personaOf(file) {
+    await useTypeScript();
+    const module_ = (await import(pathToFileURL(file).href));
+    const written = module_.default;
+    if (typeof written !== "object" || written === null) {
+        throw new Error(`${file} has no default-exported persona`);
+    }
+    return { ...written, name: basename(file, extname(file)) };
+}
+//# sourceMappingURL=caller.js.map

package/dist/cli/testing/caller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"caller.js","sourceRoot":"","sources":["../../../src/cli/testing/caller.ts"],"names":[],"mappings":"AAAA,iGAAiG;AAEjG,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAqB3C,mGAAmG;AACnG,MAAM,CAAC,MAAM,QAAQ,GAAG,eAAe,CAAC;AAExC,4FAA4F;AAC5F,MAAM,CAAC,MAAM,WAAW,GAAG,kBAAkB,QAAQ,8CAA8C,CAAC;AAEpG,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,MAAM,GAAW,QAAQ;IACxD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACrE,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACxF,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAc,EACd,MAAM,GAAW,QAAQ;IAEzB,OAAO,CAAC,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;AAC/E,CAAC;AAED,qGAAqG;AACrG,KAAK,UAAU,SAAS,CAAC,IAAY;IACnC,MAAM,aAAa,EAAE,CAAC;IACtB,MAAM,OAAO,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAA0B,CAAC;IAClF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,kCAAkC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,EAAE,GAAI,OAAmB,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;AAC1E,CAAC"}

package/dist/cli/testing/gateway.d.ts

@@ -0,0 +1,172 @@
+/** The gateway's eval doors, as this CLI knocks at them: run a suite, read a run, read a log. */
+import type { Camel, ModelConfig, SessionLine } from "@pinecall/protocol";
+import type { Golden } from "./goldens.js";
+/** What one judge answered about one golden: the number, the sentence, and what it asked. */
+export interface Score {
+    metric: string;
+    score: number;
+    passed: boolean;
+    /** The judgment's own reasoning. A hard policy writes the seqs for free; a model is asked. */
+    reason: string;
+    /** The question that was answered, as livekit hangs it on every judgment it makes. */
+    criteria: string;
+    /** How many questions this judgment put to a model. Zero for a policy that answers by code. */
+    judge_calls: number;
+}
+/** One request as the provider received it: the system blocks apart, then the messages in order. */
+export interface Asked {
+    /** One string per static block — identity, knowledge, tools — for the vendor that keeps them
+     * apart. Empty for a vendor whose system text travels inside `messages`. */
+    system: string[];
+    /** Every tool the request carried, as a JSON schema — the list the model actually had. */
+    tools: Record<string, unknown>[];
+    messages: Record<string, unknown>[];
+}
+/** One cell of the matrix: a golden under a model, every judge's answer, and the call's summary. */
+export interface Cell {
+    model: string;
+    golden: string;
+    scores: Score[];
+    summary: Summary | null;
+    /** Every request this golden's call made, present only on a cell where something broke: the
+     * prompt is not in the log, and a broken golden is the one thing a person has to reproduce.
+     * Null when the run kept no requests — a spoken run builds them in the worker process. */
+    asked?: Asked[] | null;
+}
+/** `call.summary` as the log wrote it. Nothing here is recomputed by anybody who reads it. */
+export interface Summary {
+    duration_s: number;
+    turns: number;
+    cost?: {
+        eur: number;
+    };
+}
+/** Models down one axis, goldens down the other, one judge's score in every cell. */
+export interface Matrix {
+    models: string[];
+    goldens: string[];
+    metrics: string[];
+    /** How many questions judging this whole matrix put to a model. Zero is the happy path. */
+    judge_calls: number;
+    runs: Cell[];
+    failures: {
+        model: string;
+        golden: string;
+        metric: string;
+    }[];
+}
+/** One golden under one model, and the call it opened: the log a person opens to disagree. */
+export interface Opened {
+    golden: string;
+    model: string;
+    call: string;
+}
+/** One run of a suite, as the runner stores it and both `test` and `runs` read it back. */
+export interface EvalRun {
+    id: string;
+    agent: string;
+    started_at: number;
+    finished_at: number | null;
+    status: "running" | "done" | "failed";
+    calls: Opened[];
+    matrix: Matrix | null;
+    error: string | null;
+}
+/** What one POST asks for: the agent, its goldens, the models, and which app socket to use. */
+export interface Wanted {
+    agent: string;
+    goldens: Golden[];
+    models?: Camel<ModelConfig>[];
+    app?: string;
+    /** Ring 2: the same goldens said out loud on a real line, instead of written into a text session. */
+    voice?: boolean;
+    /** How many dB under the caller's own voice an interferer sits, on a spoken run. */
+    interferer_db?: number;
+    /** The share of the caller's packets that never arrive, 0 to 1, on a spoken run. */
+    packet_loss?: number;
+}
+/** One entry of a call's log, as `GET /v1/calls/{call}/events` projects it for this reader. */
+export interface Entry {
+    seq: number;
+    type: string;
+    data: Record<string, unknown>;
+    /** Which call it belongs to. The chat socket's own frames carry it; a page of one call need not. */
+    call?: string | null;
+}
+/** Where this CLI is knocking: the gateway's HTTP address and the key that opens its doors. */
+export interface Door {
+    url: string;
+    apiKey: string;
+}
+/** Every golden through the app that is holding the agent, scored and stored, in one round trip. */
+export declare function aRun(door: Door, wanted: Wanted): Promise<EvalRun>;
+/** The runs this gateway has done, newest first — of one agent, or of the whole fleet. */
+export declare function theRuns(door: Door, limit: number, agent?: string): Promise<EvalRun[]>;
+/** One run, whole: the calls it opened while it happened, and the matrix once it finished. */
+export declare function oneRun(door: Door, id: string): Promise<EvalRun>;
+/** Who is on the phone, as the door that plays them needs it: no script, three declarations. */
+export interface Persona {
+    name: string;
+    goal: string;
+    style: string;
+    facts?: Record<string, unknown>;
+}
+/** One turn of a simulated call from the caller's side: the business, or the caller themselves. */
+export interface Spoken {
+    who: "agent" | "caller";
+    said: string;
+}
+/** What the model playing the caller answered: the line, and whether they are done talking. */
+export interface Improvised {
+    say: string;
+    hangup: boolean;
+}
+/**
+ * The next thing the caller says, improvised by a model in the runtime. It runs there because
+ * that is the process holding the provider keys — this terminal has the persona and the call so
+ * far, and the door has the only thing it is missing. See docs/decisions/simulate.md.
+ */
+export declare function theNextLine(door: Door, asking: {
+    persona: Persona;
+    heard: Spoken[];
+    turns_left: number;
+}): Promise<Improvised>;
+/** What a reader narrows a log to: how many entries, from which seq, and of which types. */
+export interface Narrowed {
+    limit?: number;
+    /** Entries above this seq only. The door calls it `after`, and it is a seq, never an index. */
+    after?: number;
+    /** `types=a.b,c.d`: the log's own filter, so a reader that wants one entry is served one. */
+    types?: string[];
+}
+/** One page of a call's log as this reader narrowed it: where the seqs a failure names are read. */
+export declare function entriesOf(door: Door, call: string, narrowed?: Narrowed): Promise<Entry[]>;
+/** Which calls this agent handled, newest first: the list drift reads its two windows out of. */
+export declare function theSessions(door: Door, agent: string, limit: number): Promise<SessionLine[]>;
+/** What the gateway answered when it did not answer 2xx: the status, and its own sentence. */
+/** What a gateway says about itself with no key: which runtime, and whether it opens a sign-up. */
+export interface Discovered {
+    version: string;
+    cloud: boolean;
+    signup: boolean;
+    /** How short a password this gateway accepts; 0 is no rule. A card asking for one says it. */
+    min_password: number;
+}
+/**
+ * `GET /.well-known/pinecall`, the one door that takes no key and asks nothing of the caller.
+ *
+ * A gateway too old to answer it, or one that does not answer at all, reads as a gateway that
+ * opens no sign-up: guessing that way offers nothing, and the other way would walk a person
+ * through typing a password for a door that refuses.
+ */
+export declare function discovered(url: string): Promise<Discovered>;
+export declare class Refused extends Error {
+    readonly status: number;
+    readonly text: string;
+    constructor(status: number, text: string);
+}
+export declare function asked<T>(door: Door, path: string, sent?: {
+    method?: string;
+    body?: unknown;
+}): Promise<T>;
+//# sourceMappingURL=gateway.d.ts.map

package/dist/cli/testing/gateway.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gateway.d.ts","sourceRoot":"","sources":["../../../src/cli/testing/gateway.ts"],"names":[],"mappings":"AAAA,iGAAiG;AAEjG,OAAO,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE1E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C,6FAA6F;AAC7F,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAC;IACf,sFAAsF;IACtF,QAAQ,EAAE,MAAM,CAAC;IACjB,+FAA+F;IAC/F,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,oGAAoG;AACpG,MAAM,WAAW,KAAK;IACpB;gFAC4E;IAC5E,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,0FAA0F;IAC1F,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IACjC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;CACrC;AAED,oGAAoG;AACpG,MAAM,WAAW,IAAI;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB;;8FAE0F;IAC1F,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;CACxB;AAED,8FAA8F;AAC9F,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;CACxB;AAED,qFAAqF;AACrF,MAAM,WAAW,MAAM;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,2FAA2F;IAC3F,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,IAAI,EAAE,CAAC;IACb,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC/D;AAED,8FAA8F;AAC9F,MAAM,WAAW,MAAM;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,2FAA2F;AAC3F,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,+FAA+F;AAC/F,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;IAC9B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,qGAAqG;IACrG,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,oFAAoF;IACpF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oFAAoF;IACpF,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+FAA+F;AAC/F,MAAM,WAAW,KAAK;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,oGAAoG;IACpG,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,+FAA+F;AAC/F,MAAM,WAAW,IAAI;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oGAAoG;AACpG,wBAAsB,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAEvE;AAED,0FAA0F;AAC1F,wBAAsB,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAK3F;AAED,8FAA8F;AAC9F,wBAAsB,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAErE;AAED,gGAAgG;AAChG,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,mGAAmG;AACnG,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,OAAO,GAAG,QAAQ,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,+FAA+F;AAC/F,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,OAAO,CAAC;CACjB;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,IAAI,EACV,MAAM,EAAE;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAChE,OAAO,CAAC,UAAU,CAAC,CAErB;AAED,4FAA4F;AAC5F,MAAM,WAAW,QAAQ;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6FAA6F;IAC7F,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,oGAAoG;AACpG,wBAAsB,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAE,QAAa,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAanG;AAED,iGAAiG;AACjG,wBAAsB,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAIlG;AAED,8FAA8F;AAC9F,mGAAmG;AACnG,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,8FAA8F;IAC9F,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAejE;AAED,qBAAa,OAAQ,SAAQ,KAAK;IAE9B,QAAQ,CAAC,MAAM,EAAE,MAAM;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM;IAFvB,YACW,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EAGtB;CACF;AAKD,wBAAsB,KAAK,CAAC,CAAC,EAC3B,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,MAAM,EACZ,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CAAO,GAC7C,OAAO,CAAC,CAAC,CAAC,CAYZ"}

package/dist/cli/testing/gateway.js

@@ -0,0 +1,96 @@
+/** The gateway's eval doors, as this CLI knocks at them: run a suite, read a run, read a log. */
+/** Every golden through the app that is holding the agent, scored and stored, in one round trip. */
+export async function aRun(door, wanted) {
+    return await asked(door, "/v1/evals/run", { method: "POST", body: wanted });
+}
+/** The runs this gateway has done, newest first — of one agent, or of the whole fleet. */
+export async function theRuns(door, limit, agent) {
+    const query = new URLSearchParams({ limit: String(limit) });
+    if (agent !== undefined)
+        query.set("agent", agent);
+    const answered = await asked(door, `/v1/evals/runs?${query.toString()}`);
+    return answered.runs;
+}
+/** One run, whole: the calls it opened while it happened, and the matrix once it finished. */
+export async function oneRun(door, id) {
+    return await asked(door, `/v1/evals/runs/${encodeURIComponent(id)}`);
+}
+/**
+ * The next thing the caller says, improvised by a model in the runtime. It runs there because
+ * that is the process holding the provider keys — this terminal has the persona and the call so
+ * far, and the door has the only thing it is missing. See docs/decisions/simulate.md.
+ */
+export async function theNextLine(door, asking) {
+    return await asked(door, "/v1/evals/caller", { method: "POST", body: asking });
+}
+/** One page of a call's log as this reader narrowed it: where the seqs a failure names are read. */
+export async function entriesOf(door, call, narrowed = {}) {
+    const query = new URLSearchParams({
+        after: String(narrowed.after ?? 0),
+        limit: String(narrowed.limit ?? 500),
+    });
+    if (narrowed.types !== undefined)
+        query.set("types", narrowed.types.join(","));
+    const answered = await asked(door, `/v1/calls/${encodeURIComponent(call)}/events?${query.toString()}`);
+    // 204 is the door saying the call is over and this cursor has all of it: an empty body, and an
+    // empty page is the honest reading of it. Every other refusal still travels as an error.
+    return answered?.entries ?? [];
+}
+/** Which calls this agent handled, newest first: the list drift reads its two windows out of. */
+export async function theSessions(door, agent, limit) {
+    const path = `/v1/agents/${encodeURIComponent(agent)}/sessions?limit=${limit}`;
+    const answered = await asked(door, path);
+    return answered.calls;
+}
+/**
+ * `GET /.well-known/pinecall`, the one door that takes no key and asks nothing of the caller.
+ *
+ * A gateway too old to answer it, or one that does not answer at all, reads as a gateway that
+ * opens no sign-up: guessing that way offers nothing, and the other way would walk a person
+ * through typing a password for a door that refuses.
+ */
+export async function discovered(url) {
+    const shut = { version: "", cloud: false, signup: false, min_password: 0 };
+    try {
+        const answer = await fetch(`${url.replace(/\/$/, "")}/.well-known/pinecall`);
+        if (!answer.ok)
+            return shut;
+        const said = (await answer.json());
+        return {
+            version: said.version ?? "",
+            cloud: said.cloud === true,
+            signup: said.signup === true,
+            min_password: said.min_password ?? 0,
+        };
+    }
+    catch {
+        return shut;
+    }
+}
+export class Refused extends Error {
+    status;
+    text;
+    constructor(status, text) {
+        super(`the gateway answered ${status}: ${text}`);
+        this.status = status;
+        this.text = text;
+    }
+}
+// One door, one fetch, one refusal: every verb that knocks here spells neither the header, the
+// JSON nor the error sentence again. What the gateway refused travels as its own words, because
+// "the agent is not registered" and "no ring installed" both arrive this way and both name the fix.
+export async function asked(door, path, sent = {}) {
+    // A door that mints the first key takes none (`/v1/signup`), so an empty key sends no header
+    // rather than an empty Bearer: what is not held is not claimed.
+    const authorization = door.apiKey === "" ? {} : { authorization: `Bearer ${door.apiKey}` };
+    const answered = await fetch(`${door.url.replace(/\/$/, "")}${path}`, {
+        method: sent.method ?? "GET",
+        headers: { ...authorization, "content-type": "application/json" },
+        ...(sent.body === undefined ? {} : { body: JSON.stringify(sent.body) }),
+    });
+    const text = await answered.text();
+    if (!answered.ok)
+        throw new Refused(answered.status, text);
+    return (text === "" ? null : JSON.parse(text));
+}
+//# sourceMappingURL=gateway.js.map

package/dist/cli/testing/gateway.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gateway.js","sourceRoot":"","sources":["../../../src/cli/testing/gateway.ts"],"names":[],"mappings":"AAAA,iGAAiG;AA2GjG,oGAAoG;AACpG,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAU,EAAE,MAAc;IACnD,OAAO,MAAM,KAAK,CAAU,IAAI,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;AACvF,CAAC;AAED,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAU,EAAE,KAAa,EAAE,KAAc;IACrE,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5D,IAAI,KAAK,KAAK,SAAS;QAAE,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAsB,IAAI,EAAE,kBAAkB,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC9F,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAU,EAAE,EAAU;IACjD,OAAO,MAAM,KAAK,CAAU,IAAI,EAAE,kBAAkB,kBAAkB,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;AAChF,CAAC;AAsBD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAU,EACV,MAAiE;IAEjE,OAAO,MAAM,KAAK,CAAa,IAAI,EAAE,kBAAkB,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;AAC7F,CAAC;AAWD,oGAAoG;AACpG,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAU,EAAE,IAAY,EAAE,QAAQ,GAAa,EAAE;IAC/E,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC;QAChC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QAClC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,GAAG,CAAC;KACrC,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS;QAAE,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,IAAI,EACJ,aAAa,kBAAkB,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,QAAQ,EAAE,EAAE,CACnE,CAAC;IACF,+FAA+F;IAC/F,yFAAyF;IACzF,OAAO,QAAQ,EAAE,OAAO,IAAI,EAAE,CAAC;AACjC,CAAC;AAED,iGAAiG;AACjG,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAU,EAAE,KAAa,EAAE,KAAa;IACxE,MAAM,IAAI,GAAG,cAAc,kBAAkB,CAAC,KAAK,CAAC,mBAAmB,KAAK,EAAE,CAAC;IAC/E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAA2B,IAAI,EAAE,IAAI,CAAC,CAAC;IACnE,OAAO,QAAQ,CAAC,KAAK,CAAC;AACxB,CAAC;AAYD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW;IAC1C,MAAM,IAAI,GAAe,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;IACvF,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,uBAAuB,CAAC,CAAC;QAC7E,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QAC5B,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAwB,CAAC;QAC1D,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,KAAK,IAAI;YAC1B,MAAM,EAAE,IAAI,CAAC,MAAM,KAAK,IAAI;YAC5B,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,CAAC;SACrC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,OAAO,OAAQ,SAAQ,KAAK;IAErB,MAAM;IACN,IAAI;IAFf,YACW,MAAc,EACd,IAAY;QAErB,KAAK,CAAC,wBAAwB,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;sBAHxC,MAAM;oBACN,IAAI;IAGf,CAAC;CACF;AAED,+FAA+F;AAC/F,gGAAgG;AAChG,oGAAoG;AACpG,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,IAAU,EACV,IAAY,EACZ,IAAI,GAAwC,EAAE;IAE9C,6FAA6F;IAC7F,gEAAgE;IAChE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IAC3F,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE;QACpE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK;QAC5B,OAAO,EAAE,EAAE,GAAG,aAAa,EAAE,cAAc,EAAE,kBAAkB,EAAE;QACjE,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;KACxE,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAM,CAAC;AACtD,CAAC"}

package/dist/cli/testing/goldens.d.ts

@@ -0,0 +1,61 @@
+/** A golden read off the tenant's disk: the schema the runner takes, and nothing more. */
+/** A fact from the tenant's backend, injected mid-conversation the way the app would send it. */
+export interface EventStep {
+    /** How many of the caller's turns have been answered when it arrives. 0 is before a word. */
+    after_turn?: number;
+    name: string;
+    data?: Record<string, unknown>;
+}
+/** What a golden is accepted against: one field is one question, and one graph answers it. */
+export interface Expect {
+    tools?: string[];
+    /** The mirror of `tools`: none of these ran. `not` is about words; this one is about the log. */
+    not_tools?: string[];
+    not?: string[];
+    says?: string[];
+    grounded?: boolean;
+    register?: "tu" | "usted";
+    replies?: boolean;
+}
+/** One conversation written down: where it starts, what is said, and what is expected of it. */
+export interface Golden {
+    name: string;
+    state?: Record<string, unknown>;
+    input: string[];
+    /** What memory already holds about this caller when the call opens, in the words a fact is
+     * written in. A golden that seeds one asks the question memory exists for — does the agent USE
+     * what it remembered — and it never touches the memory table: the facts are answered to the
+     * `recall` tool for this call and nothing is written down. */
+    memory?: string[];
+    events?: EventStep[];
+    /** `YYYY-MM-DD`: the day the call is opened on. A golden that names a weekday pins the one
+     * it means, so it reads the same in September and in a year. Without it, the real today. */
+    today?: string;
+    expect?: Expect;
+    /** The call `pinecall runs promote` read this out of. Provenance: no judge ever reads it. */
+    promoted_from?: string;
+}
+/**
+ * Where a golden lives when nobody said. `test/` itself already holds the tenant's own vitest
+ * files, the `--state` cases `pinecall prompt` reads and the captured prompts, so a golden gets a
+ * directory of its own rather than a naming convention inside a crowded one.
+ */
+export declare const GOLDENS = "test/goldens";
+/** What to say when the default directory is not there: the path, and what belongs in it. */
+export declare const NO_GOLDENS = "no goldens at test/goldens: write one, or name the file or directory to run";
+/** Every golden under those paths, in the order a person would read the directory. */
+export declare function goldensIn(paths: string[]): Promise<Golden[]>;
+/**
+ * Every case under those paths, whatever shape of golden the caller reads. A file holds one case
+ * or a list of them, and a case that named itself keeps that name — the file's own basename is
+ * what the rest are called, numbered when there are several, so a report names something a person
+ * can grep for in their own directory. The extraction goldens are read through this too.
+ */
+export declare function casesIn<T extends {
+    name?: string;
+}>(paths: string[], fallback: string): Promise<T[]>;
+/** The cases whose name carries that text, or all of them when nobody narrowed the run. */
+export declare function matching<T extends {
+    name: string;
+}>(cases: T[], grep: string | undefined): T[];
+//# sourceMappingURL=goldens.d.ts.map

package/dist/cli/testing/goldens.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"goldens.d.ts","sourceRoot":"","sources":["../../../src/cli/testing/goldens.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAK1F,iGAAiG;AACjG,MAAM,WAAW,SAAS;IACxB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,8FAA8F;AAC9F,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,iGAAiG;IACjG,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,gGAAgG;AAChG,MAAM,WAAW,MAAM;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB;;;kEAG8D;IAC9D,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IACrB;gGAC4F;IAC5F,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6FAA6F;IAC7F,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;GAIG;AACH,eAAO,MAAM,OAAO,iBAAiB,CAAC;AAEtC,6FAA6F;AAC7F,eAAO,MAAM,UAAU,gFACsD,CAAC;AAE9E,sFAAsF;AACtF,wBAAsB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAElE;AAED;;;;;GAKG;AACH,wBAAsB,OAAO,CAAC,CAAC,SAAS;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAM1G;AAED,2FAA2F;AAC3F,wBAAgB,QAAQ,CAAC,CAAC,SAAS;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,CAAC,EAAE,CAI9F"}

package/dist/cli/testing/goldens.js

@@ -0,0 +1,54 @@
+/** A golden read off the tenant's disk: the schema the runner takes, and nothing more. */
+import { readdir, readFile, stat } from "node:fs/promises";
+import { basename, extname, join, resolve } from "node:path";
+/**
+ * Where a golden lives when nobody said. `test/` itself already holds the tenant's own vitest
+ * files, the `--state` cases `pinecall prompt` reads and the captured prompts, so a golden gets a
+ * directory of its own rather than a naming convention inside a crowded one.
+ */
+export const GOLDENS = "test/goldens";
+/** What to say when the default directory is not there: the path, and what belongs in it. */
+export const NO_GOLDENS = `no goldens at ${GOLDENS}: write one, or name the file or directory to run`;
+/** Every golden under those paths, in the order a person would read the directory. */
+export async function goldensIn(paths) {
+    return await casesIn(paths, GOLDENS);
+}
+/**
+ * Every case under those paths, whatever shape of golden the caller reads. A file holds one case
+ * or a list of them, and a case that named itself keeps that name — the file's own basename is
+ * what the rest are called, numbered when there are several, so a report names something a person
+ * can grep for in their own directory. The extraction goldens are read through this too.
+ */
+export async function casesIn(paths, fallback) {
+    const found = [];
+    for (const path of paths.length > 0 ? paths : [fallback]) {
+        for (const file of await filesUnder(resolve(path)))
+            found.push(...(await casesOf(file)));
+    }
+    return found;
+}
+/** The cases whose name carries that text, or all of them when nobody narrowed the run. */
+export function matching(cases, grep) {
+    if (grep === undefined)
+        return cases;
+    const wanted = grep.toLowerCase();
+    return cases.filter((one) => one.name.toLowerCase().includes(wanted));
+}
+async function casesOf(file) {
+    const read = JSON.parse(await readFile(file, "utf8"));
+    const written = Array.isArray(read) ? read : [read];
+    const stem = basename(file, extname(file));
+    return written.map((one, index) => ({
+        ...one,
+        name: one.name ?? (written.length > 1 ? `${stem} #${index + 1}` : stem),
+    }));
+}
+// A directory is read one level deep and sorted: goldens are a flat folder of files, and the order
+// a run reports in should be the order `ls` prints, not the order the filesystem happens to answer.
+async function filesUnder(path) {
+    if (!(await stat(path).catch(() => null))?.isDirectory())
+        return [path];
+    const names = await readdir(path);
+    return names.filter((name) => extname(name) === ".json").sort().map((name) => join(path, name));
+}
+//# sourceMappingURL=goldens.js.map

package/dist/cli/testing/goldens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"goldens.js","sourceRoot":"","sources":["../../../src/cli/testing/goldens.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAE1F,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAyC7D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,cAAc,CAAC;AAEtC,6FAA6F;AAC7F,MAAM,CAAC,MAAM,UAAU,GACrB,iBAAiB,OAAO,mDAAmD,CAAC;AAE9E,sFAAsF;AACtF,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAe;IAC7C,OAAO,MAAM,OAAO,CAAS,KAAK,EAAE,OAAO,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAA8B,KAAe,EAAE,QAAgB;IAC1F,MAAM,KAAK,GAAQ,EAAE,CAAC;IACtB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzD,KAAK,MAAM,IAAI,IAAI,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9F,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,QAAQ,CAA6B,KAAU,EAAE,IAAwB;IACvF,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAClC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,OAAO,CAA8B,IAAY;IAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAY,CAAC;IACjE,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,IAAY,CAAC,CAAC,CAAC,CAAC,IAAS,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3C,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,GAAG;QACN,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;KACxE,CAAC,CAAC,CAAC;AACN,CAAC;AAED,mGAAmG;AACnG,oGAAoG;AACpG,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,EAAE;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAClG,CAAC"}

package/dist/cli/testing/latency.d.ts

@@ -0,0 +1,15 @@
+/** The latency a golden was answered with: the median of every turn's own metrics entry. */
+import { TURN_METRICS } from "../view.js";
+import type { Entry } from "./gateway.js";
+/** The three numbers a report prints per golden, in seconds, as livekit measured each turn. */
+export type Medians = Partial<Record<(typeof TURN_METRICS)[number], number>>;
+/**
+ * The medians of one call. The median and not the mean: one cold first turn would drag an average
+ * of four turns somewhere no caller ever waited, and the number here is meant to be the one a
+ * typical caller felt. Nothing is computed that the log does not already carry — a metric the
+ * session never measured is absent from the answer rather than reported as a zero.
+ */
+export declare function mediansOf(entries: Entry[]): Medians;
+/** The medians on one line, in the order a person reads them: the wait first, then where it went. */
+export declare function latencyLine(medians: Medians): string;
+//# sourceMappingURL=latency.d.ts.map

package/dist/cli/testing/latency.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"latency.d.ts","sourceRoot":"","sources":["../../../src/cli/testing/latency.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAE5F,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE1C,+FAA+F;AAC/F,MAAM,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAE7E;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAOnD;AAED,qGAAqG;AACrG,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAKpD"}