trxa-tool 0.0.0-stage → 0.2.0

package/dist/plan-schema.d.ts

@@ -0,0 +1,27 @@
+/**
+ * A TestPlan groups multiple independent Test Suites for combined scheduling/reporting -- e.g.
+ * a whole "Accounts Receivable" testing cycle spanning several suites. Deliberately flat: no
+ * dependency graph and no input bindings across the suite boundary. That's what a TestSuite's
+ * `dependsOn`/`inputBindings` already does *within* one suite; if two suites genuinely need to
+ * share data, they were never really two suites -- merge their test cases into one suite
+ * instead. A TestPlan only composes already-independently-runnable suites for reporting.
+ *
+ * This matches TestRail's actual "Test Plan" concept (a plan bundles multiple suites/runs for
+ * one release) rather than suite-of-suites nesting.
+ */
+export interface TestPlanEntry {
+    suiteId: string;
+    /** Path to the suite's JSON file, relative to this plan file. */
+    suitePath: string;
+}
+export interface TestPlan {
+    id: string;
+    name: string;
+    description: string;
+    /** Array order is the suggested run order; no dependsOn, no bindings across suites. */
+    suites: TestPlanEntry[];
+}
+/** Validates a parsed JSON object against the TestPlan shape. Does not check that referenced
+ *  suite files actually exist or are themselves valid -- that's the `validate-plan` CLI
+ *  command's job, since it needs filesystem access this pure function doesn't have. */
+export declare function validateTestPlan(value: unknown): TestPlan;

package/dist/plan-schema.js

@@ -0,0 +1,45 @@
+/**
+ * A TestPlan groups multiple independent Test Suites for combined scheduling/reporting -- e.g.
+ * a whole "Accounts Receivable" testing cycle spanning several suites. Deliberately flat: no
+ * dependency graph and no input bindings across the suite boundary. That's what a TestSuite's
+ * `dependsOn`/`inputBindings` already does *within* one suite; if two suites genuinely need to
+ * share data, they were never really two suites -- merge their test cases into one suite
+ * instead. A TestPlan only composes already-independently-runnable suites for reporting.
+ *
+ * This matches TestRail's actual "Test Plan" concept (a plan bundles multiple suites/runs for
+ * one release) rather than suite-of-suites nesting.
+ */
+function fail(field, reason) {
+    throw new Error(`Invalid TestPlan: ${field} ${reason}`);
+}
+function assertString(value, field) {
+    if (typeof value !== "string" || value.length === 0)
+        fail(field, "must be a non-empty string");
+}
+function validateTestPlanEntry(value, index) {
+    if (typeof value !== "object" || value === null)
+        fail(`suites[${index}]`, "must be an object");
+    const v = value;
+    assertString(v.suiteId, `suites[${index}].suiteId`);
+    assertString(v.suitePath, `suites[${index}].suitePath`);
+    return { suiteId: v.suiteId, suitePath: v.suitePath };
+}
+/** Validates a parsed JSON object against the TestPlan shape. Does not check that referenced
+ *  suite files actually exist or are themselves valid -- that's the `validate-plan` CLI
+ *  command's job, since it needs filesystem access this pure function doesn't have. */
+export function validateTestPlan(value) {
+    if (typeof value !== "object" || value === null)
+        fail("plan", "must be an object");
+    const v = value;
+    assertString(v.id, "id");
+    assertString(v.name, "name");
+    assertString(v.description, "description");
+    if (!Array.isArray(v.suites))
+        fail("suites", "must be an array");
+    return {
+        id: v.id,
+        name: v.name,
+        description: v.description,
+        suites: v.suites.map(validateTestPlanEntry),
+    };
+}

package/dist/planrun-schema.d.ts

@@ -0,0 +1,53 @@
+/**
+ * A PlanRun is the thin, persisted record that gives a Test Plan's execution a real identity --
+ * `{id, planId, startedAt, suiteIds}`. The plan stays the anchor and the run a dated instance of
+ * it, but a run can also carry an optional `name` of its own. People refer to a run by what it
+ * tested ("Accounts Payable regression, PQU 10.0.48"), not by its start time, and TestRail and
+ * Azure DevOps Test Plans both let a run be named for that reason. The name never replaces the
+ * plan's identity: `planId` still says which plan it ran.
+ *
+ * `suiteIds` is a snapshot of the plan's suite composition at the moment the run started --
+ * confirmed live that a PlanRun without one breaks: adding "create-new-vendor" as a second suite
+ * to the Accounts Payable plan made an already-completed, all-passed run retroactively show as
+ * "Missing runs" in Studio, because status was being recomputed against the *current* plan's
+ * suite list instead of whatever suites existed when that run actually happened. A run is a
+ * historical record; it must not change shape because the plan changed after the fact. See
+ * plan-runner.ts's `aggregatePlanStatus`, which reads this snapshot instead of `TestPlan.suites`.
+ *
+ * `suiteNames` extends the same principle to display: it snapshots each suite's human-readable
+ * `name` (suiteId -> name) at that same moment, so Studio can show "Create new vendor" instead of
+ * "create-new-vendor" without a live lookup against the suite file -- which, per the same
+ * suiteIds rationale, could have been renamed or deleted by the time someone views an old run.
+ *
+ * Suite-level TestRun files (report-schema.ts) are unchanged by this -- a PlanRun's `id` is the
+ * same string already used as the `<runId>` suffix in `testruns/<suiteId>-<runId>.json`. This
+ * file's only job is to make that id a real, validated, directly-queryable record instead of an
+ * unvalidated convention nobody checks.
+ */
+export interface PlanRun {
+    id: string;
+    planId: string;
+    startedAt: string;
+    suiteIds: string[];
+    suiteNames: Record<string, string>;
+    /** What people call this run, e.g. "Accounts Payable regression, PQU 10.0.48". Optional: a run
+     *  without one is shown by its plan's name and start time. */
+    name?: string;
+}
+/** Longest run name accepted, so a pasted paragraph can't end up as a list label. */
+export declare const RUN_NAME_MAX_LENGTH = 120;
+/** Trims a run name and checks its length; an empty name means "no name". Shared by the plan run
+ *  and test run schemas, the CLI's `--name` flag and Studio's rename endpoints. */
+export declare function normalizeRunName(value: unknown, field?: string): string | undefined;
+/** Colons aren't legal in Windows filenames, so `id` is `startedAt` with them swapped for dashes
+ *  -- still lexically sortable, still human-readable, and trivially derivable from `startedAt`
+ *  rather than needing a separate id-generation scheme. A caller never invents this string
+ *  themselves (see `planrun.ts`'s `initPlanRun`), which is the whole point: `runId` used to be an
+ *  arbitrary, unvalidated string a human or LLM picked by hand. `suites` is captured from the plan
+ *  as it exists right now, at creation time -- never re-derived later -- and split into the two
+ *  snapshot fields above. */
+export declare function createPlanRun(planId: string, suites: {
+    id: string;
+    name: string;
+}[], name?: string): PlanRun;
+export declare function validatePlanRun(value: unknown): PlanRun;

package/dist/planrun-schema.js

@@ -0,0 +1,111 @@
+/**
+ * A PlanRun is the thin, persisted record that gives a Test Plan's execution a real identity --
+ * `{id, planId, startedAt, suiteIds}`. The plan stays the anchor and the run a dated instance of
+ * it, but a run can also carry an optional `name` of its own. People refer to a run by what it
+ * tested ("Accounts Payable regression, PQU 10.0.48"), not by its start time, and TestRail and
+ * Azure DevOps Test Plans both let a run be named for that reason. The name never replaces the
+ * plan's identity: `planId` still says which plan it ran.
+ *
+ * `suiteIds` is a snapshot of the plan's suite composition at the moment the run started --
+ * confirmed live that a PlanRun without one breaks: adding "create-new-vendor" as a second suite
+ * to the Accounts Payable plan made an already-completed, all-passed run retroactively show as
+ * "Missing runs" in Studio, because status was being recomputed against the *current* plan's
+ * suite list instead of whatever suites existed when that run actually happened. A run is a
+ * historical record; it must not change shape because the plan changed after the fact. See
+ * plan-runner.ts's `aggregatePlanStatus`, which reads this snapshot instead of `TestPlan.suites`.
+ *
+ * `suiteNames` extends the same principle to display: it snapshots each suite's human-readable
+ * `name` (suiteId -> name) at that same moment, so Studio can show "Create new vendor" instead of
+ * "create-new-vendor" without a live lookup against the suite file -- which, per the same
+ * suiteIds rationale, could have been renamed or deleted by the time someone views an old run.
+ *
+ * Suite-level TestRun files (report-schema.ts) are unchanged by this -- a PlanRun's `id` is the
+ * same string already used as the `<runId>` suffix in `testruns/<suiteId>-<runId>.json`. This
+ * file's only job is to make that id a real, validated, directly-queryable record instead of an
+ * unvalidated convention nobody checks.
+ */
+/** Longest run name accepted, so a pasted paragraph can't end up as a list label. */
+export const RUN_NAME_MAX_LENGTH = 120;
+/** Trims a run name and checks its length; an empty name means "no name". Shared by the plan run
+ *  and test run schemas, the CLI's `--name` flag and Studio's rename endpoints. */
+export function normalizeRunName(value, field = "name") {
+    if (value === undefined || value === null)
+        return undefined;
+    if (typeof value !== "string")
+        throw new Error(`Invalid run name: ${field} must be a string`);
+    const name = value.trim().replace(/\s+/g, " ");
+    if (name.length > RUN_NAME_MAX_LENGTH) {
+        throw new Error(`Invalid run name: ${field} must be at most ${RUN_NAME_MAX_LENGTH} characters`);
+    }
+    return name || undefined;
+}
+/** Colons aren't legal in Windows filenames, so `id` is `startedAt` with them swapped for dashes
+ *  -- still lexically sortable, still human-readable, and trivially derivable from `startedAt`
+ *  rather than needing a separate id-generation scheme. A caller never invents this string
+ *  themselves (see `planrun.ts`'s `initPlanRun`), which is the whole point: `runId` used to be an
+ *  arbitrary, unvalidated string a human or LLM picked by hand. `suites` is captured from the plan
+ *  as it exists right now, at creation time -- never re-derived later -- and split into the two
+ *  snapshot fields above. */
+export function createPlanRun(planId, suites, name) {
+    const startedAt = new Date().toISOString();
+    const runName = normalizeRunName(name);
+    return {
+        id: startedAt.replace(/:/g, "-"),
+        planId,
+        startedAt,
+        suiteIds: suites.map((s) => s.id),
+        suiteNames: Object.fromEntries(suites.map((s) => [s.id, s.name])),
+        ...(runName ? { name: runName } : {}),
+    };
+}
+function fail(field, reason) {
+    throw new Error(`Invalid PlanRun: ${field} ${reason}`);
+}
+function assertString(value, field) {
+    if (typeof value !== "string" || value.length === 0)
+        fail(field, "must be a non-empty string");
+}
+/** Missing `suiteIds` (a PlanRun persisted before this field existed) defaults to an empty
+ *  snapshot rather than failing validation outright -- an old run still loads and displays, it
+ *  just can't show a per-suite breakdown, which is honest given the data actually on disk. */
+function assertSuiteIds(value) {
+    if (value === undefined)
+        return [];
+    if (!Array.isArray(value))
+        fail("suiteIds", "must be an array of strings");
+    for (const [i, entry] of value.entries())
+        assertString(entry, `suiteIds[${i}]`);
+    return value;
+}
+/** Same missing-field tolerance as `assertSuiteIds`, for the same reason -- a PlanRun persisted
+ *  before `suiteNames` existed defaults to an empty map; callers fall back to the raw suiteId for
+ *  display rather than failing to load the run at all. */
+function assertSuiteNames(value) {
+    if (value === undefined)
+        return {};
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+        fail("suiteNames", "must be an object mapping suiteId to name");
+    }
+    const v = value;
+    for (const [id, name] of Object.entries(v))
+        assertString(name, `suiteNames.${id}`);
+    return v;
+}
+export function validatePlanRun(value) {
+    if (typeof value !== "object" || value === null)
+        fail("planRun", "must be an object");
+    const v = value;
+    assertString(v.id, "id");
+    assertString(v.planId, "planId");
+    assertString(v.startedAt, "startedAt");
+    const suiteIds = assertSuiteIds(v.suiteIds);
+    const suiteNames = assertSuiteNames(v.suiteNames);
+    let name;
+    try {
+        name = normalizeRunName(v.name);
+    }
+    catch (err) {
+        fail("name", err.message.replace(/^Invalid run name: name /, ""));
+    }
+    return { id: v.id, planId: v.planId, startedAt: v.startedAt, suiteIds, suiteNames, ...(name ? { name } : {}) };
+}

package/dist/planrun.d.ts

@@ -0,0 +1,20 @@
+import { type PlanRun } from "./planrun-schema.js";
+export declare function readPlanRun(filePath: string): Promise<PlanRun>;
+/** Every PlanRun file in `dir`, optionally filtered to one plan, newest-started first. Shared by
+ *  the CLI and Studio server so the two can't drift on what "list the runs for this plan" means.
+ *  Silently skips a file that fails to parse/validate rather than throwing -- one malformed run
+ *  shouldn't take down the whole list, same tolerance `testruns.ts`'s listing already has. */
+export declare function listPlanRuns(dir: string, planId?: string): Promise<PlanRun[]>;
+/** Creates and persists a fresh PlanRun into `dir`, auto-naming the file from the generated id --
+ *  the caller supplies a directory, not a filename, so there's no chance of inventing an arbitrary
+ *  runId string the way `init-run`'s own `-o <path>` always required. `suites` must be the calling
+ *  plan's current suites (id + name) -- captured once, here, so this run's status/display never
+ *  drifts if the plan's suites are later added, removed, or renamed (see planrun-schema.ts's
+ *  `PlanRun.suiteIds`/`suiteNames`). */
+export declare function initPlanRun(planId: string, suites: {
+    id: string;
+    name: string;
+}[], dir: string, name?: string): Promise<{
+    planRun: PlanRun;
+    filePath: string;
+}>;

package/dist/planrun.js

@@ -0,0 +1,46 @@
+import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { createPlanRun, validatePlanRun } from "./planrun-schema.js";
+export async function readPlanRun(filePath) {
+    const raw = await readFile(filePath, "utf-8");
+    return validatePlanRun(JSON.parse(raw));
+}
+/** Every PlanRun file in `dir`, optionally filtered to one plan, newest-started first. Shared by
+ *  the CLI and Studio server so the two can't drift on what "list the runs for this plan" means.
+ *  Silently skips a file that fails to parse/validate rather than throwing -- one malformed run
+ *  shouldn't take down the whole list, same tolerance `testruns.ts`'s listing already has. */
+export async function listPlanRuns(dir, planId) {
+    let files;
+    try {
+        files = (await readdir(dir)).filter((f) => f.endsWith(".json"));
+    }
+    catch {
+        return [];
+    }
+    const runs = [];
+    for (const file of files) {
+        try {
+            const run = await readPlanRun(path.join(dir, file));
+            if (!planId || run.planId === planId)
+                runs.push(run);
+        }
+        catch {
+            // Skip malformed/unrelated files.
+        }
+    }
+    runs.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+    return runs;
+}
+/** Creates and persists a fresh PlanRun into `dir`, auto-naming the file from the generated id --
+ *  the caller supplies a directory, not a filename, so there's no chance of inventing an arbitrary
+ *  runId string the way `init-run`'s own `-o <path>` always required. `suites` must be the calling
+ *  plan's current suites (id + name) -- captured once, here, so this run's status/display never
+ *  drifts if the plan's suites are later added, removed, or renamed (see planrun-schema.ts's
+ *  `PlanRun.suiteIds`/`suiteNames`). */
+export async function initPlanRun(planId, suites, dir, name) {
+    const planRun = createPlanRun(planId, suites, name);
+    const filePath = path.join(dir, `${planRun.id}.json`);
+    await mkdir(dir, { recursive: true });
+    await writeFile(filePath, JSON.stringify(planRun, null, 2), "utf-8");
+    return { planRun, filePath };
+}

package/dist/report-schema.d.ts

@@ -0,0 +1,92 @@
+/** Persisted, durable record of a suite run -- this is the mechanism that answers "keep track
+ *  of each test case, expected input, and actual output" instead of relying on chat memory. */
+/** "skipped" is assigned automatically by `nextStep`/`cmdResolveNext` when an entry's `runIf`
+ *  (`suite-schema.ts`) evaluates false -- never something the orchestrating LLM passes to
+ *  `record-result` manually, same as "blocked". Neither a pass nor a failure: a downstream entry
+ *  that `dependsOn` a skipped one is propagated to "blocked" (see `nextStep`'s `newlyBlocked`
+ *  check), not left waiting forever. */
+export type TestCaseStatus = "pending" | "running" | "passed" | "failed" | "blocked" | "skipped";
+/** Statuses a test case never leaves once reached -- used by both `suite-runner.ts` (to decide
+ *  whether a whole suite run is done) and `plan-runner.ts` (to roll that up across suites).
+ *  Exported from here, not duplicated in each, so the two can't drift out of sync with each other
+ *  or with `TestCaseStatus` above. Kept as `Set<string>` rather than `Set<TestCaseStatus>` so
+ *  callers can `.has()` a possibly-missing status (e.g. `result?.status ?? ""`) without a type
+ *  error. */
+export declare const TERMINAL_STATUSES: ReadonlySet<string>;
+export interface AssertionResult {
+    description: string;
+    passed: boolean;
+    /** What was actually observed (an OData field value, or the matched/missing UI message). */
+    actual: string;
+}
+export interface TestCaseResult {
+    status: TestCaseStatus;
+    outputs: Record<string, string>;
+    assertionResults: AssertionResult[];
+    /** Run order this test case actually started in (1-based), assigned once by `resolve-next` the
+     *  first time it transitions to "running" -- not the suite's declared entry order, which can
+     *  differ once a DAG has parallel-eligible branches. */
+    sequence?: number;
+    /** Resolved input values this test case actually ran with -- captured automatically from
+     *  `resolveInputs`'s output when `resolve-next` marks the test case "running" (deterministic
+     *  bindings only; doesn't include values invented for `"autogenerated"` bindings, since the CLI
+     *  never sees those). `record-result` may pass a fuller set (including autogenerated values the
+     *  orchestrating LLM invented) to override this. */
+    inputs?: Record<string, string>;
+    /** ISO timestamp captured automatically when `resolve-next` marks this test case "running". */
+    startedAt?: string;
+    /** Milliseconds between `startedAt` and this result reaching a terminal status -- computed
+     *  automatically by `recordResult`, not supplied by a caller. */
+    durationMs?: number;
+    /** Measured, not self-reported, via `record-result --tokens` -- but treat it as a rough
+     *  order-of-magnitude signal, not a trustworthy number. This CLI has no way to observe token
+     *  usage itself, so `token-usage.ts`'s `measure-tokens` sources it from Claude Code's own local
+     *  session transcript (`~/.claude/projects/.../<session>.jsonl`). CONFIRMED UNRELIABLE whenever
+     *  more than one Claude Code session might be active against this project directory during the
+     *  measurement window -- live-tested 2026-08-20 running suites in parallel: +24% to +98% (nearly
+     *  2x) inflation from cross-session misattribution, cross-checked against each session's own
+     *  authoritative usage total. That's not a rare edge case; it's close to the default way anyone
+     *  actually uses this (parallel suites, or a Desktop session open alongside a background job).
+     *  See `token-usage.ts` for why there's no clean fix available today. Also deliberately a
+     *  Claude-Code-only bonus, not a core requirement: this whole pipeline (`resolve-next`/
+     *  `record-result`/the replay skill's instructions) is meant to work with any sufficiently
+     *  capable agentic LLM driving it, and nothing about pass/fail or captured outputs depends on
+     *  this field. Running under any other agent -- or under Claude Cowork, which executes on
+     *  Anthropic's own servers rather than the local machine -- simply leaves it unset, since
+     *  there's no local session transcript to read either way; that's expected, not a bug. */
+    tokensUsed?: number;
+    /** Cost in USD for this test case, when the host environment can report one directly (e.g. a
+     *  future headless `claude -p --output-format json` invocation exposes `total_cost_usd`).
+     *  Deliberately separate from `tokensUsed`: a $ figure needs a pricing table keyed by model and
+     *  cache-hit/miss that changes over time, which this CLI has no business maintaining -- so this
+     *  field exists for whichever layer *does* have an authoritative cost to report, and is left
+     *  unset otherwise rather than computed from `tokensUsed` with a guessed price. */
+    costUsd?: number;
+    /** Raw `FormState.Messages` text captured by the orchestrating LLM via `record-result
+     *  --messages` -- the D365FO-side evidence for what actually happened, distinct from
+     *  `assertionResults`' pass/fail judgments. A single entry's text can carry a tab-delimited
+     *  hierarchy (confirmed live: posting blocked by credit management returned one `Warning` as
+     *  `"Posting\tPosting\tSales order: 000963\tOrder is in credit management."`) -- this is the
+     *  same content D365FO's web client renders as an indented "Message details" tree; there's no
+     *  separate data source behind that flyout. The skill converts tabs to `" > "` before passing
+     *  this flag so the hierarchy survives instead of collapsing into a run-on sentence. */
+    messages?: string[];
+}
+export interface TestRun {
+    /** The suite's id -- or, for a `scope: "testcase"` run, the test case's id. */
+    suiteId: string;
+    startedAt: string;
+    results: Record<string, TestCaseResult>;
+    /** What this run was started for. Absent on runs the CLI created for a suite (the original
+     *  behavior). `testcase` marks a run of one test case on its own, with no suite around it. */
+    scope?: "suite" | "testcase";
+    /** The Studio agent run executing this record, when one created it. */
+    agentRunId?: string;
+    /** What people call this run, e.g. "Vendor onboarding, PQU 10.0.48". Optional; a suite run that
+     *  belongs to a plan run is named by its PlanRun instead. See planrun-schema.ts's `name`. */
+    name?: string;
+}
+/** Creates a fresh TestRun with every suite entry pre-populated as "pending". */
+export declare function createTestRun(suiteId: string, testCaseIds: string[], extra?: Pick<TestRun, "scope" | "agentRunId" | "name">): TestRun;
+/** Validates a parsed JSON object against the TestRun shape. */
+export declare function validateTestRun(value: unknown): TestRun;

package/dist/report-schema.js

@@ -0,0 +1,115 @@
+import { normalizeRunName } from "./planrun-schema.js";
+/** Statuses a test case never leaves once reached -- used by both `suite-runner.ts` (to decide
+ *  whether a whole suite run is done) and `plan-runner.ts` (to roll that up across suites).
+ *  Exported from here, not duplicated in each, so the two can't drift out of sync with each other
+ *  or with `TestCaseStatus` above. Kept as `Set<string>` rather than `Set<TestCaseStatus>` so
+ *  callers can `.has()` a possibly-missing status (e.g. `result?.status ?? ""`) without a type
+ *  error. */
+export const TERMINAL_STATUSES = new Set(["passed", "failed", "blocked", "skipped"]);
+/** Creates a fresh TestRun with every suite entry pre-populated as "pending". */
+export function createTestRun(suiteId, testCaseIds, extra = {}) {
+    const name = normalizeRunName(extra.name);
+    const results = {};
+    for (const id of testCaseIds) {
+        results[id] = { status: "pending", outputs: {}, assertionResults: [] };
+    }
+    return {
+        suiteId,
+        startedAt: new Date().toISOString(),
+        results,
+        ...(extra.scope !== undefined ? { scope: extra.scope } : {}),
+        ...(extra.agentRunId !== undefined ? { agentRunId: extra.agentRunId } : {}),
+        ...(name ? { name } : {}),
+    };
+}
+function fail(field, reason) {
+    throw new Error(`Invalid TestRun: ${field} ${reason}`);
+}
+function assertString(value, field) {
+    if (typeof value !== "string" || value.length === 0)
+        fail(field, "must be a non-empty string");
+}
+const VALID_STATUSES = ["pending", "running", "passed", "failed", "blocked", "skipped"];
+function validateTestCaseResult(value, testCaseId) {
+    if (typeof value !== "object" || value === null) {
+        fail(`results.${testCaseId}`, "must be an object");
+    }
+    const v = value;
+    if (!VALID_STATUSES.includes(v.status)) {
+        fail(`results.${testCaseId}.status`, `must be one of ${VALID_STATUSES.join(", ")}`);
+    }
+    if (typeof v.outputs !== "object" || v.outputs === null || Array.isArray(v.outputs)) {
+        fail(`results.${testCaseId}.outputs`, "must be an object");
+    }
+    if (!Array.isArray(v.assertionResults)) {
+        fail(`results.${testCaseId}.assertionResults`, "must be an array");
+    }
+    if (v.sequence !== undefined && typeof v.sequence !== "number") {
+        fail(`results.${testCaseId}.sequence`, "must be a number");
+    }
+    if (v.inputs !== undefined && (typeof v.inputs !== "object" || v.inputs === null || Array.isArray(v.inputs))) {
+        fail(`results.${testCaseId}.inputs`, "must be an object");
+    }
+    if (v.startedAt !== undefined && typeof v.startedAt !== "string") {
+        fail(`results.${testCaseId}.startedAt`, "must be a string");
+    }
+    if (v.durationMs !== undefined && typeof v.durationMs !== "number") {
+        fail(`results.${testCaseId}.durationMs`, "must be a number");
+    }
+    if (v.tokensUsed !== undefined && typeof v.tokensUsed !== "number") {
+        fail(`results.${testCaseId}.tokensUsed`, "must be a number");
+    }
+    if (v.costUsd !== undefined && typeof v.costUsd !== "number") {
+        fail(`results.${testCaseId}.costUsd`, "must be a number");
+    }
+    if (v.messages !== undefined && (!Array.isArray(v.messages) || !v.messages.every((m) => typeof m === "string"))) {
+        fail(`results.${testCaseId}.messages`, "must be an array of strings");
+    }
+    return {
+        status: v.status,
+        outputs: v.outputs,
+        assertionResults: v.assertionResults,
+        ...(v.sequence !== undefined ? { sequence: v.sequence } : {}),
+        ...(v.inputs !== undefined ? { inputs: v.inputs } : {}),
+        ...(v.startedAt !== undefined ? { startedAt: v.startedAt } : {}),
+        ...(v.durationMs !== undefined ? { durationMs: v.durationMs } : {}),
+        ...(v.tokensUsed !== undefined ? { tokensUsed: v.tokensUsed } : {}),
+        ...(v.costUsd !== undefined ? { costUsd: v.costUsd } : {}),
+        ...(v.messages !== undefined ? { messages: v.messages } : {}),
+    };
+}
+/** Validates a parsed JSON object against the TestRun shape. */
+export function validateTestRun(value) {
+    if (typeof value !== "object" || value === null)
+        fail("testRun", "must be an object");
+    const v = value;
+    assertString(v.suiteId, "suiteId");
+    assertString(v.startedAt, "startedAt");
+    if (typeof v.results !== "object" || v.results === null || Array.isArray(v.results)) {
+        fail("results", "must be an object mapping testCaseId to TestCaseResult");
+    }
+    const results = {};
+    for (const [id, result] of Object.entries(v.results)) {
+        results[id] = validateTestCaseResult(result, id);
+    }
+    if (v.scope !== undefined && v.scope !== "suite" && v.scope !== "testcase") {
+        fail("scope", "must be suite or testcase");
+    }
+    if (v.agentRunId !== undefined && typeof v.agentRunId !== "string")
+        fail("agentRunId", "must be a string");
+    let name;
+    try {
+        name = normalizeRunName(v.name);
+    }
+    catch (err) {
+        fail("name", err.message.replace(/^Invalid run name: name /, ""));
+    }
+    return {
+        suiteId: v.suiteId,
+        startedAt: v.startedAt,
+        results,
+        ...(v.scope !== undefined ? { scope: v.scope } : {}),
+        ...(v.agentRunId !== undefined ? { agentRunId: v.agentRunId } : {}),
+        ...(name ? { name } : {}),
+    };
+}

package/dist/report.d.ts

@@ -0,0 +1,45 @@
+import { type AssertionResult, type TestRun, type TestCaseStatus } from "./report-schema.js";
+import type { TestSuite } from "./suite-schema.js";
+/**
+ * A TestRun JSON file is the durable "keep track of each test case, expected input, and
+ * actual output" mechanism -- it persists across turns/sessions instead of living only in chat
+ * memory, and every update goes through recordResult so the file is always the source of truth
+ * for a suite run's progress.
+ */
+export declare function readTestRun(path: string): Promise<TestRun>;
+export declare function writeTestRun(testRun: TestRun, path: string): Promise<void>;
+/** Creates and persists a fresh TestRun for a suite, with every entry pre-populated as "pending". */
+export declare function initTestRun(suite: TestSuite, path: string, extra?: Pick<TestRun, "agentRunId" | "name">): Promise<TestRun>;
+/** Creates and persists a TestRun for one test case run on its own, with no suite: a single
+ *  pending entry keyed by the test case id. `recordResult` works on it unchanged. */
+export declare function initTestCaseRun(testCaseId: string, path: string, extra?: Pick<TestRun, "agentRunId" | "name">): Promise<TestRun>;
+export interface RecordResultInput {
+    status: TestCaseStatus;
+    outputs?: Record<string, string>;
+    assertionResults?: AssertionResult[];
+    /** Full resolved input set, including any values invented for "autogenerated" bindings --
+     *  `resolve-next` itself only knows the deterministic ones (see `sequence`/`startedAt` below for
+     *  what else this call auto-fills). Overrides whatever `resolve-next`'s "running" transition
+     *  already captured, rather than being merged field-by-field with it. */
+    inputs?: Record<string, string>;
+    tokensUsed?: number;
+    costUsd?: number;
+    messages?: string[];
+}
+/** Updates one test case's result within a TestRun file and persists it back to disk. This
+ *  is the orchestrating LLM's "check-in" call after executing a test case and evaluating its
+ *  expectations -- it's what actually records expected-vs-actual, not chat narration.
+ *
+ *  Also the single place `sequence`/`startedAt`/`durationMs` get computed -- none of these are
+ *  caller-supplied, since a CLI call is exactly the right place to capture "when did this
+ *  actually happen" instead of trusting the orchestrating LLM to report it accurately:
+ *  - `sequence`/`startedAt` are assigned once, the first time a test case transitions to
+ *    "running" (from `resolve-next`); `sequence` is one more than the highest sequence already
+ *    assigned in this run, so it reflects actual start order even across a DAG's parallel-eligible
+ *    branches.
+ *  - `durationMs` is computed whenever a result reaches a terminal status and `startedAt` is
+ *    known -- `Date.now()` minus that.
+ *  Merges rather than replaces the existing result, since a test case's full lifecycle is two
+ *  calls (an implicit "running" transition, then a terminal call from `record-result`) and the
+ *  second must not erase what the first captured. */
+export declare function recordResult(path: string, testCaseId: string, update: RecordResultInput): Promise<TestRun>;

package/dist/report.js

@@ -0,0 +1,77 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { createTestRun, validateTestRun, TERMINAL_STATUSES, } from "./report-schema.js";
+/**
+ * A TestRun JSON file is the durable "keep track of each test case, expected input, and
+ * actual output" mechanism -- it persists across turns/sessions instead of living only in chat
+ * memory, and every update goes through recordResult so the file is always the source of truth
+ * for a suite run's progress.
+ */
+export async function readTestRun(path) {
+    const raw = await readFile(path, "utf-8");
+    return validateTestRun(JSON.parse(raw));
+}
+export async function writeTestRun(testRun, path) {
+    await writeFile(path, JSON.stringify(testRun, null, 2), "utf-8");
+}
+/** Creates and persists a fresh TestRun for a suite, with every entry pre-populated as "pending". */
+export async function initTestRun(suite, path, extra = {}) {
+    const testRun = createTestRun(suite.id, suite.entries.map((e) => e.testCaseId), { ...extra, scope: "suite" });
+    await writeTestRun(testRun, path);
+    return testRun;
+}
+/** Creates and persists a TestRun for one test case run on its own, with no suite: a single
+ *  pending entry keyed by the test case id. `recordResult` works on it unchanged. */
+export async function initTestCaseRun(testCaseId, path, extra = {}) {
+    const testRun = createTestRun(testCaseId, [testCaseId], { ...extra, scope: "testcase" });
+    await writeTestRun(testRun, path);
+    return testRun;
+}
+/** Updates one test case's result within a TestRun file and persists it back to disk. This
+ *  is the orchestrating LLM's "check-in" call after executing a test case and evaluating its
+ *  expectations -- it's what actually records expected-vs-actual, not chat narration.
+ *
+ *  Also the single place `sequence`/`startedAt`/`durationMs` get computed -- none of these are
+ *  caller-supplied, since a CLI call is exactly the right place to capture "when did this
+ *  actually happen" instead of trusting the orchestrating LLM to report it accurately:
+ *  - `sequence`/`startedAt` are assigned once, the first time a test case transitions to
+ *    "running" (from `resolve-next`); `sequence` is one more than the highest sequence already
+ *    assigned in this run, so it reflects actual start order even across a DAG's parallel-eligible
+ *    branches.
+ *  - `durationMs` is computed whenever a result reaches a terminal status and `startedAt` is
+ *    known -- `Date.now()` minus that.
+ *  Merges rather than replaces the existing result, since a test case's full lifecycle is two
+ *  calls (an implicit "running" transition, then a terminal call from `record-result`) and the
+ *  second must not erase what the first captured. */
+export async function recordResult(path, testCaseId, update) {
+    const testRun = await readTestRun(path);
+    const existing = testRun.results[testCaseId];
+    if (!existing) {
+        throw new Error(`TestRun at ${path} has no entry for testCaseId "${testCaseId}"`);
+    }
+    const startedAt = existing.startedAt ?? (update.status === "running" ? new Date().toISOString() : undefined);
+    const sequence = existing.sequence ??
+        (update.status === "running"
+            ? 1 + Math.max(0, ...Object.values(testRun.results).map((r) => r.sequence ?? 0))
+            : undefined);
+    const inputs = update.inputs ?? existing.inputs;
+    const tokensUsed = update.tokensUsed ?? existing.tokensUsed;
+    const costUsd = update.costUsd ?? existing.costUsd;
+    const messages = update.messages ?? existing.messages;
+    const durationMs = TERMINAL_STATUSES.has(update.status) && startedAt !== undefined
+        ? Date.now() - new Date(startedAt).getTime()
+        : existing.durationMs;
+    testRun.results[testCaseId] = {
+        status: update.status,
+        outputs: update.outputs ?? existing.outputs,
+        assertionResults: update.assertionResults ?? existing.assertionResults,
+        ...(sequence !== undefined ? { sequence } : {}),
+        ...(inputs !== undefined ? { inputs } : {}),
+        ...(startedAt !== undefined ? { startedAt } : {}),
+        ...(durationMs !== undefined ? { durationMs } : {}),
+        ...(tokensUsed !== undefined ? { tokensUsed } : {}),
+        ...(costUsd !== undefined ? { costUsd } : {}),
+        ...(messages !== undefined ? { messages } : {}),
+    };
+    await writeTestRun(testRun, path);
+    return testRun;
+}