trxa-tool 0.0.0-stage → 0.2.0

package/dist/suite-runner.js

@@ -0,0 +1,204 @@
+import { TERMINAL_STATUSES } from "./report-schema.js";
+function entryById(suite, testCaseId) {
+    const entry = suite.entries.find((e) => e.testCaseId === testCaseId);
+    if (!entry)
+        throw new Error(`Suite "${suite.id}" has no entry for testCaseId "${testCaseId}"`);
+    return entry;
+}
+/**
+ * Kahn's algorithm over `dependsOn` edges. Throws on an unknown dependsOn id or a cycle --
+ * these must never be left to LLM judgment, unlike everything in mapping.ts's hints.
+ */
+export function topologicalOrder(suite) {
+    const ids = new Set(suite.entries.map((e) => e.testCaseId));
+    for (const entry of suite.entries) {
+        for (const dep of entry.dependsOn) {
+            if (!ids.has(dep)) {
+                throw new Error(`Suite "${suite.id}": entry "${entry.testCaseId}" depends on unknown testCaseId "${dep}"`);
+            }
+        }
+        // A runIf's testCaseId must be a direct dependency: nextStep only evaluates runIf once every
+        // dependsOn has passed, so anything else would either be unresolved (not yet run) or -- if it
+        // were allowed to be a transitive-but-not-direct ancestor -- silently rely on ordering the
+        // schema doesn't actually guarantee here.
+        if (entry.runIf && !entry.dependsOn.includes(entry.runIf.testCaseId)) {
+            throw new Error(`Suite "${suite.id}": entry "${entry.testCaseId}"'s runIf references "${entry.runIf.testCaseId}", ` +
+                `which must be one of its own dependsOn`);
+        }
+    }
+    const inDegree = new Map();
+    for (const entry of suite.entries)
+        inDegree.set(entry.testCaseId, entry.dependsOn.length);
+    const dependents = new Map();
+    for (const entry of suite.entries) {
+        for (const dep of entry.dependsOn) {
+            dependents.set(dep, [...(dependents.get(dep) ?? []), entry.testCaseId]);
+        }
+    }
+    const queue = suite.entries.filter((e) => e.dependsOn.length === 0).map((e) => e.testCaseId);
+    const order = [];
+    while (queue.length > 0) {
+        const id = queue.shift();
+        order.push(id);
+        for (const dependent of dependents.get(id) ?? []) {
+            const remaining = inDegree.get(dependent) - 1;
+            inDegree.set(dependent, remaining);
+            if (remaining === 0)
+                queue.push(dependent);
+        }
+    }
+    if (order.length !== suite.entries.length) {
+        const stuck = suite.entries.map((e) => e.testCaseId).filter((id) => !order.includes(id));
+        throw new Error(`Suite "${suite.id}" has a dependency cycle involving: ${stuck.join(", ")}`);
+    }
+    return order;
+}
+/** Resolves a test case's declared `inputs` to concrete values for one suite run, following
+ *  each `inputBindings` entry to a fixture, an earlier test case's captured output, or a
+ *  literal. Throws a clear, specific error rather than silently defaulting -- a missing or
+ *  not-yet-produced value should stop the run, not get guessed by the LLM. */
+export function resolveInputs(entry, testCase, testRun, fixtures) {
+    const resolved = {};
+    for (const param of testCase.inputs) {
+        const binding = entry.inputBindings[param.name];
+        if (!binding) {
+            if (param.required) {
+                throw new Error(`Test case "${testCase.id}": no inputBindings entry for required param "${param.name}"`);
+            }
+            continue;
+        }
+        if (binding.kind === "fixture") {
+            // Ignoring case: a fixture's name is also its file name, so two names differing only in
+            // case can't coexist on Windows and are the same fixture.
+            const fixture = fixtures.find((f) => f.name.toLowerCase() === binding.fixtureName.toLowerCase());
+            if (!fixture) {
+                throw new Error(`Test case "${testCase.id}", param "${param.name}": no fixture named "${binding.fixtureName}" found`);
+            }
+            const values = fixture.testCases[testCase.id];
+            if (!values) {
+                throw new Error(`Test case "${testCase.id}", param "${param.name}": fixture "${binding.fixtureName}" has no values for this test case`);
+            }
+            const value = values[binding.param];
+            if (value === undefined) {
+                throw new Error(`Test case "${testCase.id}", param "${param.name}": fixture "${binding.fixtureName}" has no value for "${binding.param}"`);
+            }
+            resolved[param.name] = value;
+        }
+        else if (binding.kind === "testCaseOutput") {
+            const upstream = testRun.results[binding.testCaseId];
+            if (!upstream || upstream.status !== "passed") {
+                throw new Error(`Test case "${testCase.id}", param "${param.name}": upstream test case "${binding.testCaseId}" ` +
+                    `has not passed yet (status: ${upstream?.status ?? "unknown"}) -- its output isn't available`);
+            }
+            const value = upstream.outputs[binding.outputName];
+            if (value === undefined) {
+                throw new Error(`Test case "${testCase.id}", param "${param.name}": upstream test case "${binding.testCaseId}" ` +
+                    `did not capture an output named "${binding.outputName}"`);
+            }
+            resolved[param.name] = value;
+        }
+        else if (binding.kind === "literal") {
+            resolved[param.name] = binding.value;
+        }
+        else {
+            // "autogenerated" -- deliberately left out of `resolved`. There's no deterministic value to
+            // produce here; the orchestrating LLM invents one at replay time (see cmdResolveNext in
+            // index.ts, which diffs testCase.inputs against this function's return value to find the
+            // gap and surface it as `autogeneratedParams`).
+        }
+    }
+    return resolved;
+}
+function compareValues(comparator, actual, expected) {
+    switch (comparator) {
+        case "equals":
+            return actual === expected;
+        case "notEquals":
+            return actual !== expected;
+        case "contains":
+            return actual.includes(expected);
+        case "greaterThan":
+            return Number(actual) > Number(expected);
+        case "lessThan":
+            return Number(actual) < Number(expected);
+    }
+}
+/** Evaluates a `runIf` gate against the run so far. `condition.testCaseId` is guaranteed (by
+ *  `topologicalOrder`'s validation) to be one of the entry's own `dependsOn`, so by the time this
+ *  is called that upstream test case has already passed -- this only ever reads its captured
+ *  *output*, never its status (see `RunCondition`'s doc comment in `suite-schema.ts` for why a
+ *  status check would be vacuous here). An upstream that didn't capture the referenced output at
+ *  all evaluates to false (skip), the same "missing means doesn't match" behavior a real value
+ *  comparison would need anyway. */
+export function evaluateRunCondition(condition, testRun) {
+    const actual = testRun.results[condition.testCaseId]?.outputs[condition.outputName];
+    if (actual === undefined)
+        return false;
+    return compareValues(condition.comparator, actual, condition.value);
+}
+function isRunnable(entry, testRun) {
+    return entry.dependsOn.every((dep) => testRun.results[dep]?.status === "passed");
+}
+/**
+ * Computes the next action for a suite run given its current TestRun:
+ * - If any pending entry now has a failed/blocked/skipped dependency, report it as newly-blocked
+ *   so the caller can record that status before asking again (propagation is explicit, not
+ *   silent) -- a skipped dependency counts the same as a failed one here, since nothing downstream
+ *   of it can ever satisfy "every dependency passed" otherwise, which would leave it stuck waiting
+ *   forever instead of reaching a terminal status.
+ * - Otherwise, if any pending-and-otherwise-runnable entry's `runIf` evaluates false, report it as
+ *   newly-skipped the same way, before considering anything runnable.
+ * - Otherwise return every pending entry (topological order, for determinism) that's runnable and
+ *   (if it has one) passes its `runIf` -- usually just one, but a suite whose DAG has two or more
+ *   entries that share a satisfied dependency without depending on each other (e.g. two entries
+ *   that both only depend on a common upstream entry) surfaces all of them at once here. `nextStep`
+ *   itself makes no scheduling decision beyond exposing what's currently unblocked -- whether the
+ *   caller works through `testCaseIds` one at a time or hands them to concurrent workers is
+ *   entirely the caller's call; nothing here assumes either.
+ * - Otherwise, if every entry has a terminal status, the suite is complete.
+ * - Otherwise ("waiting"): nothing is pending-and-runnable right now, but something is still
+ *   `"running"` -- a normal mid-flight state, not an error. A root entry (no dependencies) is
+ *   always immediately runnable once pending, so this can only happen when every not-yet-terminal
+ *   entry is already `"running"`.
+ */
+export function nextStep(suite, testRun) {
+    const order = topologicalOrder(suite);
+    const newlyBlocked = order.filter((id) => {
+        const result = testRun.results[id];
+        if (!result || result.status !== "pending")
+            return false;
+        const entry = entryById(suite, id);
+        return entry.dependsOn.some((dep) => {
+            const depStatus = testRun.results[dep]?.status;
+            return depStatus === "failed" || depStatus === "blocked" || depStatus === "skipped";
+        });
+    });
+    if (newlyBlocked.length > 0)
+        return { status: "newly-blocked", blockedIds: newlyBlocked };
+    const newlySkipped = order.filter((id) => {
+        const result = testRun.results[id];
+        if (!result || result.status !== "pending")
+            return false;
+        const entry = entryById(suite, id);
+        if (!isRunnable(entry, testRun))
+            return false;
+        return entry.runIf !== undefined && !evaluateRunCondition(entry.runIf, testRun);
+    });
+    if (newlySkipped.length > 0)
+        return { status: "newly-skipped", skippedIds: newlySkipped };
+    const runnable = order.filter((id) => {
+        const result = testRun.results[id];
+        if (!result || result.status !== "pending")
+            return false;
+        const entry = entryById(suite, id);
+        if (!isRunnable(entry, testRun))
+            return false;
+        return entry.runIf === undefined || evaluateRunCondition(entry.runIf, testRun);
+    });
+    if (runnable.length > 0)
+        return { status: "runnable", testCaseIds: runnable };
+    const allTerminal = order.every((id) => TERMINAL_STATUSES.has(testRun.results[id]?.status ?? ""));
+    if (allTerminal)
+        return { status: "complete" };
+    return { status: "waiting" };
+}

package/dist/suite-schema.d.ts

@@ -0,0 +1,59 @@
+/** A TestSuite is a DAG of test cases: each entry declares which earlier entries must pass
+ *  first (`dependsOn`) and where each of its declared inputs comes from (`inputBindings`) --
+ *  a named fixture, an earlier test case's captured output, a literal value, or (deliberately no
+ *  extra fields) `autogenerated`, which leaves the value for the orchestrating LLM to invent at
+ *  replay time using the param's own `description`/`example` as guidance -- see
+ *  `resolveInputs` (`suite-runner.ts`) and `cmdResolveNext` (`index.ts`) for how the gap is
+ *  surfaced rather than resolved here. */
+import type { Comparator } from "./testcase-schema.js";
+export type InputBinding = {
+    kind: "fixture";
+    fixtureName: string;
+    param: string;
+} | {
+    kind: "testCaseOutput";
+    testCaseId: string;
+    outputName: string;
+} | {
+    kind: "literal";
+    value: string;
+} | {
+    kind: "autogenerated";
+};
+/** Gates whether an entry runs at all, evaluated once its `dependsOn` have already passed --
+ *  deliberately checked against a captured *output*, not status: by the time this is evaluated,
+ *  `testCaseId` (which must be one of this entry's own `dependsOn`, enforced by
+ *  `topologicalOrder`) is already guaranteed `"passed"`, so a status-based condition would always
+ *  be vacuously true. Comparator semantics match `Expectation`'s (`testcase-schema.ts`). If the
+ *  condition evaluates false, `nextStep` marks the entry `"skipped"` instead of running it -- see
+ *  `suite-runner.ts`. No general branching/looping is supported; this is intentionally the
+ *  smallest useful primitive (see ENHANCEMENTS.md's Area 7 for the scoping discussion). */
+export interface RunCondition {
+    testCaseId: string;
+    outputName: string;
+    comparator: Comparator;
+    value: string;
+}
+export interface TestSuiteEntry {
+    testCaseId: string;
+    dependsOn: string[];
+    inputBindings: Record<string, InputBinding>;
+    /** Company (`companyId`) to pass on this entry's test case's *first* navigation call only --
+     *  subsequent calls in the same test case inherit whatever company that navigation opened, and
+     *  any mid-flow company jump (e.g. an intercompany order button) is picked up from the live
+     *  `FormState.Company` rather than re-specified here. Omit for single-company suites. */
+    legalEntity?: string;
+    /** Optional conditional-skip gate -- see `RunCondition` above. Omit to always run once
+     *  dependencies pass (the previous, only behavior). */
+    runIf?: RunCondition;
+}
+export interface TestSuite {
+    id: string;
+    name: string;
+    description: string;
+    entries: TestSuiteEntry[];
+}
+/** Validates a parsed JSON object against the TestSuite shape. Does not check for
+ *  cycles or unknown dependsOn ids -- that's suite-runner.ts's job, since it needs the full set
+ *  of entries to reason about the graph. */
+export declare function validateTestSuite(value: unknown): TestSuite;

package/dist/suite-schema.js

@@ -0,0 +1,95 @@
+/** A TestSuite is a DAG of test cases: each entry declares which earlier entries must pass
+ *  first (`dependsOn`) and where each of its declared inputs comes from (`inputBindings`) --
+ *  a named fixture, an earlier test case's captured output, a literal value, or (deliberately no
+ *  extra fields) `autogenerated`, which leaves the value for the orchestrating LLM to invent at
+ *  replay time using the param's own `description`/`example` as guidance -- see
+ *  `resolveInputs` (`suite-runner.ts`) and `cmdResolveNext` (`index.ts`) for how the gap is
+ *  surfaced rather than resolved here. */
+function fail(field, reason) {
+    throw new Error(`Invalid TestSuite: ${field} ${reason}`);
+}
+function assertString(value, field) {
+    if (typeof value !== "string" || value.length === 0)
+        fail(field, "must be a non-empty string");
+}
+function validateInputBinding(value, field) {
+    if (typeof value !== "object" || value === null)
+        fail(field, "must be an object");
+    const v = value;
+    if (v.kind === "fixture") {
+        assertString(v.fixtureName, `${field}.fixtureName`);
+        assertString(v.param, `${field}.param`);
+    }
+    else if (v.kind === "testCaseOutput") {
+        assertString(v.testCaseId, `${field}.testCaseId`);
+        assertString(v.outputName, `${field}.outputName`);
+    }
+    else if (v.kind === "literal") {
+        assertString(v.value, `${field}.value`);
+    }
+    else if (v.kind === "autogenerated") {
+        // no extra fields
+    }
+    else {
+        fail(`${field}.kind`, 'must be "fixture", "testCaseOutput", "literal", or "autogenerated"');
+    }
+    return v;
+}
+const VALID_COMPARATORS = ["equals", "notEquals", "contains", "greaterThan", "lessThan"];
+function validateRunCondition(value, field) {
+    if (typeof value !== "object" || value === null)
+        fail(field, "must be an object");
+    const v = value;
+    assertString(v.testCaseId, `${field}.testCaseId`);
+    assertString(v.outputName, `${field}.outputName`);
+    if (!VALID_COMPARATORS.includes(v.comparator)) {
+        fail(`${field}.comparator`, `must be one of ${VALID_COMPARATORS.join(", ")}`);
+    }
+    assertString(v.value, `${field}.value`);
+    return v;
+}
+function validateTestSuiteEntry(value, index) {
+    if (typeof value !== "object" || value === null)
+        fail(`entries[${index}]`, "must be an object");
+    const v = value;
+    assertString(v.testCaseId, `entries[${index}].testCaseId`);
+    if (!Array.isArray(v.dependsOn) || !v.dependsOn.every((d) => typeof d === "string")) {
+        fail(`entries[${index}].dependsOn`, "must be an array of testCaseId strings");
+    }
+    if (typeof v.inputBindings !== "object" || v.inputBindings === null || Array.isArray(v.inputBindings)) {
+        fail(`entries[${index}].inputBindings`, "must be an object mapping param names to bindings");
+    }
+    const inputBindings = {};
+    for (const [param, binding] of Object.entries(v.inputBindings)) {
+        inputBindings[param] = validateInputBinding(binding, `entries[${index}].inputBindings.${param}`);
+    }
+    if (v.legalEntity !== undefined)
+        assertString(v.legalEntity, `entries[${index}].legalEntity`);
+    const runIf = v.runIf !== undefined ? validateRunCondition(v.runIf, `entries[${index}].runIf`) : undefined;
+    return {
+        testCaseId: v.testCaseId,
+        dependsOn: v.dependsOn,
+        inputBindings,
+        ...(v.legalEntity !== undefined ? { legalEntity: v.legalEntity } : {}),
+        ...(runIf !== undefined ? { runIf } : {}),
+    };
+}
+/** Validates a parsed JSON object against the TestSuite shape. Does not check for
+ *  cycles or unknown dependsOn ids -- that's suite-runner.ts's job, since it needs the full set
+ *  of entries to reason about the graph. */
+export function validateTestSuite(value) {
+    if (typeof value !== "object" || value === null)
+        fail("suite", "must be an object");
+    const v = value;
+    assertString(v.id, "id");
+    assertString(v.name, "name");
+    assertString(v.description, "description");
+    if (!Array.isArray(v.entries))
+        fail("entries", "must be an array");
+    return {
+        id: v.id,
+        name: v.name,
+        description: v.description,
+        entries: v.entries.map(validateTestSuiteEntry),
+    };
+}

package/dist/testcase-schema.d.ts

@@ -0,0 +1,102 @@
+/**
+ * A TestCase -- the atomic pass/fail unit -- wraps a testscript.json (the mechanics) with the
+ * layer that was missing: named inputs bound to specific steps/controls, outputs a later test
+ * case can consume, and expectations to check -- so the LLM never has to guess business data or
+ * judge pass/fail from vibes.
+ */
+export type BindTarget = {
+    kind: "control";
+    stepSequence: number;
+    controlName: string;
+} | {
+    kind: "grid-row-selector";
+    stepSequence: number;
+    note: string;
+};
+export interface InputParamSpec {
+    name: string;
+    description: string;
+    /** Recorded value from the source .axtr, if suggest-inputs drafted this from a PropertyUserAction
+     *  or a grid-filter step's recorded gridColumnValue. */
+    example?: string;
+    /** The control's human-readable D365FO label (e.g. "Quantity"), if suggest-inputs drafted this
+     *  from a PropertyUserAction whose recorded ControlLabel was non-empty. Distinct from `name`,
+     *  which is the internal control identifier (e.g. "PurchLine_PurchQtyGrid"). */
+    controlLabel?: string;
+    required: boolean;
+    bindTo: BindTarget;
+}
+/**
+ * `filter` (and `Expectation`'s `filter`/`expectedValue` below) may contain `{{paramName}}`
+ * placeholders referencing this test case's resolved `inputs` (e.g. `CustomerAccount eq
+ * '{{customerAccount}}'`). This is deliberately plain string substitution the orchestrating LLM
+ * performs before calling `data_find_entities` -- not a templating engine in this codebase --
+ * consistent with trxa-tool computing only what must not be left to LLM judgment (DAG order,
+ * binding resolution) and leaving simple substitution to the orchestrator.
+ */
+/**
+ * `uiField` needs at least one of `controlName`/`hint`:
+ * - `controlName` — the exact AOT control name to read off `FormState` at `afterStepSequence`.
+ *   Free (no extra tool call -- the value is already in the response of a step being replayed
+ *   anyway) and exactly reproducible. Prefer this whenever the control name is known.
+ * - `hint` — plain-English description of the value to find (e.g. "the newly assigned purchase
+ *   order number"), for when the control name isn't known at authoring time -- common for values
+ *   Task Recorder never captured because nobody clicked them (an auto-assigned ID shown after
+ *   Save, say). The replaying LLM resolves this against the live `FormState` and is expected to
+ *   write the resolved `controlName` back into this file, upgrading the hint to an exact match so
+ *   future replays don't need to re-interpret `FormState` every time.
+ */
+export type OutputCaptureSpec = {
+    name: string;
+    kind: "odata";
+    entitySet: string;
+    filter: string;
+    field: string;
+} | {
+    name: string;
+    kind: "uiField";
+    afterStepSequence: number;
+    controlName?: string;
+    hint?: string;
+};
+export type Comparator = "equals" | "notEquals" | "contains" | "greaterThan" | "lessThan";
+export type Expectation = {
+    description: string;
+    checkAfterStepSequence: number;
+    kind: "odata";
+    entitySet: string;
+    filter: string;
+    expectedField: string;
+    expectedValue: string;
+    comparator: Comparator;
+} | {
+    description: string;
+    checkAfterStepSequence: number;
+    kind: "uiMessage";
+    severity: "Info" | "Warning" | "Error";
+    /** Plain substring match against a message's text (not a regex) -- checked against
+     *  `FormState.Messages` at replay time. Empty string means "any message of this severity",
+     *  e.g. `{ severity: "Error", pattern: "", shouldExist: false }` asserts no error occurred
+     *  at all, regardless of its text. */
+    pattern: string;
+    /** Whether a message matching `severity`+`pattern` should be present (`true`) or absent
+     *  (`false`) after the step -- not about whether a record/order exists. */
+    shouldExist: boolean;
+};
+export interface TestCase {
+    id: string;
+    name: string;
+    description: string;
+    /** Path to the testscript.json this test case replays, relative to the test case file. */
+    testScriptPath: string;
+    /** The source recording's folder under `recordings/` (e.g. "Accounts Payable/Purchase Orders"),
+     *  captured at creation time so Studio's Test Cases list can group by it. Absent for a
+     *  top-level recording or a test case created before this field existed -- both render under
+     *  an "Ungrouped" bucket rather than being backfilled. */
+    sourceRecordingFolder?: string;
+    inputs: InputParamSpec[];
+    outputs: OutputCaptureSpec[];
+    expectations: Expectation[];
+}
+/** Validates a parsed JSON object against the TestCase shape, throwing a descriptive error on the first problem found. */
+export declare function validateTestCase(value: unknown): TestCase;

package/dist/testcase-schema.js

@@ -0,0 +1,132 @@
+/**
+ * A TestCase -- the atomic pass/fail unit -- wraps a testscript.json (the mechanics) with the
+ * layer that was missing: named inputs bound to specific steps/controls, outputs a later test
+ * case can consume, and expectations to check -- so the LLM never has to guess business data or
+ * judge pass/fail from vibes.
+ */
+function fail(field, reason) {
+    throw new Error(`Invalid TestCase: ${field} ${reason}`);
+}
+function assertString(value, field) {
+    if (typeof value !== "string" || value.length === 0)
+        fail(field, "must be a non-empty string");
+}
+function assertBindTarget(value, field) {
+    if (typeof value !== "object" || value === null)
+        fail(field, "must be an object");
+    const v = value;
+    if (typeof v.stepSequence !== "number")
+        fail(`${field}.stepSequence`, "must be a number");
+    if (v.kind === "control") {
+        assertString(v.controlName, `${field}.controlName`);
+    }
+    else if (v.kind === "grid-row-selector") {
+        assertString(v.note, `${field}.note`);
+    }
+    else {
+        fail(`${field}.kind`, 'must be "control" or "grid-row-selector"');
+    }
+}
+function validateInputParamSpec(value, index) {
+    if (typeof value !== "object" || value === null)
+        fail(`inputs[${index}]`, "must be an object");
+    const v = value;
+    assertString(v.name, `inputs[${index}].name`);
+    assertString(v.description, `inputs[${index}].description`);
+    if (typeof v.required !== "boolean")
+        fail(`inputs[${index}].required`, "must be a boolean");
+    assertBindTarget(v.bindTo, `inputs[${index}].bindTo`);
+    if (v.example !== undefined)
+        assertString(v.example, `inputs[${index}].example`);
+    if (v.controlLabel !== undefined)
+        assertString(v.controlLabel, `inputs[${index}].controlLabel`);
+    return v;
+}
+function validateOutputCaptureSpec(value, index) {
+    if (typeof value !== "object" || value === null)
+        fail(`outputs[${index}]`, "must be an object");
+    const v = value;
+    assertString(v.name, `outputs[${index}].name`);
+    if (v.kind === "odata") {
+        assertString(v.entitySet, `outputs[${index}].entitySet`);
+        assertString(v.filter, `outputs[${index}].filter`);
+        assertString(v.field, `outputs[${index}].field`);
+    }
+    else if (v.kind === "uiField") {
+        if (typeof v.afterStepSequence !== "number") {
+            fail(`outputs[${index}].afterStepSequence`, "must be a number");
+        }
+        if (v.controlName !== undefined && typeof v.controlName !== "string") {
+            fail(`outputs[${index}].controlName`, "must be a string");
+        }
+        if (v.hint !== undefined && typeof v.hint !== "string") {
+            fail(`outputs[${index}].hint`, "must be a string");
+        }
+        if (!v.controlName && !v.hint) {
+            fail(`outputs[${index}]`, "must have a non-empty controlName or hint");
+        }
+    }
+    else {
+        fail(`outputs[${index}].kind`, 'must be "odata" or "uiField"');
+    }
+    return v;
+}
+function validateExpectation(value, index) {
+    if (typeof value !== "object" || value === null) {
+        fail(`expectations[${index}]`, "must be an object");
+    }
+    const v = value;
+    assertString(v.description, `expectations[${index}].description`);
+    if (typeof v.checkAfterStepSequence !== "number") {
+        fail(`expectations[${index}].checkAfterStepSequence`, "must be a number");
+    }
+    if (v.kind === "odata") {
+        assertString(v.entitySet, `expectations[${index}].entitySet`);
+        assertString(v.filter, `expectations[${index}].filter`);
+        assertString(v.expectedField, `expectations[${index}].expectedField`);
+        assertString(v.expectedValue, `expectations[${index}].expectedValue`);
+        assertString(v.comparator, `expectations[${index}].comparator`);
+    }
+    else if (v.kind === "uiMessage") {
+        assertString(v.severity, `expectations[${index}].severity`);
+        // Empty string is valid here (means "any message of this severity") -- assertString would
+        // reject it, since every other string field in this schema treats empty as missing.
+        if (typeof v.pattern !== "string")
+            fail(`expectations[${index}].pattern`, "must be a string");
+        if (typeof v.shouldExist !== "boolean") {
+            fail(`expectations[${index}].shouldExist`, "must be a boolean");
+        }
+    }
+    else {
+        fail(`expectations[${index}].kind`, 'must be "odata" or "uiMessage"');
+    }
+    return v;
+}
+/** Validates a parsed JSON object against the TestCase shape, throwing a descriptive error on the first problem found. */
+export function validateTestCase(value) {
+    if (typeof value !== "object" || value === null)
+        fail("testcase", "must be an object");
+    const v = value;
+    assertString(v.id, "id");
+    assertString(v.name, "name");
+    assertString(v.description, "description");
+    assertString(v.testScriptPath, "testScriptPath");
+    if (v.sourceRecordingFolder !== undefined)
+        assertString(v.sourceRecordingFolder, "sourceRecordingFolder");
+    if (!Array.isArray(v.inputs))
+        fail("inputs", "must be an array");
+    if (!Array.isArray(v.outputs))
+        fail("outputs", "must be an array");
+    if (!Array.isArray(v.expectations))
+        fail("expectations", "must be an array");
+    return {
+        id: v.id,
+        name: v.name,
+        description: v.description,
+        testScriptPath: v.testScriptPath,
+        ...(v.sourceRecordingFolder !== undefined ? { sourceRecordingFolder: v.sourceRecordingFolder } : {}),
+        inputs: v.inputs.map(validateInputParamSpec),
+        outputs: v.outputs.map(validateOutputCaptureSpec),
+        expectations: v.expectations.map(validateExpectation),
+    };
+}

package/dist/token-usage.d.ts

@@ -0,0 +1,38 @@
+/** Measures real per-turn token usage from Claude Code's own session transcript, instead of
+ *  relying on the orchestrating LLM to self-report a guess (`record-result --tokens` used to be
+ *  guesswork with no ground truth -- see report-schema.ts). Claude Code logs a `usage` object
+ *  (`input_tokens`/`output_tokens`/`cache_creation_input_tokens`/`cache_read_input_tokens`) on
+ *  every assistant turn to `~/.claude/projects/<project>/<session>.jsonl` -- the same data
+ *  community tools like ccusage read to produce cost reports.
+ *
+ *  CONFIRMED UNRELIABLE with concurrent sessions (live test, 2026-08-20): running one headless
+ *  suite replay alongside one sibling session sharing this project directory inflated the
+ *  measured total by +24%; running it alongside two concurrent siblings inflated it by +98% (the
+ *  reported number was nearly double the true one, cross-checked against each headless
+ *  invocation's own authoritative `total_cost_usd`/`usage`). This is not a rare edge case -- it's
+ *  close to the default way headless mode gets used (something running in the background while
+ *  other work continues, or multiple suites run in parallel, exactly what triggered it here).
+ *  Treat `tokensUsed` as a rough order-of-magnitude signal, not a trustworthy number, whenever more
+ *  than one Claude Code session might have touched this project directory during the measurement
+ *  window. See `findLatestTranscript`'s own comment for why there's currently no clean fix. */
+export declare function defaultClaudeProjectsRoot(): string;
+/** Recursively finds every `.jsonl` file under `root`, returning the most recently modified one.
+ *  Claude Code appends to the active session's transcript on every turn, so "most recently
+ *  modified across ~/.claude/projects" is a *rough* stand-in for "this session's own transcript" --
+ *  but it's a race, not a guarantee: any sibling session that happens to write a turn more recently
+ *  wins, and a fast-writing sibling (e.g. another suite replaying in parallel, one turn every few
+ *  seconds) wins that race constantly. Live-tested 2026-08-20: this measurably picked up unrelated
+ *  sessions' usage, inflating results by 24%-98% depending on how much sibling activity overlapped
+ *  the measurement window (see token-usage.ts's top-of-file comment for the numbers). There is
+ *  currently no clean fix available here -- Claude Code doesn't expose a session/transcript
+ *  identifier to Bash-tool subprocesses (open upstream feature requests, e.g.
+ *  anthropics/claude-code#18629, unresolved as of this writing), so "newest file" is the best
+ *  available signal without that, not a reliable one. Not worth chasing a fancier heuristic here
+ *  until that's exposed -- a heuristic can narrow the odds of picking wrong, never eliminate them,
+ *  and the honest fix is a real identifier, not more machinery layered on a guess. */
+export declare function findLatestTranscript(root: string): string | null;
+/** Sums real token usage for every assistant turn in `transcriptPath` whose `timestamp` falls
+ *  within [`sinceIso`, `untilIso`]. Still an approximation -- it attributes tokens by wall-clock
+ *  overlap with a test case's execution window, not a verified causal link -- but it's a measured
+ *  number from Claude Code's own log, not a self-reported guess. */
+export declare function measureTokens(transcriptPath: string, sinceIso: string, untilIso: string): number;