prose-qa 0.1.0

package/dist/agent/verdict-retry-prompt.test.js

@@ -0,0 +1,25 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { buildVerdictRetryPrompt } from "./verdict-retry-prompt.js";
+const baseScenario = {
+    filePath: "scenarios/test.md",
+    frontmatter: { name: "test" },
+    skills: [],
+    goal: "Test goal",
+    steps: "1. Do thing",
+    then: ['url contains "/home"', 'page shows "Welcome"'],
+    rawCheckpoints: [],
+    checkpoints: [],
+};
+describe("buildVerdictRetryPrompt", () => {
+    it("lists every Then checkpoint and requires JSON verdict", () => {
+        const prompt = buildVerdictRetryPrompt(baseScenario);
+        assert.match(prompt, /## Verdict required/);
+        assert.match(prompt, /2 total/);
+        assert.match(prompt, /\*\*url contains "\/home"\*\*/);
+        assert.match(prompt, /\*\*page shows "Welcome"\*\*/);
+        assert.match(prompt, /fenced ```json block/i);
+        assert.match(prompt, /Do \*\*not\*\* use bash or read tools/);
+    });
+});
+//# sourceMappingURL=verdict-retry-prompt.test.js.map

package/dist/agent/verdict-retry-prompt.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict-retry-prompt.test.js","sourceRoot":"","sources":["../../src/agent/verdict-retry-prompt.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,YAAY,GAAa;IAC7B,QAAQ,EAAE,mBAAmB;IAC7B,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;IAC7B,MAAM,EAAE,EAAE;IACV,IAAI,EAAE,WAAW;IACjB,KAAK,EAAE,aAAa;IACpB,IAAI,EAAE,CAAC,sBAAsB,EAAE,sBAAsB,CAAC;IACtD,cAAc,EAAE,EAAE;IAClB,WAAW,EAAE,EAAE;CAChB,CAAC;AAEF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,MAAM,GAAG,uBAAuB,CAAC,YAAY,CAAC,CAAC;QAErD,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;QACtD,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,8BAA8B,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,uCAAuC,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/agent/verdict.d.ts

@@ -0,0 +1,31 @@
+import type { ModelMessage } from "ai";
+import { type AgentTranscript, type BashEntry, type TranscriptBashEntry, type TranscriptMessageEntry, type Verdict } from "../types/verdict.js";
+/** Drop the last assistant turn (and trailing tool results) so the model can redo that completion. */
+export declare function stripLastAssistantTurn(messages: ModelMessage[]): ModelMessage[];
+export declare function extractVerdict(text: string): Verdict | null;
+export declare function getTranscriptMessages(transcript: AgentTranscript): TranscriptMessageEntry[];
+export declare function getTranscriptBashEntries(transcript: AgentTranscript): TranscriptBashEntry[];
+export interface TranscriptEntryTiming {
+    at?: Date;
+    durationMs?: number;
+}
+export declare function appendTranscriptMessage(transcript: AgentTranscript, role: string, content: string, thinking?: string, timing?: TranscriptEntryTiming): void;
+export declare function appendTranscriptBash(transcript: AgentTranscript, entry: BashEntry, timing?: Pick<TranscriptEntryTiming, "at">): void;
+/** Append final model text when it is not already the last assistant message. */
+export declare function appendFinalTextToTranscript(transcript: AgentTranscript, finalText: string, timing?: TranscriptEntryTiming): void;
+export interface StepTranscriptInput {
+    text: string;
+    reasoningText?: string;
+    toolCalls: Array<{
+        toolName: string;
+        input: unknown;
+    }>;
+}
+export interface StepTranscriptFormatted {
+    content: string | null;
+    thinking?: string;
+}
+/** Format one LLM step for the transcript (content, thinking, and tool calls). */
+export declare function formatStepForTranscript(step: StepTranscriptInput): StepTranscriptFormatted;
+export declare function appendStepToTranscript(transcript: AgentTranscript, step: StepTranscriptInput, bashEntries?: BashEntry[], formattedOverride?: StepTranscriptFormatted, timing?: TranscriptEntryTiming): boolean;
+//# sourceMappingURL=verdict.d.ts.map

package/dist/agent/verdict.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/agent/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AACvC,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,SAAS,EACd,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,OAAO,EACb,MAAM,qBAAqB,CAAC;AAE7B,sGAAsG;AACtG,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CAS/E;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAmC3D;AAED,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,eAAe,GAC1B,sBAAsB,EAAE,CAI1B;AAED,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,eAAe,GAC1B,mBAAmB,EAAE,CAIvB;AAaD,MAAM,WAAW,qBAAqB;IACpC,EAAE,CAAC,EAAE,IAAI,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,eAAe,EAC3B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,qBAAqB,GAC7B,IAAI,CAUN;AAED,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,eAAe,EAC3B,KAAK,EAAE,SAAS,EAChB,MAAM,CAAC,EAAE,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,GACzC,IAAI,CAMN;AAED,iFAAiF;AACjF,wBAAgB,2BAA2B,CACzC,UAAU,EAAE,eAAe,EAC3B,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,qBAAqB,GAC7B,IAAI,CAKN;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,kFAAkF;AAClF,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,mBAAmB,GACxB,uBAAuB,CAgBzB;AAED,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,eAAe,EAC3B,IAAI,EAAE,mBAAmB,EACzB,WAAW,GAAE,SAAS,EAAO,EAC7B,iBAAiB,CAAC,EAAE,uBAAuB,EAC3C,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAmBT"}

package/dist/agent/verdict.js

@@ -0,0 +1,123 @@
+import { VerdictSchema, } from "../types/verdict.js";
+/** Drop the last assistant turn (and trailing tool results) so the model can redo that completion. */
+export function stripLastAssistantTurn(messages) {
+    let end = messages.length;
+    while (end > 0 && messages[end - 1]?.role === "tool") {
+        end -= 1;
+    }
+    if (end > 0 && messages[end - 1]?.role === "assistant") {
+        return messages.slice(0, end - 1);
+    }
+    return messages;
+}
+export function extractVerdict(text) {
+    const jsonBlock = /```(?:json)?\s*([\s\S]*?)```/g;
+    let match;
+    const candidates = [];
+    while ((match = jsonBlock.exec(text)) !== null) {
+        candidates.push(match[1].trim());
+    }
+    // Also try parsing the whole text as JSON
+    candidates.push(text.trim());
+    for (const candidate of candidates.reverse()) {
+        try {
+            const parsed = JSON.parse(candidate);
+            const result = VerdictSchema.safeParse(parsed);
+            if (result.success)
+                return result.data;
+        }
+        catch {
+            // continue
+        }
+    }
+    // Try to find inline JSON object with status field
+    const inline = /\{[\s\S]*"status"\s*:\s*"(?:pass|fail)"[\s\S]*\}/.exec(text);
+    if (inline) {
+        try {
+            const parsed = JSON.parse(inline[0]);
+            const result = VerdictSchema.safeParse(parsed);
+            if (result.success)
+                return result.data;
+        }
+        catch {
+            // continue
+        }
+    }
+    return null;
+}
+export function getTranscriptMessages(transcript) {
+    return transcript.entries.filter((entry) => entry.type === "message");
+}
+export function getTranscriptBashEntries(transcript) {
+    return transcript.entries.filter((entry) => entry.type === "bash");
+}
+function getLastAssistantMessageEntry(transcript) {
+    for (let i = transcript.entries.length - 1; i >= 0; i -= 1) {
+        const entry = transcript.entries[i];
+        if (entry.type === "message" && entry.role === "assistant")
+            return entry;
+        if (entry.type === "message" && entry.role === "user")
+            break;
+    }
+    return undefined;
+}
+export function appendTranscriptMessage(transcript, role, content, thinking, timing) {
+    const entry = {
+        type: "message",
+        role,
+        thinking,
+        content,
+        at: (timing?.at ?? new Date()).toISOString(),
+        ...(timing?.durationMs !== undefined ? { durationMs: timing.durationMs } : {}),
+    };
+    transcript.entries.push(entry);
+}
+export function appendTranscriptBash(transcript, entry, timing) {
+    transcript.entries.push({
+        type: "bash",
+        ...entry,
+        at: (timing?.at ?? new Date()).toISOString(),
+    });
+}
+/** Append final model text when it is not already the last assistant message. */
+export function appendFinalTextToTranscript(transcript, finalText, timing) {
+    if (!finalText)
+        return;
+    const last = getLastAssistantMessageEntry(transcript);
+    if (last?.content === finalText)
+        return;
+    appendTranscriptMessage(transcript, "assistant", finalText, undefined, timing);
+}
+/** Format one LLM step for the transcript (content, thinking, and tool calls). */
+export function formatStepForTranscript(step) {
+    const contentParts = [];
+    if (step.text)
+        contentParts.push(step.text);
+    for (const call of step.toolCalls) {
+        const input = typeof call.input === "string"
+            ? call.input
+            : JSON.stringify(call.input);
+        contentParts.push(`[tool ${call.toolName}] ${input}`);
+    }
+    const content = contentParts.length > 0 ? contentParts.join("\n\n") : null;
+    const result = { content };
+    const thinking = step.reasoningText?.trim();
+    if (thinking)
+        result.thinking = thinking;
+    return result;
+}
+export function appendStepToTranscript(transcript, step, bashEntries = [], formattedOverride, timing) {
+    let changed = false;
+    const formatted = formattedOverride ?? formatStepForTranscript(step);
+    const recordedAt = timing?.at ?? new Date();
+    if (formatted.content || formatted.thinking) {
+        appendTranscriptMessage(transcript, "assistant", formatted.content ?? "", formatted.thinking, { at: recordedAt, durationMs: timing?.durationMs });
+        changed = true;
+    }
+    for (const entry of bashEntries) {
+        appendTranscriptBash(transcript, entry, { at: recordedAt });
+        changed = true;
+    }
+    return changed;
+}
+//# sourceMappingURL=verdict.js.map

package/dist/agent/verdict.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/agent/verdict.ts"],"names":[],"mappings":"AACA,OAAO,EACL,aAAa,GAMd,MAAM,qBAAqB,CAAC;AAE7B,sGAAsG;AACtG,MAAM,UAAU,sBAAsB,CAAC,QAAwB;IAC7D,IAAI,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC1B,OAAO,GAAG,GAAG,CAAC,IAAI,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,IAAI,KAAK,MAAM,EAAE,CAAC;QACrD,GAAG,IAAI,CAAC,CAAC;IACX,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,IAAI,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,IAAI,KAAK,WAAW,EAAE,CAAC;QACvD,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,SAAS,GAAG,+BAA+B,CAAC;IAClD,IAAI,KAA6B,CAAC;IAClC,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/C,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,0CAA0C;IAC1C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7B,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAY,CAAC;YAChD,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC,IAAI,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,WAAW;QACb,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,MAAM,MAAM,GAAG,kDAAkD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7E,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAY,CAAC;YAChD,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC,IAAI,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,WAAW;QACb,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,UAA2B;IAE3B,OAAO,UAAU,CAAC,OAAO,CAAC,MAAM,CAC9B,CAAC,KAAK,EAAmC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CACrE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,UAA2B;IAE3B,OAAO,UAAU,CAAC,OAAO,CAAC,MAAM,CAC9B,CAAC,KAAK,EAAgC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAC/D,CAAC;AACJ,CAAC;AAED,SAAS,4BAA4B,CACnC,UAA2B;IAE3B,KAAK,IAAI,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACrC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW;YAAE,OAAO,KAAK,CAAC;QACzE,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;YAAE,MAAM;IAC/D,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAOD,MAAM,UAAU,uBAAuB,CACrC,UAA2B,EAC3B,IAAY,EACZ,OAAe,EACf,QAAiB,EACjB,MAA8B;IAE9B,MAAM,KAAK,GAA2B;QACpC,IAAI,EAAE,SAAS;QACf,IAAI;QACJ,QAAQ;QACR,OAAO;QACP,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE;QAC5C,GAAG,CAAC,MAAM,EAAE,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/E,CAAC;IACF,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,UAA2B,EAC3B,KAAgB,EAChB,MAA0C;IAE1C,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC;QACtB,IAAI,EAAE,MAAM;QACZ,GAAG,KAAK;QACR,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE;KAC7C,CAAC,CAAC;AACL,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,2BAA2B,CACzC,UAA2B,EAC3B,SAAiB,EACjB,MAA8B;IAE9B,IAAI,CAAC,SAAS;QAAE,OAAO;IACvB,MAAM,IAAI,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;IACtD,IAAI,IAAI,EAAE,OAAO,KAAK,SAAS;QAAE,OAAO;IACxC,uBAAuB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;AACjF,CAAC;AAaD,kFAAkF;AAClF,MAAM,UAAU,uBAAuB,CACrC,IAAyB;IAEzB,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,IAAI,CAAC,IAAI;QAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QAClC,MAAM,KAAK,GACT,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;YAC5B,CAAC,CAAC,IAAI,CAAC,KAAK;YACZ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,YAAY,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,OAAO,GACX,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,MAAM,MAAM,GAA4B,EAAE,OAAO,EAAE,CAAC;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC;IAC5C,IAAI,QAAQ;QAAE,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACzC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,UAA2B,EAC3B,IAAyB,EACzB,cAA2B,EAAE,EAC7B,iBAA2C,EAC3C,MAA8B;IAE9B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,SAAS,GAAG,iBAAiB,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,MAAM,EAAE,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;IAC5C,IAAI,SAAS,CAAC,OAAO,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QAC5C,uBAAuB,CACrB,UAAU,EACV,WAAW,EACX,SAAS,CAAC,OAAO,IAAI,EAAE,EACvB,SAAS,CAAC,QAAQ,EAClB,EAAE,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,CACnD,CAAC;QACF,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;QAChC,oBAAoB,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAC5D,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/agent/verdict.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=verdict.test.d.ts.map

package/dist/agent/verdict.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.test.d.ts","sourceRoot":"","sources":["../../src/agent/verdict.test.ts"],"names":[],"mappings":""}

package/dist/agent/verdict.test.js

@@ -0,0 +1,156 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { appendFinalTextToTranscript, appendStepToTranscript, formatStepForTranscript, stripLastAssistantTurn, } from "./verdict.js";
+describe("stripLastAssistantTurn", () => {
+    it("removes the last assistant message", () => {
+        const messages = [
+            { role: "user", content: "go" },
+            { role: "assistant", content: "no verdict yet" },
+        ];
+        assert.deepEqual(stripLastAssistantTurn(messages), [
+            { role: "user", content: "go" },
+        ]);
+    });
+    it("removes trailing tool results with the assistant turn", () => {
+        const messages = [
+            { role: "user", content: "go" },
+            {
+                role: "assistant",
+                content: [
+                    {
+                        type: "tool-call",
+                        toolCallId: "1",
+                        toolName: "bash",
+                        input: { command: "true" },
+                    },
+                ],
+            },
+            {
+                role: "tool",
+                content: [
+                    {
+                        type: "tool-result",
+                        toolCallId: "1",
+                        toolName: "bash",
+                        output: { type: "json", value: { exitCode: 0 } },
+                    },
+                ],
+            },
+        ];
+        assert.deepEqual(stripLastAssistantTurn(messages), [
+            { role: "user", content: "go" },
+        ]);
+    });
+});
+describe("formatStepForTranscript", () => {
+    it("returns empty content when the step has no text, reasoning, or tool calls", () => {
+        assert.deepEqual(formatStepForTranscript({ text: "", toolCalls: [] }), {
+            content: null,
+        });
+    });
+    it("includes assistant text in content", () => {
+        assert.deepEqual(formatStepForTranscript({
+            text: "Opening the app.",
+            toolCalls: [],
+        }), { content: "Opening the app." });
+    });
+    it("includes tool calls in content when the step has no assistant text", () => {
+        assert.deepEqual(formatStepForTranscript({
+            text: "",
+            toolCalls: [{ toolName: "bash", input: { command: "agent-browser open /" } }],
+        }), {
+            content: '[tool bash] {"command":"agent-browser open /"}',
+        });
+    });
+    it("keeps text and tool calls in content and reasoning in thinking", () => {
+        assert.deepEqual(formatStepForTranscript({
+            text: "Checking the page.",
+            reasoningText: "Need a snapshot first.",
+            toolCalls: [{ toolName: "bash", input: { command: "agent-browser snapshot -i" } }],
+        }), {
+            content: [
+                "Checking the page.",
+                '[tool bash] {"command":"agent-browser snapshot -i"}',
+            ].join("\n\n"),
+            thinking: "Need a snapshot first.",
+        });
+    });
+});
+describe("appendFinalTextToTranscript", () => {
+    it("skips append only when the last assistant message exactly matches finalText", () => {
+        const transcript = {
+            entries: [
+                { type: "message", role: "assistant", content: "The test passed successfully", at: "2026-01-01T00:00:00.000Z" },
+            ],
+        };
+        appendFinalTextToTranscript(transcript, "passed");
+        assert.equal(transcript.entries.length, 2);
+        assert.equal(transcript.entries[1].type === "message" ? transcript.entries[1].content : "", "passed");
+    });
+    it("does not append duplicate when finalText equals the last assistant message", () => {
+        const transcript = {
+            entries: [{ type: "message", role: "assistant", content: "Done.", at: "2026-01-01T00:00:00.000Z" }],
+        };
+        appendFinalTextToTranscript(transcript, "Done.");
+        assert.equal(transcript.entries.length, 1);
+    });
+});
+describe("appendStepToTranscript", () => {
+    it("appends one assistant message per step", () => {
+        const transcript = { entries: [] };
+        assert.equal(appendStepToTranscript(transcript, {
+            text: "",
+            toolCalls: [{ toolName: "bash", input: { command: "agent-browser open /" } }],
+        }), true);
+        assert.equal(appendStepToTranscript(transcript, {
+            text: "Done.",
+            toolCalls: [],
+        }), true);
+        assert.equal(transcript.entries.length, 2);
+        const first = transcript.entries[0];
+        const second = transcript.entries[1];
+        assert.equal(first.type, "message");
+        assert.equal(first.role, "assistant");
+        assert.match(first.content, /agent-browser open/);
+        assert.ok(first.at);
+        assert.equal(second.type, "message");
+        assert.equal(second.content, "Done.");
+        assert.ok(second.at);
+    });
+    it("records timestamps and LLM step duration on assistant messages", () => {
+        const transcript = { entries: [] };
+        const at = new Date("2026-01-01T00:00:05.000Z");
+        appendStepToTranscript(transcript, { text: "Checking.", toolCalls: [] }, [], undefined, { at, durationMs: 4200 });
+        assert.equal(transcript.entries.length, 1);
+        const message = transcript.entries[0];
+        assert.equal(message.type, "message");
+        assert.equal(message.at, "2026-01-01T00:00:05.000Z");
+        assert.equal(message.durationMs, 4200);
+    });
+    it("writes the assistant message before bash results for a step", () => {
+        const transcript = { entries: [] };
+        appendStepToTranscript(transcript, {
+            text: "Opening the app.",
+            reasoningText: "Need to load the page first.",
+            toolCalls: [{ toolName: "bash", input: { command: "agent-browser open /" } }],
+        }, [
+            {
+                command: "agent-browser open /",
+                stdout: "ok",
+                stderr: "",
+                exitCode: 0,
+                durationMs: 10,
+            },
+        ]);
+        assert.equal(transcript.entries.length, 2);
+        const message = transcript.entries[0];
+        const bash = transcript.entries[1];
+        assert.equal(message.type, "message");
+        assert.equal(bash.type, "bash");
+        assert.match(message.content, /Opening the app/);
+        assert.equal(message.thinking, "Need to load the page first.");
+        assert.equal(message.at, bash.at);
+        assert.equal(bash.command, "agent-browser open /");
+    });
+});
+//# sourceMappingURL=verdict.test.js.map

package/dist/agent/verdict.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.test.js","sourceRoot":"","sources":["../../src/agent/verdict.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,cAAc,CAAC;AAEtB,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,QAAQ,GAAmB;YAC/B,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;YAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,gBAAgB,EAAE;SACjD,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,sBAAsB,CAAC,QAAQ,CAAC,EAAE;YACjD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;SAChC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,QAAQ,GAAmB;YAC/B,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;YAC/B;gBACE,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,WAAW;wBACjB,UAAU,EAAE,GAAG;wBACf,QAAQ,EAAE,MAAM;wBAChB,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;qBAC3B;iBACF;aACF;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,aAAa;wBACnB,UAAU,EAAE,GAAG;wBACf,QAAQ,EAAE,MAAM;wBAChB,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE;qBACjD;iBACF;aACF;SACF,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,sBAAsB,CAAC,QAAQ,CAAC,EAAE;YACjD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;SAChC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,EAAE;YACrE,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,SAAS,CACd,uBAAuB,CAAC;YACtB,IAAI,EAAE,kBAAkB;YACxB,SAAS,EAAE,EAAE;SACd,CAAC,EACF,EAAE,OAAO,EAAE,kBAAkB,EAAE,CAChC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;QAC5E,MAAM,CAAC,SAAS,CACd,uBAAuB,CAAC;YACtB,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAAE,CAAC;SAC9E,CAAC,EACF;YACE,OAAO,EAAE,gDAAgD;SAC1D,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,SAAS,CACd,uBAAuB,CAAC;YACtB,IAAI,EAAE,oBAAoB;YAC1B,aAAa,EAAE,wBAAwB;YACvC,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,2BAA2B,EAAE,EAAE,CAAC;SACnF,CAAC,EACF;YACE,OAAO,EAAE;gBACP,oBAAoB;gBACpB,qDAAqD;aACtD,CAAC,IAAI,CAAC,MAAM,CAAC;YACd,QAAQ,EAAE,wBAAwB;SACnC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;IAC3C,EAAE,CAAC,6EAA6E,EAAE,GAAG,EAAE;QACrF,MAAM,UAAU,GAAG;YACjB,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,SAAkB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,8BAA8B,EAAE,EAAE,EAAE,0BAA0B,EAAE;aACzH;SACF,CAAC;QAEF,2BAA2B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAClD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,KAAK,CACV,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAC/E,QAAQ,CACT,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,GAAG,EAAE;QACpF,MAAM,UAAU,GAAG;YACjB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAkB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,0BAA0B,EAAE,CAAC;SAC7G,CAAC;QAEF,2BAA2B,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,UAAU,GAAoB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAEpD,MAAM,CAAC,KAAK,CACV,sBAAsB,CAAC,UAAU,EAAE;YACjC,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAAE,CAAC;SAC9E,CAAC,EACF,IAAI,CACL,CAAC;QACF,MAAM,CAAC,KAAK,CACV,sBAAsB,CAAC,UAAU,EAAE;YACjC,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,EAAE;SACd,CAAC,EACF,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACrC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QAClD,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,UAAU,GAAoB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAEhD,sBAAsB,CACpB,UAAU,EACV,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,EAAE,EACpC,EAAE,EACF,SAAS,EACT,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CACzB,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,0BAA0B,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,UAAU,GAAoB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAEpD,sBAAsB,CACpB,UAAU,EACV;YACE,IAAI,EAAE,kBAAkB;YACxB,aAAa,EAAE,8BAA8B;YAC7C,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAAE,CAAC;SAC9E,EACD;YACE;gBACE,OAAO,EAAE,sBAAsB;gBAC/B,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,CAAC;gBACX,UAAU,EAAE,EAAE;aACf;SACF,CACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACvC,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,8BAA8B,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/analyze/build-context.d.ts

@@ -0,0 +1,58 @@
+import type { AnalyzeFinding } from "./index.js";
+import type { FlakyScenarioFinding, RunOccurrence } from "./compare-runs.js";
+import type { ScenarioResult, TranscriptEntry } from "../types/verdict.js";
+export interface ScenarioIntentContext {
+    name: string;
+    url?: string;
+    auth?: string;
+    tags?: string[];
+    goal: string;
+    steps: string;
+    then: string[];
+}
+export interface TruncatedScenarioResult {
+    scenario: string;
+    filePath: string;
+    status: string;
+    error?: string;
+    durationMs?: number;
+    verdict: ScenarioResult["verdict"];
+    transcript: {
+        entries: TranscriptEntry[];
+    };
+    healing?: ScenarioResult["healing"];
+}
+export interface LlmAnalyzeContext {
+    heuristicFinding: AnalyzeFinding;
+    scenarioIntent: ScenarioIntentContext | null;
+    scenarioResult: TruncatedScenarioResult;
+    scenarioMarkdown: string;
+}
+export interface FlakyLlmAnalyzeContext extends LlmAnalyzeContext {
+    runComparison: {
+        runIds: string[];
+        stats: {
+            pass: number;
+            fail: number;
+            error: number;
+        };
+        inconsistentCheckpoints: FlakyScenarioFinding["inconsistentCheckpoints"];
+        filePathWarnings: string[];
+        representativeRuns: {
+            pass?: TruncatedScenarioResult;
+            fail?: TruncatedScenarioResult;
+        };
+        otherRuns: Array<{
+            runId: string;
+            status: RunOccurrence["status"];
+            verdictSummary?: string;
+            failedCheckpoints: string[];
+        }>;
+    };
+}
+export declare function buildScenarioIntent(filePath: string): ScenarioIntentContext | null;
+export declare function truncateScenarioResult(result: ScenarioResult): TruncatedScenarioResult;
+export declare function buildLlmAnalyzeContext(finding: AnalyzeFinding, result: ScenarioResult, cwd: string): LlmAnalyzeContext;
+export declare function buildFlakyAnalyzeContext(finding: FlakyScenarioFinding, runDirs: string[], cwd: string): FlakyLlmAnalyzeContext;
+export declare function flakyFindingToAnalyzeFinding(finding: FlakyScenarioFinding): AnalyzeFinding;
+//# sourceMappingURL=build-context.d.ts.map

package/dist/analyze/build-context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-context.d.ts","sourceRoot":"","sources":["../../src/analyze/build-context.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,KAAK,EACV,oBAAoB,EACpB,aAAa,EACd,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAW3E,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC,SAAS,CAAC,CAAC;IACnC,UAAU,EAAE;QACV,OAAO,EAAE,eAAe,EAAE,CAAC;KAC5B,CAAC;IACF,OAAO,CAAC,EAAE,cAAc,CAAC,SAAS,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,iBAAiB;IAChC,gBAAgB,EAAE,cAAc,CAAC;IACjC,cAAc,EAAE,qBAAqB,GAAG,IAAI,CAAC;IAC7C,cAAc,EAAE,uBAAuB,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,sBAAuB,SAAQ,iBAAiB;IAC/D,aAAa,EAAE;QACb,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,KAAK,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC;QACrD,uBAAuB,EAAE,oBAAoB,CAAC,yBAAyB,CAAC,CAAC;QACzE,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B,kBAAkB,EAAE;YAClB,IAAI,CAAC,EAAE,uBAAuB,CAAC;YAC/B,IAAI,CAAC,EAAE,uBAAuB,CAAC;SAChC,CAAC;QACF,SAAS,EAAE,KAAK,CAAC;YACf,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;YAChC,cAAc,CAAC,EAAE,MAAM,CAAC;YACxB,iBAAiB,EAAE,MAAM,EAAE,CAAC;SAC7B,CAAC,CAAC;KACJ,CAAC;CACH;AAED,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,GACf,qBAAqB,GAAG,IAAI,CAe9B;AAED,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,cAAc,GACrB,uBAAuB,CAqCzB;AAED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,cAAc,EACvB,MAAM,EAAE,cAAc,EACtB,GAAG,EAAE,MAAM,GACV,iBAAiB,CAcnB;AAiBD,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,oBAAoB,EAC7B,OAAO,EAAE,MAAM,EAAE,EACjB,GAAG,EAAE,MAAM,GACV,sBAAsB,CA0DxB;AAED,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,oBAAoB,GAC5B,cAAc,CAEhB"}

package/dist/analyze/build-context.js

@@ -0,0 +1,141 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { parseScenarioFile } from "../scenarios/parser.js";
+import { selectRepresentativeRuns } from "./compare-runs.js";
+import { loadRunReport } from "./index.js";
+const MAX_TRANSCRIPT_ENTRIES = 28;
+const MAX_STDOUT_CHARS = 1500;
+const MAX_MESSAGE_CHARS = 2000;
+function truncate(text, max) {
+    if (text.length <= max)
+        return text;
+    return `${text.slice(0, max)}…`;
+}
+export function buildScenarioIntent(filePath) {
+    try {
+        const scenario = parseScenarioFile(filePath);
+        return {
+            name: scenario.frontmatter.name,
+            url: scenario.frontmatter.url,
+            auth: scenario.frontmatter.auth,
+            tags: scenario.frontmatter.tags,
+            goal: scenario.goal,
+            steps: scenario.steps,
+            then: scenario.then,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+export function truncateScenarioResult(result) {
+    const entries = result.transcript.entries
+        .slice(-MAX_TRANSCRIPT_ENTRIES)
+        .map((entry) => {
+        if (entry.type === "message") {
+            return {
+                type: "message",
+                role: entry.role,
+                content: truncate(entry.content, MAX_MESSAGE_CHARS),
+                at: entry.at,
+                ...(entry.thinking
+                    ? { thinking: truncate(entry.thinking, MAX_MESSAGE_CHARS) }
+                    : {}),
+                ...(entry.durationMs !== undefined ? { durationMs: entry.durationMs } : {}),
+            };
+        }
+        return {
+            type: "bash",
+            command: entry.command,
+            exitCode: entry.exitCode,
+            stdout: truncate(entry.stdout, MAX_STDOUT_CHARS),
+            stderr: truncate(entry.stderr, 500),
+            durationMs: entry.durationMs,
+            at: entry.at,
+        };
+    });
+    return {
+        scenario: result.scenario,
+        filePath: result.filePath,
+        status: result.status,
+        error: result.error,
+        durationMs: result.durationMs,
+        verdict: result.verdict,
+        transcript: { entries },
+        healing: result.healing,
+    };
+}
+export function buildLlmAnalyzeContext(finding, result, cwd) {
+    const scenarioPath = path.isAbsolute(result.filePath)
+        ? result.filePath
+        : path.resolve(cwd, result.filePath);
+    const scenarioMarkdown = readFileSync(scenarioPath, "utf-8");
+    const scenarioIntent = buildScenarioIntent(scenarioPath);
+    return {
+        heuristicFinding: finding,
+        scenarioIntent,
+        scenarioResult: truncateScenarioResult(result),
+        scenarioMarkdown,
+    };
+}
+function syntheticFindingFromFlaky(finding) {
+    const assessment = finding.heuristicAssessment;
+    return {
+        scenario: finding.scenario,
+        filePath: finding.filePath,
+        status: finding.failCount > 0 ? "fail" : "error",
+        failureKind: assessment.dominantKind,
+        confidence: assessment.likelyFalseNegative ? "high" : "medium",
+        suggestions: assessment.suggestions,
+        signals: finding.runs.flatMap((r) => r.signals ?? []).slice(0, 6),
+    };
+}
+export function buildFlakyAnalyzeContext(finding, runDirs, cwd) {
+    const reports = runDirs.map((runDir) => ({
+        runDir,
+        report: loadRunReport(runDir),
+    }));
+    const entries = reports.flatMap(({ report }) => report.results
+        .filter((r) => (r.scenario || r.filePath) === finding.scenario)
+        .map((result) => ({ runId: report.runId, result })));
+    const { pass: passEntry, fail: failEntry } = selectRepresentativeRuns(entries);
+    const representativeRunIds = new Set([passEntry?.runId, failEntry?.runId].filter(Boolean));
+    const otherRuns = entries
+        .filter(({ runId }) => !representativeRunIds.has(runId))
+        .map(({ runId, result }) => ({
+        runId,
+        status: result.status,
+        verdictSummary: result.verdict?.summary,
+        failedCheckpoints: result.verdict?.checkpoints
+            .filter((c) => !c.pass)
+            .map((c) => c.assertion) ?? [],
+    }));
+    const baseResult = failEntry?.result ?? passEntry?.result ?? entries[0].result;
+    const base = buildLlmAnalyzeContext(syntheticFindingFromFlaky(finding), baseResult, cwd);
+    return {
+        ...base,
+        runComparison: {
+            runIds: finding.runs.map((r) => r.runId),
+            stats: {
+                pass: finding.passCount,
+                fail: finding.failCount,
+                error: finding.errorCount,
+            },
+            inconsistentCheckpoints: finding.inconsistentCheckpoints,
+            filePathWarnings: finding.filePathWarnings,
+            representativeRuns: {
+                ...(passEntry
+                    ? { pass: truncateScenarioResult(passEntry.result) }
+                    : {}),
+                ...(failEntry
+                    ? { fail: truncateScenarioResult(failEntry.result) }
+                    : {}),
+            },
+            otherRuns,
+        },
+    };
+}
+export function flakyFindingToAnalyzeFinding(finding) {
+    return syntheticFindingFromFlaky(finding);
+}
+//# sourceMappingURL=build-context.js.map

package/dist/analyze/build-context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-context.js","sourceRoot":"","sources":["../../src/analyze/build-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAM3D,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG3C,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B,SAAS,QAAQ,CAAC,IAAY,EAAE,GAAW;IACzC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACpC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC;AAClC,CAAC;AAmDD,MAAM,UAAU,mBAAmB,CACjC,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC7C,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,IAAI;YAC/B,GAAG,EAAE,QAAQ,CAAC,WAAW,CAAC,GAAG;YAC7B,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,IAAI;YAC/B,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,IAAI;YAC/B,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,IAAI,EAAE,QAAQ,CAAC,IAAI;SACpB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,MAAsB;IAEtB,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO;SACtC,KAAK,CAAC,CAAC,sBAAsB,CAAC;SAC9B,GAAG,CAAC,CAAC,KAAK,EAAmB,EAAE;QAC9B,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,iBAAiB,CAAC;gBACnD,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,GAAG,CAAC,KAAK,CAAC,QAAQ;oBAChB,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,iBAAiB,CAAC,EAAE;oBAC3D,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5E,CAAC;QACJ,CAAC;QACD,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,gBAAgB,CAAC;YAChD,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC;YACnC,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,EAAE,EAAE,KAAK,CAAC,EAAE;SACb,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU,EAAE,EAAE,OAAO,EAAE;QACvB,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,OAAuB,EACvB,MAAsB,EACtB,GAAW;IAEX,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC;QACnD,CAAC,CAAC,MAAM,CAAC,QAAQ;QACjB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEvC,MAAM,gBAAgB,GAAG,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,cAAc,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;IAEzD,OAAO;QACL,gBAAgB,EAAE,OAAO;QACzB,cAAc;QACd,cAAc,EAAE,sBAAsB,CAAC,MAAM,CAAC;QAC9C,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,yBAAyB,CAChC,OAA6B;IAE7B,MAAM,UAAU,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAC/C,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;QAChD,WAAW,EAAE,UAAU,CAAC,YAAY;QACpC,UAAU,EAAE,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;QAC9D,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,OAA6B,EAC7B,OAAiB,EACjB,GAAW;IAEX,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACvC,MAAM;QACN,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC;KAC9B,CAAC,CAAC,CAAC;IAEJ,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAC7C,MAAM,CAAC,OAAO;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,OAAO,CAAC,QAAQ,CAAC;SAC9D,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CACtD,CAAC;IAEF,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC/E,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAClC,CAAC,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CACrD,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO;SACtB,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;SACvD,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3B,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,cAAc,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO;QACvC,iBAAiB,EACf,MAAM,CAAC,OAAO,EAAE,WAAW;aACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;aACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE;KACnC,CAAC,CAAC,CAAC;IAEN,MAAM,UAAU,GAAG,SAAS,EAAE,MAAM,IAAI,SAAS,EAAE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IAChF,MAAM,IAAI,GAAG,sBAAsB,CACjC,yBAAyB,CAAC,OAAO,CAAC,EAClC,UAAU,EACV,GAAG,CACJ,CAAC;IAEF,OAAO;QACL,GAAG,IAAI;QACP,aAAa,EAAE;YACb,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YACxC,KAAK,EAAE;gBACL,IAAI,EAAE,OAAO,CAAC,SAAS;gBACvB,IAAI,EAAE,OAAO,CAAC,SAAS;gBACvB,KAAK,EAAE,OAAO,CAAC,UAAU;aAC1B;YACD,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;YACxD,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,kBAAkB,EAAE;gBAClB,GAAG,CAAC,SAAS;oBACX,CAAC,CAAC,EAAE,IAAI,EAAE,sBAAsB,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;oBACpD,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,SAAS;oBACX,CAAC,CAAC,EAAE,IAAI,EAAE,sBAAsB,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;oBACpD,CAAC,CAAC,EAAE,CAAC;aACR;YACD,SAAS;SACV;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,OAA6B;IAE7B,OAAO,yBAAyB,CAAC,OAAO,CAAC,CAAC;AAC5C,CAAC"}

package/dist/analyze/build-context.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=build-context.test.d.ts.map

package/dist/analyze/build-context.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-context.test.d.ts","sourceRoot":"","sources":["../../src/analyze/build-context.test.ts"],"names":[],"mappings":""}