prose-qa 0.1.0

package/dist/analyze/build-context.test.js

@@ -0,0 +1,118 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { describe, it } from "node:test";
+import { buildFlakyAnalyzeContext, buildScenarioIntent, } from "./build-context.js";
+import { mkdirSync } from "node:fs";
+describe("buildScenarioIntent", () => {
+    it("extracts goal, steps, and then from a scenario file", () => {
+        const dir = mkdtempSync(path.join(tmpdir(), "pqa-intent-"));
+        const file = path.join(dir, "demo.md");
+        writeFileSync(file, `---
+name: demo-flow
+url: http://localhost:3000/invoices
+tags: [smoke]
+---
+
+# Goal
+Verify invoice status can be changed.
+
+# Steps
+1. Open the first invoice.
+2. Change status to Paid.
+
+# Then
+- page shows "Paid"
+`, "utf-8");
+        const intent = buildScenarioIntent(file);
+        assert.ok(intent);
+        assert.equal(intent.name, "demo-flow");
+        assert.match(intent.goal, /invoice status/i);
+        assert.match(intent.steps, /Open the first invoice/);
+        assert.deepEqual(intent.then, ['page shows "Paid"']);
+    });
+});
+describe("buildFlakyAnalyzeContext", () => {
+    it("includes pass and fail representative runs", () => {
+        const cwd = mkdtempSync(path.join(tmpdir(), "pqa-flaky-ctx-"));
+        const scenarioPath = path.join(cwd, "scenarios", "demo.md");
+        mkdirSync(path.dirname(scenarioPath), { recursive: true });
+        writeFileSync(scenarioPath, `---
+name: demo
+---
+
+# Goal
+Verify demo.
+
+# Steps
+1. Open app.
+
+# Then
+- page shows "OK"
+`);
+        const makeResult = (status, runId) => ({
+            scenario: "demo",
+            filePath: "scenarios/demo.md",
+            status,
+            durationMs: 1000,
+            verdict: {
+                status: status === "pass" ? "pass" : "fail",
+                summary: status,
+                checkpoints: [
+                    {
+                        assertion: 'page shows "OK"',
+                        pass: status === "pass",
+                        reason: status,
+                    },
+                ],
+            },
+            transcript: { entries: [] },
+        });
+        const writeRun = (runId, result) => {
+            const runDir = path.join(cwd, ".pqa", "runs", runId);
+            mkdirSync(runDir, { recursive: true });
+            const report = {
+                runId,
+                startedAt: new Date().toISOString(),
+                finishedAt: new Date().toISOString(),
+                results: [result],
+                summary: { total: 1, passed: 0, failed: 0, errors: 0, skipped: 0 },
+            };
+            writeFileSync(path.join(runDir, "report.json"), JSON.stringify(report));
+            return runDir;
+        };
+        const runPass = writeRun("run-pass", makeResult("pass", "run-pass"));
+        const runFail = writeRun("run-fail", makeResult("fail", "run-fail"));
+        const finding = {
+            scenario: "demo",
+            filePath: "scenarios/demo.md",
+            filePathWarnings: [],
+            runCount: 2,
+            passCount: 1,
+            failCount: 1,
+            errorCount: 0,
+            inconsistentCheckpoints: [
+                {
+                    assertion: 'page shows "OK"',
+                    passedIn: ["run-pass"],
+                    failedIn: ["run-fail"],
+                },
+            ],
+            runs: [],
+            heuristicAssessment: {
+                dominantKind: "scenario_issue",
+                likelyFalseNegative: true,
+                likelyFalsePositive: false,
+                suggestions: [],
+            },
+        };
+        const context = buildFlakyAnalyzeContext(finding, [runPass, runFail], cwd);
+        assert.ok(context.runComparison.representativeRuns.pass);
+        assert.ok(context.runComparison.representativeRuns.fail);
+        assert.equal(context.runComparison.stats.pass, 1);
+        assert.equal(context.runComparison.stats.fail, 1);
+        assert.match(context.scenarioMarkdown, /# Goal/);
+    });
+});
+//# sourceMappingURL=build-context.test.js.map

package/dist/analyze/build-context.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-context.test.js","sourceRoot":"","sources":["../../src/analyze/build-context.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EACL,wBAAwB,EACxB,mBAAmB,GACpB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEpC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACvC,aAAa,CACX,IAAI,EACJ;;;;;;;;;;;;;;;CAeL,EACK,OAAO,CACR,CAAC;QAEF,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,MAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,MAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,MAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,CAAC;QACtD,MAAM,CAAC,SAAS,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC,mBAAmB,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QAC5D,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,aAAa,CACX,YAAY,EACZ;;;;;;;;;;;;CAYL,CACI,CAAC;QAEF,MAAM,UAAU,GAAG,CAAC,MAAuB,EAAE,KAAa,EAAkB,EAAE,CAAC,CAAC;YAC9E,QAAQ,EAAE,MAAM;YAChB,QAAQ,EAAE,mBAAmB;YAC7B,MAAM;YACN,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAC3C,OAAO,EAAE,MAAM;gBACf,WAAW,EAAE;oBACX;wBACE,SAAS,EAAE,iBAAiB;wBAC5B,IAAI,EAAE,MAAM,KAAK,MAAM;wBACvB,MAAM,EAAE,MAAM;qBACf;iBACF;aACF;YACD,UAAU,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;SAC5B,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,MAAsB,EAAU,EAAE;YACjE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACrD,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACvC,MAAM,MAAM,GAAc;gBACxB,KAAK;gBACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,OAAO,EAAE,CAAC,MAAM,CAAC;gBACjB,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;aACnE,CAAC;YACF,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YACxE,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;QACrE,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;QAErE,MAAM,OAAO,GAAyB;YACpC,QAAQ,EAAE,MAAM;YAChB,QAAQ,EAAE,mBAAmB;YAC7B,gBAAgB,EAAE,EAAE;YACpB,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC;YACb,uBAAuB,EAAE;gBACvB;oBACE,SAAS,EAAE,iBAAiB;oBAC5B,QAAQ,EAAE,CAAC,UAAU,CAAC;oBACtB,QAAQ,EAAE,CAAC,UAAU,CAAC;iBACvB;aACF;YACD,IAAI,EAAE,EAAE;YACR,mBAAmB,EAAE;gBACnB,YAAY,EAAE,gBAAgB;gBAC9B,mBAAmB,EAAE,IAAI;gBACzB,mBAAmB,EAAE,KAAK;gBAC1B,WAAW,EAAE,EAAE;aAChB;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,wBAAwB,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;QAC3E,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAClD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAClD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/analyze/compare-runs.d.ts

@@ -0,0 +1,49 @@
+import type { FailureKind } from "../types/verdict.js";
+import type { ScenarioResult } from "../types/verdict.js";
+export interface RunOccurrence {
+    runId: string;
+    status: "pass" | "fail" | "error" | "skipped";
+    durationMs: number;
+    failedCheckpoints: string[];
+    failureKind?: FailureKind;
+    signals?: string[];
+    healingUsed?: boolean;
+    errorIntermittent?: boolean;
+}
+export interface FlakyScenarioFinding {
+    scenario: string;
+    filePath: string;
+    filePathWarnings: string[];
+    runCount: number;
+    passCount: number;
+    failCount: number;
+    errorCount: number;
+    inconsistentCheckpoints: Array<{
+        assertion: string;
+        passedIn: string[];
+        failedIn: string[];
+    }>;
+    runs: RunOccurrence[];
+    heuristicAssessment: {
+        dominantKind: FailureKind;
+        likelyFalseNegative: boolean;
+        likelyFalsePositive: boolean;
+        suggestions: string[];
+    };
+}
+export interface FlakyAnalyzeReport {
+    runIds: string[];
+    analyzedAt: string;
+    findings: FlakyScenarioFinding[];
+}
+interface ScenarioRunEntry {
+    runId: string;
+    result: ScenarioResult;
+}
+export declare function selectRepresentativeRuns(entries: ScenarioRunEntry[]): {
+    pass?: ScenarioRunEntry;
+    fail?: ScenarioRunEntry;
+};
+export declare function compareRuns(runDirs: string[], cwd: string): FlakyAnalyzeReport;
+export {};
+//# sourceMappingURL=compare-runs.d.ts.map

package/dist/analyze/compare-runs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare-runs.d.ts","sourceRoot":"","sources":["../../src/analyze/compare-runs.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAa,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAMrE,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,uBAAuB,EAAE,KAAK,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC,CAAC;IACH,IAAI,EAAE,aAAa,EAAE,CAAC;IACtB,mBAAmB,EAAE;QACnB,YAAY,EAAE,WAAW,CAAC;QAC1B,mBAAmB,EAAE,OAAO,CAAC;QAC7B,mBAAmB,EAAE,OAAO,CAAC;QAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;CACH;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,oBAAoB,EAAE,CAAC;CAClC;AAED,UAAU,gBAAgB;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,cAAc,CAAC;CACxB;AA6LD,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,gBAAgB,EAAE,GAC1B;IAAE,IAAI,CAAC,EAAE,gBAAgB,CAAC;IAAC,IAAI,CAAC,EAAE,gBAAgB,CAAA;CAAE,CAWtD;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,kBAAkB,CAuF9E"}

package/dist/analyze/compare-runs.js

@@ -0,0 +1,214 @@
+import path from "node:path";
+import { classifyFailure } from "../healing/classify.js";
+import { parseScenarioFile } from "../scenarios/parser.js";
+import { suggestScenarioFixes } from "./suggest.js";
+import { loadRunReport } from "./index.js";
+const INFRA_ERROR_MS = 5000;
+function scenarioKey(result) {
+    return result.scenario || result.filePath;
+}
+function failedCheckpoints(result) {
+    return (result.verdict?.checkpoints.filter((c) => !c.pass).map((c) => c.assertion) ??
+        []);
+}
+function isInfraError(result) {
+    return result.status === "error" && result.durationMs < INFRA_ERROR_MS;
+}
+function statusSet(entries) {
+    const relevant = entries.filter((e) => !e.errorIntermittent);
+    return new Set(relevant.map((e) => e.status));
+}
+function isVerdictInconsistent(entries) {
+    const relevant = entries.filter((e) => !e.errorIntermittent);
+    if (relevant.length < 2)
+        return false;
+    const statuses = statusSet(entries);
+    const hasPass = statuses.has("pass");
+    const hasFail = statuses.has("fail") || statuses.has("error");
+    if (hasPass && hasFail)
+        return true;
+    return false;
+}
+function buildInconsistentCheckpoints(entries) {
+    const byAssertion = new Map();
+    for (const { runId, result } of entries) {
+        if (isInfraError(result))
+            continue;
+        for (const cp of result.verdict?.checkpoints ?? []) {
+            let bucket = byAssertion.get(cp.assertion);
+            if (!bucket) {
+                bucket = { passedIn: [], failedIn: [] };
+                byAssertion.set(cp.assertion, bucket);
+            }
+            if (cp.pass) {
+                bucket.passedIn.push(runId);
+            }
+            else {
+                bucket.failedIn.push(runId);
+            }
+        }
+    }
+    const inconsistent = [];
+    for (const [assertion, bucket] of byAssertion) {
+        if (bucket.passedIn.length > 0 && bucket.failedIn.length > 0) {
+            inconsistent.push({ assertion, ...bucket });
+        }
+    }
+    return inconsistent;
+}
+function buildRunOccurrence(runId, result, cwd) {
+    const errorIntermittent = isInfraError(result);
+    const occurrence = {
+        runId,
+        status: result.status,
+        durationMs: result.durationMs,
+        failedCheckpoints: failedCheckpoints(result),
+        healingUsed: result.healing?.used ?? false,
+        errorIntermittent,
+    };
+    if (result.status === "fail" || result.status === "error") {
+        let scenario;
+        try {
+            scenario = parseScenarioFile(path.isAbsolute(result.filePath)
+                ? result.filePath
+                : path.resolve(cwd, result.filePath));
+        }
+        catch {
+            scenario = undefined;
+        }
+        const classified = classifyFailure(result, scenario);
+        occurrence.failureKind = classified.kind;
+        occurrence.signals = classified.signals;
+    }
+    return occurrence;
+}
+function dominantFailureKind(runs) {
+    const counts = new Map();
+    for (const run of runs) {
+        if (!run.failureKind)
+            continue;
+        counts.set(run.failureKind, (counts.get(run.failureKind) ?? 0) + 1);
+    }
+    let best = "unknown";
+    let bestCount = 0;
+    for (const [kind, count] of counts) {
+        if (count > bestCount) {
+            best = kind;
+            bestCount = count;
+        }
+    }
+    return best;
+}
+function buildHeuristicAssessment(runs, inconsistentCheckpoints, failEntries, cwd) {
+    const dominantKind = dominantFailureKind(runs);
+    const hasPass = runs.some((r) => r.status === "pass" && !r.errorIntermittent);
+    const hasFail = runs.some((r) => (r.status === "fail" || r.status === "error") && !r.errorIntermittent);
+    const passWithHealing = runs.some((r) => r.status === "pass" && r.healingUsed === true);
+    const likelyFalseNegative = hasPass &&
+        hasFail &&
+        (dominantKind === "scenario_issue" ||
+            inconsistentCheckpoints.some((cp) => cp.failedIn.length > 0));
+    const likelyFalsePositive = hasPass && (passWithHealing || hasFail === false);
+    const suggestions = new Set();
+    if (likelyFalseNegative) {
+        suggestions.add("Pass and fail disagree — review Then placement or assertion strictness (possible false negative).");
+    }
+    if (passWithHealing) {
+        suggestions.add("At least one pass used healing recovery — pass may hide an unstable flow (possible false positive).");
+    }
+    if (dominantKind === "transient") {
+        suggestions.add("Failures look timing-related — add explicit waits after navigation or submit.");
+    }
+    for (const cp of inconsistentCheckpoints) {
+        suggestions.add(`Checkpoint "${cp.assertion}" flips between pass and fail — stabilize timing or move check earlier.`);
+    }
+    for (const { result } of failEntries) {
+        let scenario;
+        try {
+            scenario = parseScenarioFile(path.isAbsolute(result.filePath)
+                ? result.filePath
+                : path.resolve(cwd, result.filePath));
+        }
+        catch {
+            scenario = undefined;
+        }
+        const classified = classifyFailure(result, scenario);
+        for (const s of suggestScenarioFixes(result, scenario, classified)) {
+            suggestions.add(s);
+        }
+    }
+    return {
+        dominantKind,
+        likelyFalseNegative,
+        likelyFalsePositive,
+        suggestions: [...suggestions].slice(0, 8),
+    };
+}
+export function selectRepresentativeRuns(entries) {
+    const sorted = [...entries].sort((a, b) => b.runId.localeCompare(a.runId));
+    const pass = sorted.find((e) => e.result.status === "pass" && !isInfraError(e.result));
+    const fail = sorted.find((e) => (e.result.status === "fail" || e.result.status === "error") &&
+        !isInfraError(e.result));
+    return { pass, fail };
+}
+export function compareRuns(runDirs, cwd) {
+    const reports = runDirs.map((runDir) => ({
+        runDir,
+        report: loadRunReport(runDir),
+    }));
+    const runIds = reports.map(({ report }) => report.runId);
+    const byScenario = new Map();
+    for (const { report } of reports) {
+        for (const result of report.results) {
+            const key = scenarioKey(result);
+            const list = byScenario.get(key) ?? [];
+            list.push({ runId: report.runId, result });
+            byScenario.set(key, list);
+        }
+    }
+    const findings = [];
+    for (const [scenario, entries] of byScenario) {
+        if (entries.length < 2)
+            continue;
+        const runs = entries.map(({ runId, result }) => buildRunOccurrence(runId, result, cwd));
+        const inconsistentCheckpoints = buildInconsistentCheckpoints(entries);
+        const verdictInconsistent = isVerdictInconsistent(runs);
+        const checkpointInconsistent = inconsistentCheckpoints.length > 0;
+        if (!verdictInconsistent && !checkpointInconsistent)
+            continue;
+        const filePaths = [...new Set(entries.map((e) => e.result.filePath))];
+        const filePathWarnings = filePaths.length > 1
+            ? [
+                `Scenario "${scenario}" uses different file paths across runs: ${filePaths.join(", ")}`,
+            ]
+            : [];
+        const failEntries = entries.filter((e) => (e.result.status === "fail" || e.result.status === "error") &&
+            !isInfraError(e.result));
+        const passCount = runs.filter((r) => r.status === "pass" && !r.errorIntermittent).length;
+        const failCount = runs.filter((r) => r.status === "fail").length;
+        const errorCount = runs.filter((r) => r.status === "error" && !r.errorIntermittent).length;
+        findings.push({
+            scenario,
+            filePath: filePaths[0],
+            filePathWarnings,
+            runCount: runs.length,
+            passCount,
+            failCount,
+            errorCount,
+            inconsistentCheckpoints,
+            runs,
+            heuristicAssessment: buildHeuristicAssessment(runs, inconsistentCheckpoints, failEntries, cwd),
+        });
+    }
+    findings.sort((a, b) => {
+        const aFlips = a.failCount + a.errorCount;
+        const bFlips = b.failCount + b.errorCount;
+        return bFlips - aFlips || b.passCount - a.passCount;
+    });
+    return {
+        runIds,
+        analyzedAt: new Date().toISOString(),
+        findings,
+    };
+}
+//# sourceMappingURL=compare-runs.js.map

package/dist/analyze/compare-runs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare-runs.js","sourceRoot":"","sources":["../../src/analyze/compare-runs.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAG3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,cAAc,GAAG,IAAI,CAAC;AA8C5B,SAAS,WAAW,CAAC,MAAsB;IACzC,OAAO,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;AAC5C,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAsB;IAC/C,OAAO,CACL,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,EAAE,CACH,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,MAAsB;IAC1C,OAAO,MAAM,CAAC,MAAM,KAAK,OAAO,IAAI,MAAM,CAAC,UAAU,GAAG,cAAc,CAAC;AACzE,CAAC;AAED,SAAS,SAAS,CAAC,OAAwB;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAC7D,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAwB;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAC7D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtC,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC9D,IAAI,OAAO,IAAI,OAAO;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,4BAA4B,CACnC,OAA2B;IAE3B,MAAM,WAAW,GAAG,IAAI,GAAG,EAGxB,CAAC;IAEJ,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;QACxC,IAAI,YAAY,CAAC,MAAM,CAAC;YAAE,SAAS;QACnC,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE,WAAW,IAAI,EAAE,EAAE,CAAC;YACnD,IAAI,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;gBACxC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACxC,CAAC;YACD,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACZ,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAoD,EAAE,CAAC;IACzE,KAAK,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC9C,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7D,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAa,EACb,MAAsB,EACtB,GAAW;IAEX,MAAM,iBAAiB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAkB;QAChC,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,iBAAiB,EAAE,iBAAiB,CAAC,MAAM,CAAC;QAC5C,WAAW,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI,IAAI,KAAK;QAC1C,iBAAiB;KAClB,CAAC;IAEF,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QAC1D,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,iBAAiB,CAC1B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAC9B,CAAC,CAAC,MAAM,CAAC,QAAQ;gBACjB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CACvC,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,GAAG,SAAS,CAAC;QACvB,CAAC;QACD,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACrD,UAAU,CAAC,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC;QACzC,UAAU,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;IAC1C,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,mBAAmB,CAC1B,IAAqB;IAErB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,GAAG,CAAC,WAAW;YAAE,SAAS;QAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,IAAI,GAAgB,SAAS,CAAC;IAClC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,IAAI,GAAG,IAAI,CAAC;YACZ,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,wBAAwB,CAC/B,IAAqB,EACrB,uBAAwE,EACxE,WAA+B,EAC/B,GAAW;IAEX,MAAM,YAAY,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CACvB,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CACxE,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,WAAW,KAAK,IAAI,CACrD,CAAC;IAEF,MAAM,mBAAmB,GACvB,OAAO;QACP,OAAO;QACP,CAAC,YAAY,KAAK,gBAAgB;YAChC,uBAAuB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAElE,MAAM,mBAAmB,GAAG,OAAO,IAAI,CAAC,eAAe,IAAI,OAAO,KAAK,KAAK,CAAC,CAAC;IAE9E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,IAAI,mBAAmB,EAAE,CAAC;QACxB,WAAW,CAAC,GAAG,CACb,mGAAmG,CACpG,CAAC;IACJ,CAAC;IACD,IAAI,eAAe,EAAE,CAAC;QACpB,WAAW,CAAC,GAAG,CACb,qGAAqG,CACtG,CAAC;IACJ,CAAC;IACD,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;QACjC,WAAW,CAAC,GAAG,CAAC,+EAA+E,CAAC,CAAC;IACnG,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,uBAAuB,EAAE,CAAC;QACzC,WAAW,CAAC,GAAG,CACb,eAAe,EAAE,CAAC,SAAS,yEAAyE,CACrG,CAAC;IACJ,CAAC;IAED,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,iBAAiB,CAC1B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAC9B,CAAC,CAAC,MAAM,CAAC,QAAQ;gBACjB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CACvC,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,GAAG,SAAS,CAAC;QACvB,CAAC;QACD,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,oBAAoB,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;YACnE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO;QACL,YAAY;QACZ,mBAAmB;QACnB,mBAAmB;QACnB,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,OAA2B;IAE3B,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CACtB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAC7D,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CACtB,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC;QAC3D,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAC1B,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAiB,EAAE,GAAW;IACxD,MAAM,OAAO,GAAiD,OAAO,CAAC,GAAG,CACvE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACX,MAAM;QACN,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC;KAC9B,CAAC,CACH,CAAC;IAEF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,IAAI,GAAG,EAA8B,CAAC;IAEzD,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;QACjC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;YAChC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3C,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAE5C,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,UAAU,EAAE,CAAC;QAC7C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAEjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAC7C,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CACvC,CAAC;QAEF,MAAM,uBAAuB,GAAG,4BAA4B,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,mBAAmB,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,sBAAsB,GAAG,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC;QAElE,IAAI,CAAC,mBAAmB,IAAI,CAAC,sBAAsB;YAAE,SAAS;QAE9D,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,gBAAgB,GACpB,SAAS,CAAC,MAAM,GAAG,CAAC;YAClB,CAAC,CAAC;gBACE,aAAa,QAAQ,4CAA4C,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aACxF;YACH,CAAC,CAAC,EAAE,CAAC;QAET,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC;YAC3D,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAC1B,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAC3B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,iBAAiB,CACnD,CAAC,MAAM,CAAC;QACT,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,IAAI,CAAC,CAAC,CAAC,iBAAiB,CACpD,CAAC,MAAM,CAAC;QAET,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ;YACR,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAE;YACvB,gBAAgB;YAChB,QAAQ,EAAE,IAAI,CAAC,MAAM;YACrB,SAAS;YACT,SAAS;YACT,UAAU;YACV,uBAAuB;YACvB,IAAI;YACJ,mBAAmB,EAAE,wBAAwB,CAC3C,IAAI,EACJ,uBAAuB,EACvB,WAAW,EACX,GAAG,CACJ;SACF,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,UAAU,CAAC;QAC1C,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,UAAU,CAAC;QAC1C,OAAO,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,MAAM;QACN,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/analyze/compare-runs.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=compare-runs.test.d.ts.map

package/dist/analyze/compare-runs.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare-runs.test.d.ts","sourceRoot":"","sources":["../../src/analyze/compare-runs.test.ts"],"names":[],"mappings":""}

package/dist/analyze/compare-runs.test.js

@@ -0,0 +1,139 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { describe, it } from "node:test";
+import { compareRuns, selectRepresentativeRuns } from "./compare-runs.js";
+function minimalResult(overrides) {
+    return {
+        filePath: "scenarios/demo.md",
+        durationMs: 10_000,
+        verdict: {
+            status: overrides.status === "pass" ? "pass" : "fail",
+            summary: "summary",
+            checkpoints: [],
+        },
+        transcript: { entries: [] },
+        ...overrides,
+    };
+}
+function writeRun(cwd, runId, results) {
+    const runDir = path.join(cwd, ".pqa", "runs", runId);
+    mkdirSync(runDir, { recursive: true });
+    const report = {
+        runId,
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        results,
+        summary: {
+            total: results.length,
+            passed: results.filter((r) => r.status === "pass").length,
+            failed: results.filter((r) => r.status === "fail").length,
+            errors: results.filter((r) => r.status === "error").length,
+            skipped: results.filter((r) => r.status === "skipped").length,
+        },
+    };
+    writeFileSync(path.join(runDir, "report.json"), JSON.stringify(report));
+    return runDir;
+}
+describe("compareRuns", () => {
+    it("detects pass/fail flip for the same scenario", () => {
+        const cwd = mkdtempSync(path.join(tmpdir(), "pqa-compare-"));
+        const runA = writeRun(cwd, "run-a", [
+            minimalResult({ scenario: "demo", status: "pass" }),
+        ]);
+        const runB = writeRun(cwd, "run-b", [
+            minimalResult({ scenario: "demo", status: "fail" }),
+        ]);
+        const report = compareRuns([runA, runB], cwd);
+        assert.equal(report.findings.length, 1);
+        assert.equal(report.findings[0].scenario, "demo");
+        assert.equal(report.findings[0].passCount, 1);
+        assert.equal(report.findings[0].failCount, 1);
+    });
+    it("detects inconsistent checkpoints", () => {
+        const cwd = mkdtempSync(path.join(tmpdir(), "pqa-compare-"));
+        const runA = writeRun(cwd, "run-a", [
+            minimalResult({
+                scenario: "demo",
+                status: "pass",
+                verdict: {
+                    status: "pass",
+                    summary: "ok",
+                    checkpoints: [
+                        { assertion: 'page shows "Projects"', pass: true, reason: "found" },
+                    ],
+                },
+            }),
+        ]);
+        const runB = writeRun(cwd, "run-b", [
+            minimalResult({
+                scenario: "demo",
+                status: "fail",
+                verdict: {
+                    status: "fail",
+                    summary: "fail",
+                    checkpoints: [
+                        {
+                            assertion: 'page shows "Projects"',
+                            pass: false,
+                            reason: "not found",
+                        },
+                    ],
+                },
+            }),
+        ]);
+        const report = compareRuns([runA, runB], cwd);
+        assert.equal(report.findings.length, 1);
+        assert.equal(report.findings[0].inconsistentCheckpoints.length, 1);
+        assert.equal(report.findings[0].inconsistentCheckpoints[0].assertion, 'page shows "Projects"');
+    });
+    it("ignores always-pass scenarios", () => {
+        const cwd = mkdtempSync(path.join(tmpdir(), "pqa-compare-"));
+        const runA = writeRun(cwd, "run-a", [
+            minimalResult({ scenario: "stable", status: "pass" }),
+        ]);
+        const runB = writeRun(cwd, "run-b", [
+            minimalResult({ scenario: "stable", status: "pass" }),
+        ]);
+        const report = compareRuns([runA, runB], cwd);
+        assert.equal(report.findings.length, 0);
+    });
+    it("tags short errors as intermittent and skips flaky flag when only infra errors flip", () => {
+        const cwd = mkdtempSync(path.join(tmpdir(), "pqa-compare-"));
+        const runA = writeRun(cwd, "run-a", [
+            minimalResult({ scenario: "demo", status: "pass" }),
+        ]);
+        const runB = writeRun(cwd, "run-b", [
+            minimalResult({
+                scenario: "demo",
+                status: "error",
+                durationMs: 100,
+                error: "browser crashed",
+                verdict: null,
+            }),
+        ]);
+        const report = compareRuns([runA, runB], cwd);
+        assert.equal(report.findings.length, 0);
+    });
+});
+describe("selectRepresentativeRuns", () => {
+    it("picks most recent pass and fail by runId", () => {
+        const pass = {
+            runId: "run-z",
+            result: minimalResult({ scenario: "demo", status: "pass" }),
+        };
+        const fail = {
+            runId: "run-y",
+            result: minimalResult({ scenario: "demo", status: "fail" }),
+        };
+        const olderPass = {
+            runId: "run-a",
+            result: minimalResult({ scenario: "demo", status: "pass" }),
+        };
+        const selected = selectRepresentativeRuns([olderPass, fail, pass]);
+        assert.equal(selected.pass?.runId, "run-z");
+        assert.equal(selected.fail?.runId, "run-y");
+    });
+});
+//# sourceMappingURL=compare-runs.test.js.map

package/dist/analyze/compare-runs.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare-runs.test.js","sourceRoot":"","sources":["../../src/analyze/compare-runs.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAE1E,SAAS,aAAa,CACpB,SAAgF;IAEhF,OAAO;QACL,QAAQ,EAAE,mBAAmB;QAC7B,UAAU,EAAE,MAAM;QAClB,OAAO,EAAE;YACP,MAAM,EAAE,SAAS,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YACrD,OAAO,EAAE,SAAS;YAClB,WAAW,EAAE,EAAE;SAChB;QACD,UAAU,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;QAC3B,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,KAAa,EAAE,OAAyB;IACrE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACrD,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,MAAM,MAAM,GAAc;QACxB,KAAK;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,OAAO;QACP,OAAO,EAAE;YACP,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;YACzD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;YACzD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;YAC1D,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;SAC9D;KACF,CAAC;IACF,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACxE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE;YAClC,aAAa,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;SACpD,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE;YAClC,aAAa,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;SACpD,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACnD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE;YAClC,aAAa,CAAC;gBACZ,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE;wBACX,EAAE,SAAS,EAAE,uBAAuB,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;qBACpE;iBACF;aACF,CAAC;SACH,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE;YAClC,aAAa,CAAC;gBACZ,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,MAAM;oBACf,WAAW,EAAE;wBACX;4BACE,SAAS,EAAE,uBAAuB;4BAClC,IAAI,EAAE,KAAK;4BACX,MAAM,EAAE,WAAW;yBACpB;qBACF;iBACF;aACF,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,uBAAuB,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,MAAM,CAAC,KAAK,CACV,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,uBAAuB,CAAC,CAAC,CAAE,CAAC,SAAS,EACzD,uBAAuB,CACxB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE;YAClC,aAAa,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;SACtD,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE;YAClC,aAAa,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;SACtD,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oFAAoF,EAAE,GAAG,EAAE;QAC5F,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE;YAClC,aAAa,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;SACpD,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE;YAClC,aAAa,CAAC;gBACZ,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,OAAO;gBACf,UAAU,EAAE,GAAG;gBACf,KAAK,EAAE,iBAAiB;gBACxB,OAAO,EAAE,IAAI;aACd,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,OAAO;YACd,MAAM,EAAE,aAAa,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;SAC5D,CAAC;QACF,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,OAAO;YACd,MAAM,EAAE,aAAa,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;SAC5D,CAAC;QACF,MAAM,SAAS,GAAG;YAChB,KAAK,EAAE,OAAO;YACd,MAAM,EAAE,aAAa,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;SAC5D,CAAC;QAEF,MAAM,QAAQ,GAAG,wBAAwB,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/analyze/diff-hunks.d.ts

@@ -0,0 +1,16 @@
+export interface DiffHunk {
+    /** 0-based line index in the original file where removed lines start (or insertion point). */
+    oldStart: number;
+    removed: string[];
+    added: string[];
+    contextBefore: string[];
+    contextAfter: string[];
+}
+export declare function computeDiffHunks(before: string, after: string): DiffHunk[];
+export declare function applyHunks(original: string, hunks: DiffHunk[], acceptedIndices: Set<number>): string;
+export declare function formatHunkHeader(hunk: DiffHunk, index: number, total: number, filePath: string): string;
+export declare function formatHunkBody(hunk: DiffHunk): string;
+export declare function hunkPatchLines(hunk: DiffHunk): string[];
+export declare function canSplitHunk(hunk: DiffHunk): boolean;
+export declare function splitHunk(hunk: DiffHunk): [DiffHunk, DiffHunk] | null;
+//# sourceMappingURL=diff-hunks.d.ts.map

package/dist/analyze/diff-hunks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff-hunks.d.ts","sourceRoot":"","sources":["../../src/analyze/diff-hunks.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,QAAQ;IACvB,8FAA8F;IAC9F,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAwDD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,QAAQ,EAAE,CA0F1E;AAED,wBAAgB,UAAU,CACxB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,QAAQ,EAAE,EACjB,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,GAC3B,MAAM,CAeR;AAED,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,QAAQ,EACd,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,GACf,MAAM,CAMR;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAOrD;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,EAAE,CAEvD;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAmBpD;AA0CD,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,CAmErE"}