scai 0.1.178 → 1.0.1

package/dist/__tests__/EvalReportCmd.test.js

@@ -0,0 +1,645 @@
+/**
+ * Covers eval-report behavior for latest-run log parsing and path-aware flow checks.
+ *
+ * Why this exists:
+ * - `eval-report` now combines persisted task history with the latest run log
+ * - direct-answer and explicit-target runs must not look broken just because they skip task steps
+ * - the report should degrade cleanly when no matching run log is available
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+const state = {
+    latestTaskId: 0,
+    tasks: new Map(),
+    taskSteps: new Map(),
+    runLog: "",
+    runLogExists: false,
+    dirEntries: new Map(),
+    fileContents: new Map(),
+};
+function resetMockState() {
+    state.latestTaskId = null;
+    state.tasks = new Map();
+    state.taskSteps = new Map();
+    state.runLog = "";
+    state.runLogExists = false;
+    state.dirEntries = new Map();
+    state.fileContents = new Map();
+}
+function installDbMocks() {
+    vi.doMock("../db/client.js", () => ({
+        getDbForRepo: vi.fn(() => ({
+            prepare: vi.fn((sql) => ({
+                get: (taskId) => {
+                    if (sql.includes("ORDER BY id DESC LIMIT 1")) {
+                        return typeof state.latestTaskId === "number" ? { id: state.latestTaskId } : undefined;
+                    }
+                    if (sql.includes("SELECT * FROM tasks WHERE id = ?")) {
+                        return taskId != null ? state.tasks.get(taskId) : undefined;
+                    }
+                    return undefined;
+                },
+                all: (taskId) => {
+                    if (sql.includes("FROM steps")) {
+                        return state.taskSteps.get(taskId) ?? [];
+                    }
+                    return [];
+                },
+            })),
+        })),
+    }));
+}
+function installFsMocks() {
+    const readFileSync = vi.fn((filePath) => {
+        if (state.fileContents.has(filePath))
+            return state.fileContents.get(filePath) ?? "";
+        return state.runLog;
+    });
+    const existsSync = vi.fn((filePath) => {
+        if (state.fileContents.has(filePath) || state.dirEntries.has(filePath))
+            return true;
+        return state.runLogExists;
+    });
+    const readdirSync = vi.fn((dirPath, options) => {
+        const entries = state.dirEntries.get(dirPath) ?? [];
+        if (options?.withFileTypes) {
+            return entries.map((name) => ({
+                name,
+                isDirectory: () => true,
+            }));
+        }
+        return entries;
+    });
+    vi.doMock("fs", () => ({
+        default: {
+            existsSync,
+            readFileSync,
+            readdirSync,
+        },
+        existsSync,
+        readFileSync,
+        readdirSync,
+    }));
+}
+function installRepoIdentityMock() {
+    vi.doMock("../utils/repoIdentity.js", () => ({
+        resolveCanonicalRepoIdentity: vi.fn(() => ({
+            repoRootPath: "/repo",
+        })),
+    }));
+}
+function routingDecisionJson(overrides) {
+    return JSON.stringify({
+        decision: "has-info",
+        allowSearch: false,
+        allowResearch: false,
+        allowTransform: false,
+        scopeLocked: true,
+        confidence: 0.9,
+        rationale: "route=explicit-target",
+        ...overrides,
+    });
+}
+function payload(step, value) {
+    return [
+        "",
+        "====================================================================",
+        `📂 OUTPUT | ${step}`,
+        "====================================================================",
+        typeof value === "string" ? value : JSON.stringify(value, null, 2),
+        "====================================================================",
+        "",
+    ].join("\n");
+}
+async function loadModule() {
+    vi.resetModules();
+    installDbMocks();
+    installFsMocks();
+    installRepoIdentityMock();
+    return import("../commands/EvalReportCmd.js");
+}
+function captureConsole(run) {
+    const spy = vi.spyOn(console, "log").mockImplementation(() => { });
+    try {
+        run();
+        return spy.mock.calls.map((args) => args.join(" ")).join("\n");
+    }
+    finally {
+        spy.mockRestore();
+    }
+}
+describe("EvalReportCmd", () => {
+    beforeEach(() => {
+        resetMockState();
+        vi.resetModules();
+    });
+    it("reports explicit-target analysis runs without false failures", async () => {
+        state.latestTaskId = 12;
+        state.tasks.set(12, {
+            id: 12,
+            status: "completed",
+            initial_query: "Explain routing in MainAgent.ts",
+            summary: "MainAgent.ts routing summary",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: false,
+                allowTransform: false,
+                rationale: "route=explicit-target",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(12, []);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("runtimeState", { phase: "read", lane: "coding", status: "active" }),
+            payload("runtimeState", { phase: "understand", lane: "coding", status: "active" }),
+            payload("resolveExplicitTargetsStep", {
+                resolvedTargetFiles: ["/repo/cli/src/agents/MainAgent.ts"],
+                enteredBroadFallback: false,
+            }),
+            payload("runtimeState", { phase: "route", lane: "explicit-target", status: "active" }),
+            payload("runtimeState", { phase: "evidence", lane: "explicit-target", status: "active" }),
+            payload("decideNextAction", {
+                decision: {
+                    action: "continue",
+                    reason: "Research step pending: research-impact-map",
+                    confidence: 0.95,
+                    targetFile: "__research__/impact-map",
+                    source: "reason-next-step-selection",
+                },
+                validation: {
+                    accepted: true,
+                    normalizedAction: "continue",
+                },
+            }),
+            payload("runtimeState", { phase: "finalize", lane: "explicit-target", status: "active" }),
+            payload("finalAnswerModule", { data: "MainAgent.ts routing summary" }),
+        ].join("");
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(12, { mermaid: false }));
+        expect(output).toContain("Lane: explicit-target");
+        expect(output).toContain("Answer path: finalAnswerModule");
+        expect(output).toContain("explicit target resolution: resolved=1, broadFallback=false");
+        expect(output).toContain("next action decision: continue (Research step pending: research-impact-map)");
+        expect(output).toContain("PASS execution-path-valid");
+        expect(output).not.toContain("FAIL execution-path-valid");
+        expect(output).toContain("Diagnosis:");
+        expect(output).toContain("Category: unknown");
+    });
+    it("shows seed-plan decisions in the route timeline", async () => {
+        state.latestTaskId = 24;
+        state.tasks.set(24, {
+            id: 24,
+            status: "completed",
+            initial_query: "Refactor MainAgent.ts",
+            summary: "planned and finished",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: false,
+                allowResearch: false,
+                allowTransform: true,
+                rationale: "route=explicit-target",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(24, [
+            {
+                id: 1,
+                file_path: "/repo/cli/src/agents/MainAgent.ts",
+                action: null,
+                status: "completed",
+                step_index: 1,
+                start_time: 1,
+                end_time: 2,
+            },
+        ]);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("runtimeState", { phase: "route", lane: "explicit-target", status: "active" }),
+            payload("runtimeState", { phase: "execute", lane: "explicit-target", status: "active" }),
+            payload("decideNextAction", {
+                decision: {
+                    action: "seed-plan",
+                    reason: "No eligible execution step exists yet; seed the current execution plan first.",
+                    confidence: 0.9,
+                    source: "reason-next-step-selection",
+                },
+                validation: {
+                    accepted: true,
+                    normalizedAction: "seed-plan",
+                },
+            }),
+            payload("finalAnswerModule", { data: "planned and finished" }),
+        ].join("");
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(24, { mermaid: false }));
+        expect(output).toContain("next action decision: seed-plan (No eligible execution step exists yet; seed the current execution plan first.)");
+    });
+    it("shows run-search decisions in the route timeline", async () => {
+        state.latestTaskId = 27;
+        state.tasks.set(27, {
+            id: 27,
+            status: "completed",
+            initial_query: "Refactor MainAgent.ts",
+            summary: "searched and finished",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: true,
+                allowResearch: true,
+                allowTransform: true,
+                rationale: "route=needs-info",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(27, []);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("runtimeState", { phase: "route", lane: "evidence", status: "active" }),
+            payload("runtimeState", { phase: "execute", lane: "evidence", status: "active" }),
+            payload("decideNextAction", {
+                decision: {
+                    action: "run-search",
+                    reason: "Initial evidence bootstrap has not run yet; run search before verify or execution seeding.",
+                    confidence: 0.93,
+                    source: "reason-next-step-selection",
+                },
+                validation: {
+                    accepted: true,
+                    normalizedAction: "run-search",
+                },
+            }),
+            payload("finalAnswerModule", { data: "searched and finished" }),
+        ].join("");
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(27, { mermaid: false }));
+        expect(output).toContain("next action decision: run-search (Initial evidence bootstrap has not run yet; run search before verify or execution seeding.)");
+    });
+    it("shows run-verify decisions in the route timeline", async () => {
+        state.latestTaskId = 26;
+        state.tasks.set(26, {
+            id: 26,
+            status: "completed",
+            initial_query: "Refactor MainAgent.ts",
+            summary: "verified and finished",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: false,
+                allowResearch: true,
+                allowTransform: true,
+                rationale: "route=explicit-target",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(26, [
+            {
+                id: 1,
+                file_path: "/repo/cli/src/agents/MainAgent.ts",
+                action: null,
+                status: "completed",
+                step_index: 1,
+                start_time: 1,
+                end_time: 2,
+            },
+        ]);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("runtimeState", { phase: "route", lane: "explicit-target", status: "active" }),
+            payload("runtimeState", { phase: "execute", lane: "explicit-target", status: "active" }),
+            payload("decideNextAction", {
+                decision: {
+                    action: "run-verify",
+                    reason: "Execution is not ready yet; run verify before seeding research or execution work.",
+                    confidence: 0.92,
+                    source: "reason-next-step-selection",
+                },
+                validation: {
+                    accepted: true,
+                    normalizedAction: "run-verify",
+                },
+            }),
+            payload("finalAnswerModule", { data: "verified and finished" }),
+        ].join("");
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(26, { mermaid: false }));
+        expect(output).toContain("next action decision: run-verify (Execution is not ready yet; run verify before seeding research or execution work.)");
+    });
+    it("shows seed-research decisions in the route timeline", async () => {
+        state.latestTaskId = 25;
+        state.tasks.set(25, {
+            id: 25,
+            status: "completed",
+            initial_query: "Refactor MainAgent.ts",
+            summary: "researched and finished",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: false,
+                allowResearch: true,
+                allowTransform: true,
+                rationale: "route=explicit-target",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(25, [
+            {
+                id: 1,
+                file_path: "__research__/impact-map",
+                action: "research-impact-map",
+                status: "completed",
+                step_index: 1,
+                start_time: 1,
+                end_time: 2,
+            },
+        ]);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("runtimeState", { phase: "route", lane: "explicit-target", status: "active" }),
+            payload("runtimeState", { phase: "execute", lane: "explicit-target", status: "active" }),
+            payload("decideNextAction", {
+                decision: {
+                    action: "seed-research",
+                    reason: "No research step exists yet; seed research before execution planning.",
+                    confidence: 0.9,
+                    source: "reason-next-step-selection",
+                },
+                validation: {
+                    accepted: true,
+                    normalizedAction: "seed-research",
+                },
+            }),
+            payload("finalAnswerModule", { data: "researched and finished" }),
+        ].join("");
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(25, { mermaid: false }));
+        expect(output).toContain("next action decision: seed-research (No research step exists yet; seed research before execution planning.)");
+    });
+    it("reports resume-continuity direct answers without requiring execution steps", async () => {
+        state.latestTaskId = 21;
+        state.tasks.set(21, {
+            id: 21,
+            status: "completed",
+            initial_query: "where were we?",
+            summary: "resume summary",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: false,
+                allowTransform: false,
+                rationale: "route=resume-continuity",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(21, []);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("runtimeState", { phase: "read", lane: "coding", status: "active" }),
+            payload("runtimeState", { phase: "understand", lane: "coding", status: "active" }),
+            payload("resumeContinuityStep", { kind: "continuity-summary" }),
+            payload("continuityRoutingAnalysisStep", {
+                kind: "direct-resume-answer",
+                answerNow: true,
+            }),
+            payload("runtimeState", { phase: "route", lane: "resume-continuity", status: "active" }),
+            payload("runtimeState", { phase: "finalize", lane: "resume-continuity", status: "ready" }),
+            payload("continuityAnswerModule", { data: "direct answer" }),
+        ].join("");
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(21, { mermaid: false }));
+        expect(output).toContain("Lane: resume-continuity");
+        expect(output).toContain("Answer path: continuityAnswerModule");
+        expect(output).toContain("PASS direct-answer-path-valid");
+        expect(output).toContain("PASS execution-path-valid");
+        expect(output).toContain("resume continuity intent: continuity-summary");
+        expect(output).toContain("Category: unknown");
+    });
+    it("prints persistence diagnosis for final-answer deferred runs", async () => {
+        state.latestTaskId = 22;
+        state.tasks.set(22, {
+            id: 22,
+            status: "deferred",
+            initial_query: "Explain routing in MainAgent.ts",
+            summary: "answer",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: false,
+                allowTransform: false,
+                rationale: "route=explicit-target",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(22, []);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("resolveExplicitTargetsStep", {
+                resolvedTargetFiles: ["/repo/cli/src/agents/MainAgent.ts"],
+                enteredBroadFallback: false,
+            }),
+            payload("runtimeState", { phase: "finalize", lane: "explicit-target", status: "active" }),
+            payload("finalAnswerModule", { data: "answer" }),
+        ].join("");
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(22, { mermaid: false }));
+        expect(output).toContain("Category: persistence");
+        expect(output).toContain("task status deferred");
+    });
+    it("treats completed zero-step final answers as valid answer-only runs", async () => {
+        state.latestTaskId = 23;
+        state.tasks.set(23, {
+            id: 23,
+            status: "completed",
+            initial_query: "Explain cli/src/agents/MainAgent.ts",
+            summary: "MainAgent.ts answer",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: false,
+                allowTransform: false,
+                rationale: "route=explicit-target",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(23, []);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("resolveExplicitTargetsStep", {
+                resolvedTargetFiles: ["/repo/cli/src/agents/MainAgent.ts"],
+                enteredBroadFallback: false,
+            }),
+            payload("runtimeState", { phase: "finalize", lane: "explicit-target", status: "complete" }),
+            payload("finalAnswerModule", { data: "MainAgent.ts answer" }),
+        ].join("");
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(23, { mermaid: false }));
+        expect(output).toContain("PASS execution-path-valid");
+        expect(output).toContain("Category: unknown");
+    });
+    it("shows runtime-state timeline and mermaid route shape for execution runs", async () => {
+        state.latestTaskId = 31;
+        state.tasks.set(31, {
+            id: 31,
+            status: "completed",
+            initial_query: "Add comments",
+            summary: "done",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: true,
+                allowResearch: true,
+                allowTransform: true,
+                rationale: "route=single-file-focused",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(31, [
+            {
+                id: 1,
+                file_path: "/repo/cli/src/file.ts",
+                action: "codeTransform",
+                status: "completed",
+                step_index: 1,
+                start_time: null,
+                end_time: null,
+            },
+        ]);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("runtimeState", { phase: "read", lane: "coding", status: "active", rationale: "Boot" }),
+            payload("runtimeState", { phase: "understand", lane: "coding", status: "active", rationale: "Understand" }),
+            payload("runtimeState", { phase: "route", lane: "coding", status: "active", rationale: "Route" }),
+            payload("runtimeState", { phase: "evidence", lane: "coding", status: "ready", rationale: "Evidence" }),
+            payload("runtimeState", { phase: "execute", lane: "coding", status: "complete", rationale: "Execute" }),
+            payload("runtimeState", { phase: "finalize", lane: "coding", status: "active", rationale: "Finalize" }),
+            payload("finalAnswerModule", { data: "answer" }),
+        ].join("");
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(31));
+        expect(output).toContain("Route Timeline:");
+        expect(output).toContain("- read -> coding [active] (Boot)");
+        expect(output).toContain("Mermaid:");
+        expect(output).toContain("evidence[\"Evidence");
+        expect(output).toContain("execute[\"Execute");
+        expect(output).toContain("Answer\\nfinalAnswerModule");
+    });
+    it("builds Markdown reports with fenced Mermaid blocks", async () => {
+        state.latestTaskId = 32;
+        state.tasks.set(32, {
+            id: 32,
+            status: "completed",
+            initial_query: "summarize architecture",
+            summary: "done",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: true,
+                allowResearch: true,
+                allowTransform: false,
+                rationale: "route=repo-research",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(32, []);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("runtimeState", { phase: "read", lane: "coding", status: "active" }),
+            payload("runtimeState", { phase: "route", lane: "evidence", status: "active" }),
+            payload("runtimeState", { phase: "evidence", lane: "evidence", status: "active" }),
+            payload("finalAnswerModule", { data: "answer" }),
+        ].join("");
+        const mod = await loadModule();
+        const markdown = mod.buildEvalReportMarkdown(32, { title: "Integration Report | 32" });
+        expect(markdown).toContain("# Integration Report | 32");
+        expect(markdown).toContain("## Mermaid");
+        expect(markdown).toContain("```mermaid");
+        expect(markdown).toContain("flowchart TD");
+        expect(markdown).toContain("## Flow Eval");
+        expect(markdown).toContain("## Context Eval");
+    });
+    it("renders context warnings from saved llm trace files", async () => {
+        state.latestTaskId = 33;
+        state.tasks.set(33, {
+            id: 33,
+            status: "gave-up",
+            initial_query: "summarize this repo architecture and identify weak coupling points",
+            summary: "architecture answer",
+            routing_decision_json: routingDecisionJson({
+                allowSearch: true,
+                allowResearch: true,
+                allowTransform: false,
+                rationale: "route=repo-research",
+            }),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(33, []);
+        state.runLogExists = true;
+        state.runLog = [
+            payload("runResearch", {
+                result: {
+                    architectureInputFiles: ["/repo/a.ts", "/repo/b.ts", "/repo/c.ts"],
+                    priorResearchSummaries: [{ action: "research-impact-map", summary: "Impact map across files." }],
+                },
+                stepReasoning: {
+                    rationale: "Research step completed: Architecture synthesis completed from 3 analyzed file(s).",
+                },
+            }),
+            payload("finalAnswerModule", { data: "architecture answer" }),
+        ].join("");
+        state.dirEntries.set("/artifact/llm_calls", ["0018_finalAnswerModule.json"]);
+        state.fileContents.set("/artifact/llm_calls/0018_finalAnswerModule.json", JSON.stringify({
+            callId: "llm-0018-finalAnswerModule",
+            caller: "finalAnswerModule",
+            query: "summarize this repo architecture and identify weak coupling points",
+            promptText: [
+                "User query:",
+                "summarize this repo architecture and identify weak coupling points",
+                "User query:",
+                "summarize this repo architecture and identify weak coupling points",
+                "==================== PROPOSED CHANGES ====================",
+            ].join("\n"),
+            inputContext: {
+                analyzedFiles: ["/repo/a.ts"],
+                combinedSection: "[No combined analysis available]",
+            },
+        }, null, 2));
+        const mod = await loadModule();
+        const markdown = mod.buildEvalReportMarkdown(33, { artifactDir: "/artifact", mermaid: false });
+        expect(markdown).toContain("## Context Eval");
+        expect(markdown).toContain("WARN missing-high-value-context");
+        expect(markdown).toContain("WARN redundant-low-signal-context");
+        expect(markdown).toContain("WARN prompt-stage-mismatch");
+        expect(markdown).toContain("/artifact/llm_calls/0018_finalAnswerModule.json");
+    });
+    it("keeps Mermaid node labels on one line with literal newline escapes", async () => {
+        state.latestTaskId = 52;
+        state.tasks.set(52, {
+            id: 52,
+            status: "completed",
+            initial_query: "Explain routing in MainAgent.ts",
+            summary: "routing summary",
+            routing_decision_json: routingDecisionJson(),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(52, []);
+        const mod = await loadModule();
+        const markdown = mod.buildEvalReportMarkdown(52, { mermaid: true });
+        expect(markdown).toContain('boot["Boot\\nTask 52"]');
+        expect(markdown).toContain('understand["Understand\\nlane=unknown"]');
+        expect(markdown).not.toContain('boot["Boot\\\nTask 52"]');
+    });
+    it("degrades cleanly when run-log data is missing for a non-latest task", async () => {
+        state.latestTaskId = 41;
+        state.tasks.set(40, {
+            id: 40,
+            status: "completed",
+            initial_query: "Older task",
+            summary: "done",
+            routing_decision_json: routingDecisionJson(),
+            created_at: "now",
+            updated_at: "now",
+        });
+        state.taskSteps.set(40, []);
+        state.runLogExists = true;
+        state.runLog = payload("runtimeState", { phase: "read", lane: "coding", status: "active" });
+        const mod = await loadModule();
+        const output = captureConsole(() => mod.printEvalReport(40, { mermaid: false }));
+        expect(output).toContain("Lane: unknown");
+        expect(output).toContain("Answer path: unknown");
+        expect(output).toContain("No run-log timeline available for this task.");
+    });
+});
+//# sourceMappingURL=EvalReportCmd.test.js.map