scai 0.1.178 → 1.0.0

package/dist/__tests__/testing/contextEval.test.js

@@ -0,0 +1,244 @@
+/**
+ * Verifies that context eval flags repeated planner prompts, prompt bloat, and missing decomposition.
+ *
+ * Why this exists:
+ * - context audit rules should stay deterministic across saved trace shapes
+ * - planner retries and final-answer prompt growth are easy to reintroduce during routing work
+ * - multi-question queries need explicit regression coverage so they do not collapse silently
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+const state = vi.hoisted(() => ({
+    dirEntries: new Map(),
+    fileContents: new Map(),
+}));
+vi.mock("fs", () => ({
+    default: {
+        existsSync: vi.fn((filePath) => state.fileContents.has(filePath) || state.dirEntries.has(filePath)),
+        readFileSync: vi.fn((filePath) => state.fileContents.get(filePath) ?? ""),
+        readdirSync: vi.fn((dirPath, options) => {
+            const entries = state.dirEntries.get(dirPath) ?? [];
+            if (options?.withFileTypes) {
+                return entries.map((name) => ({
+                    name,
+                    isDirectory: () => true,
+                }));
+            }
+            return entries;
+        }),
+    },
+    existsSync: vi.fn((filePath) => state.fileContents.has(filePath) || state.dirEntries.has(filePath)),
+    readFileSync: vi.fn((filePath) => state.fileContents.get(filePath) ?? ""),
+    readdirSync: vi.fn((dirPath, options) => {
+        const entries = state.dirEntries.get(dirPath) ?? [];
+        if (options?.withFileTypes) {
+            return entries.map((name) => ({
+                name,
+                isDirectory: () => true,
+            }));
+        }
+        return entries;
+    }),
+}));
+vi.mock("../../utils/repoIdentity.js", () => ({
+    resolveCanonicalRepoIdentity: vi.fn(() => ({
+        repoRootPath: "/repo",
+    })),
+}));
+import { buildContextEval } from "../../testing/contextEval.js";
+describe("contextEval", () => {
+    beforeEach(() => {
+        state.dirEntries = new Map();
+        state.fileContents = new Map();
+    });
+    it("warns when planner retries reuse identical prompts", () => {
+        state.dirEntries.set("/artifact/llm_calls", [
+            "0006_analysisPlanGenStep.json",
+            "0007_analysisPlanGenStep.json",
+        ]);
+        const promptText = "User query:\nsummarize architecture\nAllowed actions:\n[semanticAnalysis]";
+        for (const fileName of ["0006_analysisPlanGenStep.json", "0007_analysisPlanGenStep.json"]) {
+            state.fileContents.set(`/artifact/llm_calls/${fileName}`, JSON.stringify({
+                callId: `llm-${fileName.slice(0, 4)}-analysisPlanGenStep`,
+                caller: "analysisPlanGenStep",
+                query: "summarize architecture",
+                promptText,
+                inputContext: {},
+            }, null, 2));
+        }
+        const result = buildContextEval({ taskId: 1, artifactDir: "/artifact", runLog: "" });
+        const duplicateCheck = result.checks.find((check) => check.name === "duplicate-replan-calls");
+        expect(duplicateCheck?.ok).toBe(false);
+        expect(duplicateCheck?.finding?.actualContext).toContain("2 identical analysisPlanGenStep prompts");
+    });
+    it("warns when progress state records stalled planner retries", () => {
+        state.dirEntries.set("/artifact/llm_calls", ["0001_analysisPlanGenStep.json"]);
+        state.fileContents.set("/artifact/llm_calls/0001_analysisPlanGenStep.json", JSON.stringify({
+            callId: "llm-0001-analysisPlanGenStep",
+            caller: "analysisPlanGenStep",
+            query: "Explain how tests run",
+            promptText: "Prompt",
+            inputContext: {},
+        }, null, 2));
+        const divider = "=".repeat(68);
+        const runLog = [
+            "📂 OUTPUT | progressState",
+            divider,
+            JSON.stringify({
+                research: {
+                    status: "optional",
+                    pendingSteps: [],
+                    noNewInfoCount: 0,
+                    reason: "No research is currently required. Planner retried the same target without frontier change (3 identical attempt(s)).",
+                    plannerRetry: {
+                        targetFile: "/repo/cli/src/testing/testCommands.ts",
+                        signature: "sig:testCommands",
+                        count: 3,
+                        stalled: true,
+                        reason: "Planner retried the same target without frontier change (3 identical attempt(s)).",
+                    },
+                },
+            }, null, 2),
+            divider,
+            "",
+        ].join("\n");
+        const result = buildContextEval({ taskId: 1, artifactDir: "/artifact", runLog });
+        const stalledCheck = result.checks.find((check) => check.name === "stalled-planner-progress");
+        expect(stalledCheck?.ok).toBe(false);
+        expect(stalledCheck?.finding?.actualContext).toContain("target=/repo/cli/src/testing/testCommands.ts");
+        expect(stalledCheck?.finding?.actualContext).toContain("count=3");
+    });
+    it("warns when a continuity query carries a large resumed capsule", () => {
+        state.dirEntries.set("/artifact/llm_calls", ["0008_scopeClassificationStep.json"]);
+        state.fileContents.set("/artifact/llm_calls/0008_scopeClassificationStep.json", JSON.stringify({
+            callId: "llm-0008-scopeClassificationStep",
+            caller: "scopeClassificationStep",
+            query: "where were we?",
+            promptText: "User query:\nwhere were we?",
+            inputContext: {
+                resumedContextCapsule: "Original task:\n" + "x".repeat(400),
+            },
+        }, null, 2));
+        const result = buildContextEval({ taskId: 1, artifactDir: "/artifact", runLog: "" });
+        const continuityCheck = result.checks.find((check) => check.name === "continuity-overprompting");
+        expect(continuityCheck?.ok).toBe(false);
+        expect(continuityCheck?.finding?.details).toContain("short continuity query");
+    });
+    it("warns when final grounding carries oversized supporting excerpts", () => {
+        state.dirEntries.set("/artifact/llm_calls", ["0010_finalAnswerModule.json"]);
+        state.fileContents.set("/artifact/llm_calls/0010_finalAnswerModule.json", JSON.stringify({
+            callId: "llm-0010-finalAnswerModule",
+            caller: "finalAnswerModule",
+            query: "Where are SQLite queries defined?",
+            promptText: "User query:\nWhere are SQLite queries defined?",
+            inputContext: {
+                supportingExcerpts: [
+                    {
+                        filePath: "/repo/cli/src/db/fileIndex.ts",
+                        excerpts: ["a".repeat(1600)],
+                    },
+                    {
+                        filePath: "/repo/cli/src/db/client.ts",
+                        excerpts: ["b".repeat(700)],
+                    },
+                ],
+            },
+        }, null, 2));
+        const result = buildContextEval({ taskId: 1, artifactDir: "/artifact", runLog: "" });
+        const sizeCheck = result.checks.find((check) => check.name === "oversized-final-grounding-context");
+        expect(sizeCheck?.ok).toBe(false);
+        expect(sizeCheck?.finding?.actualContext).toContain("supporting excerpts chars=2300");
+    });
+    it("warns when adjacent planner retries reuse the same rationale but change other prompt sections", () => {
+        state.dirEntries.set("/artifact/llm_calls", [
+            "0006_analysisPlanGenStep.json",
+            "0007_analysisPlanGenStep.json",
+        ]);
+        state.fileContents.set("/artifact/llm_calls/0006_analysisPlanGenStep.json", JSON.stringify({
+            callId: "llm-0006-analysisPlanGenStep",
+            caller: "analysisPlanGenStep",
+            query: "Where are SQLite queries defined in db/fileIndex.ts and db/client.ts?",
+            promptText: "Prompt A",
+            inputContext: {
+                targetFile: "/repo/cli/src/db/fileIndex.ts",
+                rationaleText: "Both files are relevant to the SQLite query definitions.",
+                allowedActions: [
+                    { actionId: "semanticAnalysis" },
+                    { actionId: "read-file" },
+                    { actionId: "research-impact-map" },
+                    { actionId: "research-symbol-trace" },
+                    { actionId: "research-risk-check" },
+                    { actionId: "research-architecture-synthesis" },
+                ],
+            },
+        }, null, 2));
+        state.fileContents.set("/artifact/llm_calls/0007_analysisPlanGenStep.json", JSON.stringify({
+            callId: "llm-0007-analysisPlanGenStep",
+            caller: "analysisPlanGenStep",
+            query: "Where are SQLite queries defined in db/fileIndex.ts and db/client.ts?",
+            promptText: "Prompt B",
+            inputContext: {
+                targetFile: "/repo/cli/src/db/fileIndex.ts",
+                rationaleText: "Both files are relevant to the SQLite query definitions.",
+                allowedActions: [
+                    { actionId: "semanticAnalysis" },
+                    { actionId: "read-file" },
+                    { actionId: "research-impact-map" },
+                    { actionId: "research-symbol-trace" },
+                    { actionId: "research-risk-check" },
+                    { actionId: "research-architecture-synthesis" },
+                ],
+            },
+        }, null, 2));
+        const result = buildContextEval({ taskId: 1, artifactDir: "/artifact", runLog: "" });
+        const rationaleCheck = result.checks.find((check) => check.name === "stale-planner-rationale");
+        const catalogCheck = result.checks.find((check) => check.name === "repeated-planner-action-catalog");
+        expect(rationaleCheck?.ok).toBe(false);
+        expect(rationaleCheck?.finding?.details).toContain("same focus rationale");
+        expect(catalogCheck?.ok).toBe(false);
+        expect(catalogCheck?.finding?.actualContext).toContain("6 actions repeated");
+    });
+    it("warns when a multi-part query never becomes ordered question parts", () => {
+        state.dirEntries.set("/artifact/llm_calls", [
+            "0001_understandIntentStep.json",
+            "0002_scopeClassificationStep.json",
+            "0006_analysisPlanGenStep.json",
+        ]);
+        state.fileContents.set("/artifact/llm_calls/0001_understandIntentStep.json", JSON.stringify({
+            callId: "llm-0001-understandIntentStep",
+            caller: "understandIntentStep",
+            query: "Where is search routed? How is verify triggered?",
+            promptText: "User query:\nWhere is search routed? How is verify triggered?",
+            inputContext: {
+                userQuery: "Where is search routed? How is verify triggered?",
+            },
+        }, null, 2));
+        state.fileContents.set("/artifact/llm_calls/0002_scopeClassificationStep.json", JSON.stringify({
+            callId: "llm-0002-scopeClassificationStep",
+            caller: "scopeClassificationStep",
+            query: "Where is search routed? How is verify triggered?",
+            promptText: "User query:\nWhere is search routed? How is verify triggered?",
+            inputContext: {
+                query: "Where is search routed? How is verify triggered?",
+            },
+        }, null, 2));
+        state.fileContents.set("/artifact/llm_calls/0006_analysisPlanGenStep.json", JSON.stringify({
+            callId: "llm-0006-analysisPlanGenStep",
+            caller: "analysisPlanGenStep",
+            query: "Where is search routed? How is verify triggered?",
+            promptText: "Prompt",
+            inputContext: {
+                orderedQuestions: [
+                    {
+                        id: "q1",
+                        text: "Where is search routed? How is verify triggered?",
+                    },
+                ],
+            },
+        }, null, 2));
+        const result = buildContextEval({ taskId: 1, artifactDir: "/artifact", runLog: "" });
+        const questionCheck = result.checks.find((check) => check.name === "missing-question-decomposition");
+        expect(questionCheck?.ok).toBe(false);
+        expect(questionCheck?.finding?.actualContext).toContain("max orderedQuestions=1");
+    });
+});
+//# sourceMappingURL=contextEval.test.js.map

package/dist/__tests__/testing/contextEval.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextEval.test.js","sourceRoot":"","sources":["../../../src/__tests__/testing/contextEval.test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9D,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9B,UAAU,EAAE,IAAI,GAAG,EAAoB;IACvC,YAAY,EAAE,IAAI,GAAG,EAAkB;CACxC,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACnB,OAAO,EAAE;QACP,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3G,YAAY,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjF,WAAW,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,OAAe,EAAE,OAAqC,EAAE,EAAE;YAC5E,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACpD,IAAI,OAAO,EAAE,aAAa,EAAE,CAAC;gBAC3B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAC5B,IAAI;oBACJ,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI;iBACxB,CAAC,CAAC,CAAC;YACN,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC;KACH;IACD,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3G,YAAY,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACjF,WAAW,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,OAAe,EAAE,OAAqC,EAAE,EAAE;QAC5E,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACpD,IAAI,OAAO,EAAE,aAAa,EAAE,CAAC;YAC3B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC5B,IAAI;gBACJ,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI;aACxB,CAAC,CAAC,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5C,4BAA4B,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QACzC,YAAY,EAAE,OAAO;KACtB,CAAC,CAAC;CACJ,CAAC,CAAC,CAAC;AAEJ,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAEhE,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,UAAU,CAAC,GAAG,EAAE;QACd,KAAK,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC7B,KAAK,CAAC,YAAY,GAAG,IAAI,GAAG,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,qBAAqB,EAAE;YAC1C,+BAA+B;YAC/B,+BAA+B;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,2EAA2E,CAAC;QAC/F,KAAK,MAAM,QAAQ,IAAI,CAAC,+BAA+B,EAAE,+BAA+B,CAAC,EAAE,CAAC;YAC1F,KAAK,CAAC,YAAY,CAAC,GAAG,CACpB,uBAAuB,QAAQ,EAAE,EACjC,IAAI,CAAC,SAAS,CACZ;gBACE,MAAM,EAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,sBAAsB;gBACzD,MAAM,EAAE,qBAAqB;gBAC7B,KAAK,EAAE,wBAAwB;gBAC/B,UAAU;gBACV,YAAY,EAAE,EAAE;aACjB,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,wBAAwB,CAAC,CAAC;QAE9F,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,CAAC,cAAc,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,SAAS,CAAC,yCAAyC,CAAC,CAAC;IACtG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,+BAA+B,CAAC,CAAC,CAAC;QAC/E,KAAK,CAAC,YAAY,CAAC,GAAG,CACpB,mDAAmD,EACnD,IAAI,CAAC,SAAS,CACZ;YACE,MAAM,EAAE,8BAA8B;YACtC,MAAM,EAAE,qBAAqB;YAC7B,KAAK,EAAE,uBAAuB;YAC9B,UAAU,EAAE,QAAQ;YACpB,YAAY,EAAE,EAAE;SACjB,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG;YACb,2BAA2B;YAC3B,OAAO;YACP,IAAI,CAAC,SAAS,CACZ;gBACE,QAAQ,EAAE;oBACR,MAAM,EAAE,UAAU;oBAClB,YAAY,EAAE,EAAE;oBAChB,cAAc,EAAE,CAAC;oBACjB,MAAM,EAAE,sHAAsH;oBAC9H,YAAY,EAAE;wBACZ,UAAU,EAAE,uCAAuC;wBACnD,SAAS,EAAE,kBAAkB;wBAC7B,KAAK,EAAE,CAAC;wBACR,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,mFAAmF;qBAC5F;iBACF;aACF,EACD,IAAI,EACJ,CAAC,CACF;YACD,OAAO;YACP,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,MAAM,GAAG,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QACjF,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,0BAA0B,CAAC,CAAC;QAE9F,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,CAAC,YAAY,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,SAAS,CAAC,8CAA8C,CAAC,CAAC;QACvG,MAAM,CAAC,YAAY,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,mCAAmC,CAAC,CAAC,CAAC;QACnF,KAAK,CAAC,YAAY,CAAC,GAAG,CACpB,uDAAuD,EACvD,IAAI,CAAC,SAAS,CACZ;YACE,MAAM,EAAE,kCAAkC;YAC1C,MAAM,EAAE,yBAAyB;YACjC,KAAK,EAAE,gBAAgB;YACvB,UAAU,EAAE,6BAA6B;YACzC,YAAY,EAAE;gBACZ,qBAAqB,EAAE,kBAAkB,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;aAC5D;SACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QAEF,MAAM,MAAM,GAAG,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,0BAA0B,CAAC,CAAC;QAEjG,MAAM,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,CAAC,eAAe,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC1E,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,6BAA6B,CAAC,CAAC,CAAC;QAC7E,KAAK,CAAC,YAAY,CAAC,GAAG,CACpB,iDAAiD,EACjD,IAAI,CAAC,SAAS,CACZ;YACE,MAAM,EAAE,4BAA4B;YACpC,MAAM,EAAE,mBAAmB;YAC3B,KAAK,EAAE,mCAAmC;YAC1C,UAAU,EAAE,gDAAgD;YAC5D,YAAY,EAAE;gBACZ,kBAAkB,EAAE;oBAClB;wBACE,QAAQ,EAAE,+BAA+B;wBACzC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;qBAC7B;oBACD;wBACE,QAAQ,EAAE,4BAA4B;wBACtC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;qBAC5B;iBACF;aACF;SACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QAEF,MAAM,MAAM,GAAG,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,mCAAmC,CAAC,CAAC;QAEpG,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC;IACxF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+FAA+F,EAAE,GAAG,EAAE;QACvG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,qBAAqB,EAAE;YAC1C,+BAA+B;YAC/B,+BAA+B;SAChC,CAAC,CAAC;QACH,KAAK,CAAC,YAAY,CAAC,GAAG,CACpB,mDAAmD,EACnD,IAAI,CAAC,SAAS,CACZ;YACE,MAAM,EAAE,8BAA8B;YACtC,MAAM,EAAE,qBAAqB;YAC7B,KAAK,EAAE,uEAAuE;YAC9E,UAAU,EAAE,UAAU;YACtB,YAAY,EAAE;gBACZ,UAAU,EAAE,+BAA+B;gBAC3C,aAAa,EAAE,0DAA0D;gBACzE,cAAc,EAAE;oBACd,EAAE,QAAQ,EAAE,kBAAkB,EAAE;oBAChC,EAAE,QAAQ,EAAE,WAAW,EAAE;oBACzB,EAAE,QAAQ,EAAE,qBAAqB,EAAE;oBACnC,EAAE,QAAQ,EAAE,uBAAuB,EAAE;oBACrC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;oBACnC,EAAE,QAAQ,EAAE,iCAAiC,EAAE;iBAChD;aACF;SACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,KAAK,CAAC,YAAY,CAAC,GAAG,CACpB,mDAAmD,EACnD,IAAI,CAAC,SAAS,CACZ;YACE,MAAM,EAAE,8BAA8B;YACtC,MAAM,EAAE,qBAAqB;YAC7B,KAAK,EAAE,uEAAuE;YAC9E,UAAU,EAAE,UAAU;YACtB,YAAY,EAAE;gBACZ,UAAU,EAAE,+BAA+B;gBAC3C,aAAa,EAAE,0DAA0D;gBACzE,cAAc,EAAE;oBACd,EAAE,QAAQ,EAAE,kBAAkB,EAAE;oBAChC,EAAE,QAAQ,EAAE,WAAW,EAAE;oBACzB,EAAE,QAAQ,EAAE,qBAAqB,EAAE;oBACnC,EAAE,QAAQ,EAAE,uBAAuB,EAAE;oBACrC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;oBACnC,EAAE,QAAQ,EAAE,iCAAiC,EAAE;iBAChD;aACF;SACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QAEF,MAAM,MAAM,GAAG,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,yBAAyB,CAAC,CAAC;QAC/F,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,iCAAiC,CAAC,CAAC;QAErG,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;QAC3E,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,CAAC,YAAY,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;QAC5E,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,qBAAqB,EAAE;YAC1C,gCAAgC;YAChC,mCAAmC;YACnC,+BAA+B;SAChC,CAAC,CAAC;QACH,KAAK,CAAC,YAAY,CAAC,GAAG,CACpB,oDAAoD,EACpD,IAAI,CAAC,SAAS,CACZ;YACE,MAAM,EAAE,+BAA+B;YACvC,MAAM,EAAE,sBAAsB;YAC9B,KAAK,EAAE,kDAAkD;YACzD,UAAU,EAAE,+DAA+D;YAC3E,YAAY,EAAE;gBACZ,SAAS,EAAE,kDAAkD;aAC9D;SACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,KAAK,CAAC,YAAY,CAAC,GAAG,CACpB,uDAAuD,EACvD,IAAI,CAAC,SAAS,CACZ;YACE,MAAM,EAAE,kCAAkC;YAC1C,MAAM,EAAE,yBAAyB;YACjC,KAAK,EAAE,kDAAkD;YACzD,UAAU,EAAE,+DAA+D;YAC3E,YAAY,EAAE;gBACZ,KAAK,EAAE,kDAAkD;aAC1D;SACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,KAAK,CAAC,YAAY,CAAC,GAAG,CACpB,mDAAmD,EACnD,IAAI,CAAC,SAAS,CACZ;YACE,MAAM,EAAE,8BAA8B;YACtC,MAAM,EAAE,qBAAqB;YAC7B,KAAK,EAAE,kDAAkD;YACzD,UAAU,EAAE,QAAQ;YACpB,YAAY,EAAE;gBACZ,gBAAgB,EAAE;oBAChB;wBACE,EAAE,EAAE,IAAI;wBACR,IAAI,EAAE,kDAAkD;qBACzD;iBACF;aACF;SACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QAEF,MAAM,MAAM,GAAG,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,gCAAgC,CAAC,CAAC;QAErG,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,CAAC,aAAa,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["/**\n * Verifies that context eval flags repeated planner prompts, prompt bloat, and missing decomposition.\n *\n * Why this exists:\n * - context audit rules should stay deterministic across saved trace shapes\n * - planner retries and final-answer prompt growth are easy to reintroduce during routing work\n * - multi-question queries need explicit regression coverage so they do not collapse silently\n */\nimport { beforeEach, describe, expect, it, vi } from \"vitest\";\n\nconst state = vi.hoisted(() => ({\n  dirEntries: new Map<string, string[]>(),\n  fileContents: new Map<string, string>(),\n}));\n\nvi.mock(\"fs\", () => ({\n  default: {\n    existsSync: vi.fn((filePath: string) => state.fileContents.has(filePath) || state.dirEntries.has(filePath)),\n    readFileSync: vi.fn((filePath: string) => state.fileContents.get(filePath) ?? \"\"),\n    readdirSync: vi.fn((dirPath: string, options?: { withFileTypes?: boolean }) => {\n      const entries = state.dirEntries.get(dirPath) ?? [];\n      if (options?.withFileTypes) {\n        return entries.map((name) => ({\n          name,\n          isDirectory: () => true,\n        }));\n      }\n      return entries;\n    }),\n  },\n  existsSync: vi.fn((filePath: string) => state.fileContents.has(filePath) || state.dirEntries.has(filePath)),\n  readFileSync: vi.fn((filePath: string) => state.fileContents.get(filePath) ?? \"\"),\n  readdirSync: vi.fn((dirPath: string, options?: { withFileTypes?: boolean }) => {\n    const entries = state.dirEntries.get(dirPath) ?? [];\n    if (options?.withFileTypes) {\n      return entries.map((name) => ({\n        name,\n        isDirectory: () => true,\n      }));\n    }\n    return entries;\n  }),\n}));\n\nvi.mock(\"../../utils/repoIdentity.js\", () => ({\n  resolveCanonicalRepoIdentity: vi.fn(() => ({\n    repoRootPath: \"/repo\",\n  })),\n}));\n\nimport { buildContextEval } from \"../../testing/contextEval.js\";\n\ndescribe(\"contextEval\", () => {\n  beforeEach(() => {\n    state.dirEntries = new Map();\n    state.fileContents = new Map();\n  });\n\n  it(\"warns when planner retries reuse identical prompts\", () => {\n    state.dirEntries.set(\"/artifact/llm_calls\", [\n      \"0006_analysisPlanGenStep.json\",\n      \"0007_analysisPlanGenStep.json\",\n    ]);\n    const promptText = \"User query:\\nsummarize architecture\\nAllowed actions:\\n[semanticAnalysis]\";\n    for (const fileName of [\"0006_analysisPlanGenStep.json\", \"0007_analysisPlanGenStep.json\"]) {\n      state.fileContents.set(\n        `/artifact/llm_calls/${fileName}`,\n        JSON.stringify(\n          {\n            callId: `llm-${fileName.slice(0, 4)}-analysisPlanGenStep`,\n            caller: \"analysisPlanGenStep\",\n            query: \"summarize architecture\",\n            promptText,\n            inputContext: {},\n          },\n          null,\n          2\n        )\n      );\n    }\n\n    const result = buildContextEval({ taskId: 1, artifactDir: \"/artifact\", runLog: \"\" });\n    const duplicateCheck = result.checks.find((check) => check.name === \"duplicate-replan-calls\");\n\n    expect(duplicateCheck?.ok).toBe(false);\n    expect(duplicateCheck?.finding?.actualContext).toContain(\"2 identical analysisPlanGenStep prompts\");\n  });\n\n  it(\"warns when progress state records stalled planner retries\", () => {\n    state.dirEntries.set(\"/artifact/llm_calls\", [\"0001_analysisPlanGenStep.json\"]);\n    state.fileContents.set(\n      \"/artifact/llm_calls/0001_analysisPlanGenStep.json\",\n      JSON.stringify(\n        {\n          callId: \"llm-0001-analysisPlanGenStep\",\n          caller: \"analysisPlanGenStep\",\n          query: \"Explain how tests run\",\n          promptText: \"Prompt\",\n          inputContext: {},\n        },\n        null,\n        2\n      )\n    );\n    const divider = \"=\".repeat(68);\n    const runLog = [\n      \"📂 OUTPUT | progressState\",\n      divider,\n      JSON.stringify(\n        {\n          research: {\n            status: \"optional\",\n            pendingSteps: [],\n            noNewInfoCount: 0,\n            reason: \"No research is currently required. Planner retried the same target without frontier change (3 identical attempt(s)).\",\n            plannerRetry: {\n              targetFile: \"/repo/cli/src/testing/testCommands.ts\",\n              signature: \"sig:testCommands\",\n              count: 3,\n              stalled: true,\n              reason: \"Planner retried the same target without frontier change (3 identical attempt(s)).\",\n            },\n          },\n        },\n        null,\n        2\n      ),\n      divider,\n      \"\",\n    ].join(\"\\n\");\n\n    const result = buildContextEval({ taskId: 1, artifactDir: \"/artifact\", runLog });\n    const stalledCheck = result.checks.find((check) => check.name === \"stalled-planner-progress\");\n\n    expect(stalledCheck?.ok).toBe(false);\n    expect(stalledCheck?.finding?.actualContext).toContain(\"target=/repo/cli/src/testing/testCommands.ts\");\n    expect(stalledCheck?.finding?.actualContext).toContain(\"count=3\");\n  });\n\n  it(\"warns when a continuity query carries a large resumed capsule\", () => {\n    state.dirEntries.set(\"/artifact/llm_calls\", [\"0008_scopeClassificationStep.json\"]);\n    state.fileContents.set(\n      \"/artifact/llm_calls/0008_scopeClassificationStep.json\",\n      JSON.stringify(\n        {\n          callId: \"llm-0008-scopeClassificationStep\",\n          caller: \"scopeClassificationStep\",\n          query: \"where were we?\",\n          promptText: \"User query:\\nwhere were we?\",\n          inputContext: {\n            resumedContextCapsule: \"Original task:\\n\" + \"x\".repeat(400),\n          },\n        },\n        null,\n        2\n      )\n    );\n\n    const result = buildContextEval({ taskId: 1, artifactDir: \"/artifact\", runLog: \"\" });\n    const continuityCheck = result.checks.find((check) => check.name === \"continuity-overprompting\");\n\n    expect(continuityCheck?.ok).toBe(false);\n    expect(continuityCheck?.finding?.details).toContain(\"short continuity query\");\n  });\n\n  it(\"warns when final grounding carries oversized supporting excerpts\", () => {\n    state.dirEntries.set(\"/artifact/llm_calls\", [\"0010_finalAnswerModule.json\"]);\n    state.fileContents.set(\n      \"/artifact/llm_calls/0010_finalAnswerModule.json\",\n      JSON.stringify(\n        {\n          callId: \"llm-0010-finalAnswerModule\",\n          caller: \"finalAnswerModule\",\n          query: \"Where are SQLite queries defined?\",\n          promptText: \"User query:\\nWhere are SQLite queries defined?\",\n          inputContext: {\n            supportingExcerpts: [\n              {\n                filePath: \"/repo/cli/src/db/fileIndex.ts\",\n                excerpts: [\"a\".repeat(1600)],\n              },\n              {\n                filePath: \"/repo/cli/src/db/client.ts\",\n                excerpts: [\"b\".repeat(700)],\n              },\n            ],\n          },\n        },\n        null,\n        2\n      )\n    );\n\n    const result = buildContextEval({ taskId: 1, artifactDir: \"/artifact\", runLog: \"\" });\n    const sizeCheck = result.checks.find((check) => check.name === \"oversized-final-grounding-context\");\n\n    expect(sizeCheck?.ok).toBe(false);\n    expect(sizeCheck?.finding?.actualContext).toContain(\"supporting excerpts chars=2300\");\n  });\n\n  it(\"warns when adjacent planner retries reuse the same rationale but change other prompt sections\", () => {\n    state.dirEntries.set(\"/artifact/llm_calls\", [\n      \"0006_analysisPlanGenStep.json\",\n      \"0007_analysisPlanGenStep.json\",\n    ]);\n    state.fileContents.set(\n      \"/artifact/llm_calls/0006_analysisPlanGenStep.json\",\n      JSON.stringify(\n        {\n          callId: \"llm-0006-analysisPlanGenStep\",\n          caller: \"analysisPlanGenStep\",\n          query: \"Where are SQLite queries defined in db/fileIndex.ts and db/client.ts?\",\n          promptText: \"Prompt A\",\n          inputContext: {\n            targetFile: \"/repo/cli/src/db/fileIndex.ts\",\n            rationaleText: \"Both files are relevant to the SQLite query definitions.\",\n            allowedActions: [\n              { actionId: \"semanticAnalysis\" },\n              { actionId: \"read-file\" },\n              { actionId: \"research-impact-map\" },\n              { actionId: \"research-symbol-trace\" },\n              { actionId: \"research-risk-check\" },\n              { actionId: \"research-architecture-synthesis\" },\n            ],\n          },\n        },\n        null,\n        2\n      )\n    );\n    state.fileContents.set(\n      \"/artifact/llm_calls/0007_analysisPlanGenStep.json\",\n      JSON.stringify(\n        {\n          callId: \"llm-0007-analysisPlanGenStep\",\n          caller: \"analysisPlanGenStep\",\n          query: \"Where are SQLite queries defined in db/fileIndex.ts and db/client.ts?\",\n          promptText: \"Prompt B\",\n          inputContext: {\n            targetFile: \"/repo/cli/src/db/fileIndex.ts\",\n            rationaleText: \"Both files are relevant to the SQLite query definitions.\",\n            allowedActions: [\n              { actionId: \"semanticAnalysis\" },\n              { actionId: \"read-file\" },\n              { actionId: \"research-impact-map\" },\n              { actionId: \"research-symbol-trace\" },\n              { actionId: \"research-risk-check\" },\n              { actionId: \"research-architecture-synthesis\" },\n            ],\n          },\n        },\n        null,\n        2\n      )\n    );\n\n    const result = buildContextEval({ taskId: 1, artifactDir: \"/artifact\", runLog: \"\" });\n    const rationaleCheck = result.checks.find((check) => check.name === \"stale-planner-rationale\");\n    const catalogCheck = result.checks.find((check) => check.name === \"repeated-planner-action-catalog\");\n\n    expect(rationaleCheck?.ok).toBe(false);\n    expect(rationaleCheck?.finding?.details).toContain(\"same focus rationale\");\n    expect(catalogCheck?.ok).toBe(false);\n    expect(catalogCheck?.finding?.actualContext).toContain(\"6 actions repeated\");\n  });\n\n  it(\"warns when a multi-part query never becomes ordered question parts\", () => {\n    state.dirEntries.set(\"/artifact/llm_calls\", [\n      \"0001_understandIntentStep.json\",\n      \"0002_scopeClassificationStep.json\",\n      \"0006_analysisPlanGenStep.json\",\n    ]);\n    state.fileContents.set(\n      \"/artifact/llm_calls/0001_understandIntentStep.json\",\n      JSON.stringify(\n        {\n          callId: \"llm-0001-understandIntentStep\",\n          caller: \"understandIntentStep\",\n          query: \"Where is search routed? How is verify triggered?\",\n          promptText: \"User query:\\nWhere is search routed? How is verify triggered?\",\n          inputContext: {\n            userQuery: \"Where is search routed? How is verify triggered?\",\n          },\n        },\n        null,\n        2\n      )\n    );\n    state.fileContents.set(\n      \"/artifact/llm_calls/0002_scopeClassificationStep.json\",\n      JSON.stringify(\n        {\n          callId: \"llm-0002-scopeClassificationStep\",\n          caller: \"scopeClassificationStep\",\n          query: \"Where is search routed? How is verify triggered?\",\n          promptText: \"User query:\\nWhere is search routed? How is verify triggered?\",\n          inputContext: {\n            query: \"Where is search routed? How is verify triggered?\",\n          },\n        },\n        null,\n        2\n      )\n    );\n    state.fileContents.set(\n      \"/artifact/llm_calls/0006_analysisPlanGenStep.json\",\n      JSON.stringify(\n        {\n          callId: \"llm-0006-analysisPlanGenStep\",\n          caller: \"analysisPlanGenStep\",\n          query: \"Where is search routed? How is verify triggered?\",\n          promptText: \"Prompt\",\n          inputContext: {\n            orderedQuestions: [\n              {\n                id: \"q1\",\n                text: \"Where is search routed? How is verify triggered?\",\n              },\n            ],\n          },\n        },\n        null,\n        2\n      )\n    );\n\n    const result = buildContextEval({ taskId: 1, artifactDir: \"/artifact\", runLog: \"\" });\n    const questionCheck = result.checks.find((check) => check.name === \"missing-question-decomposition\");\n\n    expect(questionCheck?.ok).toBe(false);\n    expect(questionCheck?.finding?.actualContext).toContain(\"max orderedQuestions=1\");\n  });\n});\n"]}

package/dist/__tests__/testing/harnessArtifacts.test.js

@@ -0,0 +1,124 @@
+/**
+ * Verifies that saved harness artifacts include diagnosis data and Markdown reports.
+ *
+ * Why this exists:
+ * - artifact folders should be reviewable without rerunning the agent
+ * - diagnosis.json is the quick pointer to the likely subsystem
+ * - report.md should preserve Mermaid blocks for artifact viewers
+ * - artifact folder timestamps should stay in local time with offset, not UTC `Z`
+ * - the writer should persist both files for saved runs
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+const state = vi.hoisted(() => ({
+    task: {
+        id: 7,
+        status: "deferred",
+        initial_query: "Explain cli/src/agents/MainAgent.ts",
+        summary: "answer",
+        routing_decision_json: JSON.stringify({
+            allowSearch: false,
+            allowResearch: false,
+            allowTransform: false,
+            rationale: "route=explicit-target",
+        }),
+    },
+    steps: [],
+    writes: [],
+}));
+vi.mock("fs", () => ({
+    default: {
+        mkdirSync: vi.fn(),
+        writeFileSync: vi.fn((filePath, content) => {
+            state.writes.push({ path: filePath, content: String(content) });
+        }),
+    },
+    mkdirSync: vi.fn(),
+    writeFileSync: vi.fn((filePath, content) => {
+        state.writes.push({ path: filePath, content: String(content) });
+    }),
+}));
+vi.mock("../../db/client.js", () => ({
+    getDbForRepo: vi.fn(() => ({
+        prepare: vi.fn((sql) => ({
+            get: () => (sql.includes("FROM tasks") ? state.task : undefined),
+            all: () => (sql.includes("FROM steps") ? state.steps : []),
+        })),
+    })),
+}));
+vi.mock("../../utils/repoIdentity.js", () => ({
+    resolveCanonicalRepoIdentity: vi.fn(() => ({
+        repoRootPath: "/repo",
+    })),
+}));
+import { writeHarnessArtifacts } from "../../testing/harnessArtifacts.js";
+describe("harnessArtifacts", () => {
+    beforeEach(() => {
+        state.writes = [];
+    });
+    it("writes diagnosis.json for saved runs", () => {
+        writeHarnessArtifacts({
+            kind: "evals",
+            caseId: "diagnosis_case",
+            query: "Explain cli/src/agents/MainAgent.ts",
+            taskId: 7,
+            runLog: [
+                "",
+                "====================================================================",
+                "📂 OUTPUT | finalAnswerModule",
+                "====================================================================",
+                JSON.stringify({ data: "answer" }, null, 2),
+                "====================================================================",
+                "",
+            ].join("\n"),
+            finalAnswerText: "answer",
+            result: {
+                id: "diagnosis_case",
+                passed: false,
+                failures: [],
+            },
+        });
+        const diagnosisWrite = state.writes.find((entry) => entry.path.endsWith("/diagnosis.json"));
+        expect(diagnosisWrite).toBeDefined();
+        expect(diagnosisWrite?.content).toContain("\"primaryCategory\": \"persistence\"");
+    });
+    it("writes report.md when report text is provided", () => {
+        writeHarnessArtifacts({
+            kind: "integration",
+            caseId: "markdown_report",
+            query: "summarize repo architecture",
+            taskId: 7,
+            runLog: "",
+            finalAnswerText: "answer",
+            reportText: "# Integration Report | 7\n\n```mermaid\nflowchart TD\n```",
+            result: {
+                id: "markdown_report",
+                passed: true,
+                failures: [],
+            },
+        });
+        const reportWrite = state.writes.find((entry) => entry.path.endsWith("/report.md"));
+        expect(reportWrite).toBeDefined();
+        expect(reportWrite?.content).toContain("```mermaid");
+    });
+    it("uses a local-time offset timestamp in generated artifact paths", () => {
+        writeHarnessArtifacts({
+            kind: "evals",
+            caseId: "timestamp_case",
+            query: "Explain cli/src/agents/MainAgent.ts",
+            taskId: 7,
+            runLog: "",
+            finalAnswerText: "answer",
+            result: {
+                id: "timestamp_case",
+                passed: true,
+                failures: [],
+            },
+        });
+        const caseWrite = state.writes.find((entry) => entry.path.endsWith("/case.json"));
+        expect(caseWrite).toBeDefined();
+        expect(caseWrite?.path).toContain("/repo/artifacts/evals/");
+        expect(caseWrite?.path).not.toContain("Z_timestamp_case");
+        expect(caseWrite?.path).toMatch(/[+-]\d{4}_timestamp_case\/case\.json$/);
+    });
+});
+//# sourceMappingURL=harnessArtifacts.test.js.map

package/dist/__tests__/testing/harnessArtifacts.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harnessArtifacts.test.js","sourceRoot":"","sources":["../../../src/__tests__/testing/harnessArtifacts.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9D,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9B,IAAI,EAAE;QACJ,EAAE,EAAE,CAAC;QACL,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,qCAAqC;QACpD,OAAO,EAAE,QAAQ;QACjB,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC;YACpC,WAAW,EAAE,KAAK;YAClB,aAAa,EAAE,KAAK;YACpB,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,uBAAuB;SACnC,CAAC;KACH;IACD,KAAK,EAAE,EAA+B;IACtC,MAAM,EAAE,EAA8C;CACvD,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACnB,OAAO,EAAE;QACP,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;QAClB,aAAa,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,OAAe,EAAE,EAAE;YACzD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAClE,CAAC,CAAC;KACH;IACD,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,aAAa,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,OAAe,EAAE,EAAE;QACzD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,YAAY,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QACzB,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC;YAC/B,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;SAC3D,CAAC,CAAC;KACJ,CAAC,CAAC;CACJ,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5C,4BAA4B,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QACzC,YAAY,EAAE,OAAO;KACtB,CAAC,CAAC;CACJ,CAAC,CAAC,CAAC;AAEJ,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,UAAU,CAAC,GAAG,EAAE;QACd,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,qBAAqB,CAAC;YACpB,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,gBAAgB;YACxB,KAAK,EAAE,qCAAqC;YAC5C,MAAM,EAAE,CAAC;YACT,MAAM,EAAE;gBACN,EAAE;gBACF,sEAAsE;gBACtE,+BAA+B;gBAC/B,sEAAsE;gBACtE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC3C,sEAAsE;gBACtE,EAAE;aACH,CAAC,IAAI,CAAC,IAAI,CAAC;YACZ,eAAe,EAAE,QAAQ;YACzB,MAAM,EAAE;gBACN,EAAE,EAAE,gBAAgB;gBACpB,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,EAAE;aACb;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAC5F,MAAM,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,sCAAsC,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,qBAAqB,CAAC;YACpB,IAAI,EAAE,aAAa;YACnB,MAAM,EAAE,iBAAiB;YACzB,KAAK,EAAE,6BAA6B;YACpC,MAAM,EAAE,CAAC;YACT,MAAM,EAAE,EAAE;YACV,eAAe,EAAE,QAAQ;YACzB,UAAU,EAAE,2DAA2D;YACvE,MAAM,EAAE;gBACN,EAAE,EAAE,iBAAiB;gBACrB,MAAM,EAAE,IAAI;gBACZ,QAAQ,EAAE,EAAE;aACb;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;QACpF,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,qBAAqB,CAAC;YACpB,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,gBAAgB;YACxB,KAAK,EAAE,qCAAqC;YAC5C,MAAM,EAAE,CAAC;YACT,MAAM,EAAE,EAAE;YACV,eAAe,EAAE,QAAQ;YACzB,MAAM,EAAE;gBACN,EAAE,EAAE,gBAAgB;gBACpB,MAAM,EAAE,IAAI;gBACZ,QAAQ,EAAE,EAAE;aACb;SACF,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;QAClF,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;QAC5D,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QAC1D,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["/**\n * Verifies that saved harness artifacts include diagnosis data and Markdown reports.\n *\n * Why this exists:\n * - artifact folders should be reviewable without rerunning the agent\n * - diagnosis.json is the quick pointer to the likely subsystem\n * - report.md should preserve Mermaid blocks for artifact viewers\n * - artifact folder timestamps should stay in local time with offset, not UTC `Z`\n * - the writer should persist both files for saved runs\n */\nimport { beforeEach, describe, expect, it, vi } from \"vitest\";\n\nconst state = vi.hoisted(() => ({\n  task: {\n    id: 7,\n    status: \"deferred\",\n    initial_query: \"Explain cli/src/agents/MainAgent.ts\",\n    summary: \"answer\",\n    routing_decision_json: JSON.stringify({\n      allowSearch: false,\n      allowResearch: false,\n      allowTransform: false,\n      rationale: \"route=explicit-target\",\n    }),\n  },\n  steps: [] as Record<string, unknown>[],\n  writes: [] as Array<{ path: string; content: string }>,\n}));\n\nvi.mock(\"fs\", () => ({\n  default: {\n    mkdirSync: vi.fn(),\n    writeFileSync: vi.fn((filePath: string, content: string) => {\n      state.writes.push({ path: filePath, content: String(content) });\n    }),\n  },\n  mkdirSync: vi.fn(),\n  writeFileSync: vi.fn((filePath: string, content: string) => {\n    state.writes.push({ path: filePath, content: String(content) });\n  }),\n}));\n\nvi.mock(\"../../db/client.js\", () => ({\n  getDbForRepo: vi.fn(() => ({\n    prepare: vi.fn((sql: string) => ({\n      get: () => (sql.includes(\"FROM tasks\") ? state.task : undefined),\n      all: () => (sql.includes(\"FROM steps\") ? state.steps : []),\n    })),\n  })),\n}));\n\nvi.mock(\"../../utils/repoIdentity.js\", () => ({\n  resolveCanonicalRepoIdentity: vi.fn(() => ({\n    repoRootPath: \"/repo\",\n  })),\n}));\n\nimport { writeHarnessArtifacts } from \"../../testing/harnessArtifacts.js\";\n\ndescribe(\"harnessArtifacts\", () => {\n  beforeEach(() => {\n    state.writes = [];\n  });\n\n  it(\"writes diagnosis.json for saved runs\", () => {\n    writeHarnessArtifacts({\n      kind: \"evals\",\n      caseId: \"diagnosis_case\",\n      query: \"Explain cli/src/agents/MainAgent.ts\",\n      taskId: 7,\n      runLog: [\n        \"\",\n        \"====================================================================\",\n        \"📂 OUTPUT | finalAnswerModule\",\n        \"====================================================================\",\n        JSON.stringify({ data: \"answer\" }, null, 2),\n        \"====================================================================\",\n        \"\",\n      ].join(\"\\n\"),\n      finalAnswerText: \"answer\",\n      result: {\n        id: \"diagnosis_case\",\n        passed: false,\n        failures: [],\n      },\n    });\n\n    const diagnosisWrite = state.writes.find((entry) => entry.path.endsWith(\"/diagnosis.json\"));\n    expect(diagnosisWrite).toBeDefined();\n    expect(diagnosisWrite?.content).toContain(\"\\\"primaryCategory\\\": \\\"persistence\\\"\");\n  });\n\n  it(\"writes report.md when report text is provided\", () => {\n    writeHarnessArtifacts({\n      kind: \"integration\",\n      caseId: \"markdown_report\",\n      query: \"summarize repo architecture\",\n      taskId: 7,\n      runLog: \"\",\n      finalAnswerText: \"answer\",\n      reportText: \"# Integration Report | 7\\n\\n```mermaid\\nflowchart TD\\n```\",\n      result: {\n        id: \"markdown_report\",\n        passed: true,\n        failures: [],\n      },\n    });\n\n    const reportWrite = state.writes.find((entry) => entry.path.endsWith(\"/report.md\"));\n    expect(reportWrite).toBeDefined();\n    expect(reportWrite?.content).toContain(\"```mermaid\");\n  });\n\n  it(\"uses a local-time offset timestamp in generated artifact paths\", () => {\n    writeHarnessArtifacts({\n      kind: \"evals\",\n      caseId: \"timestamp_case\",\n      query: \"Explain cli/src/agents/MainAgent.ts\",\n      taskId: 7,\n      runLog: \"\",\n      finalAnswerText: \"answer\",\n      result: {\n        id: \"timestamp_case\",\n        passed: true,\n        failures: [],\n      },\n    });\n\n    const caseWrite = state.writes.find((entry) => entry.path.endsWith(\"/case.json\"));\n    expect(caseWrite).toBeDefined();\n    expect(caseWrite?.path).toContain(\"/repo/artifacts/evals/\");\n    expect(caseWrite?.path).not.toContain(\"Z_timestamp_case\");\n    expect(caseWrite?.path).toMatch(/[+-]\\d{4}_timestamp_case\\/case\\.json$/);\n  });\n});\n"]}

package/dist/__tests__/testing/llmTraceSession.test.js

@@ -0,0 +1,67 @@
+/**
+ * Verifies that integration trace sessions write numbered LLM input files with stable ids.
+ *
+ * Why this exists:
+ * - each model call should leave one reviewable JSON file on disk
+ * - repeated calls from the same caller must not overwrite each other
+ * - suspicious full-context inputs should be marked for later review
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+const state = vi.hoisted(() => ({
+    writes: [],
+}));
+vi.mock("fs", () => ({
+    default: {
+        mkdirSync: vi.fn(),
+        writeFileSync: vi.fn((filePath, content) => {
+            state.writes.push({ path: filePath, content: String(content) });
+        }),
+    },
+    mkdirSync: vi.fn(),
+    writeFileSync: vi.fn((filePath, content) => {
+        state.writes.push({ path: filePath, content: String(content) });
+    }),
+}));
+import { startLlmTraceSession, stopLlmTraceSession, writeLlmTraceRecord } from "../../testing/llmTraceSession.js";
+describe("llmTraceSession", () => {
+    beforeEach(() => {
+        state.writes = [];
+        stopLlmTraceSession();
+    });
+    it("writes numbered trace files with stable call ids", () => {
+        startLlmTraceSession("/repo/artifacts/integration/case1");
+        const first = writeLlmTraceRecord({
+            caller: "finalAnswerModule",
+            query: "q1",
+            model: "test-model",
+            requestBody: { prompt: "one" },
+            promptText: "one",
+            inputContext: { query: "q1" },
+            startedAt: "2026-04-01T02:00:00.000+02:00",
+            finishedAt: "2026-04-01T02:00:01.500+02:00",
+            durationMs: 1500,
+            attemptCount: 1,
+        });
+        const second = writeLlmTraceRecord({
+            caller: "finalAnswerModule",
+            query: "q2",
+            model: "test-model",
+            requestBody: { prompt: "two" },
+            promptText: "two",
+            inputContext: { initContext: {}, task: {} },
+            startedAt: "2026-04-01T02:00:02.000+02:00",
+            finishedAt: "2026-04-01T02:00:05.000+02:00",
+            durationMs: 3000,
+            attemptCount: 2,
+        });
+        expect(first?.callId).toBe("llm-0001-finalAnswerModule");
+        expect(second?.callId).toBe("llm-0002-finalAnswerModule");
+        expect(state.writes[0]?.path).toBe("/repo/artifacts/integration/case1/llm_calls/0001_finalAnswerModule.json");
+        expect(state.writes[1]?.path).toBe("/repo/artifacts/integration/case1/llm_calls/0002_finalAnswerModule.json");
+        expect(state.writes[1]?.content).toContain('"suspiciousFullContext": true');
+        expect(state.writes[0]?.content).toContain('"startedAt": "2026-04-01T02:00:00.000+02:00"');
+        expect(state.writes[1]?.content).toContain('"durationMs": 3000');
+        expect(state.writes[1]?.content).toContain('"attemptCount": 2');
+    });
+});
+//# sourceMappingURL=llmTraceSession.test.js.map

package/dist/__tests__/testing/llmTraceSession.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llmTraceSession.test.js","sourceRoot":"","sources":["../../../src/__tests__/testing/llmTraceSession.test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9D,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9B,MAAM,EAAE,EAA8C;CACvD,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACnB,OAAO,EAAE;QACP,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;QAClB,aAAa,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,OAAe,EAAE,EAAE;YACzD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAClE,CAAC,CAAC;KACH;IACD,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,aAAa,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,OAAe,EAAE,EAAE;QACzD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAElH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,UAAU,CAAC,GAAG,EAAE;QACd,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC;QAClB,mBAAmB,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,oBAAoB,CAAC,mCAAmC,CAAC,CAAC;QAE1D,MAAM,KAAK,GAAG,mBAAmB,CAAC;YAChC,MAAM,EAAE,mBAAmB;YAC3B,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,YAAY;YACnB,WAAW,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;YAC9B,UAAU,EAAE,KAAK;YACjB,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YAC7B,SAAS,EAAE,+BAA+B;YAC1C,UAAU,EAAE,+BAA+B;YAC3C,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE,CAAC;SAChB,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,mBAAmB,CAAC;YACjC,MAAM,EAAE,mBAAmB;YAC3B,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,YAAY;YACnB,WAAW,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;YAC9B,UAAU,EAAE,KAAK;YACjB,YAAY,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;YAC3C,SAAS,EAAE,+BAA+B;YAC1C,UAAU,EAAE,+BAA+B;YAC3C,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE,CAAC;SAChB,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC1D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,yEAAyE,CAAC,CAAC;QAC9G,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,yEAAyE,CAAC,CAAC;QAC9G,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,8CAA8C,CAAC,CAAC;QAC3F,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["/**\n * Verifies that integration trace sessions write numbered LLM input files with stable ids.\n *\n * Why this exists:\n * - each model call should leave one reviewable JSON file on disk\n * - repeated calls from the same caller must not overwrite each other\n * - suspicious full-context inputs should be marked for later review\n */\nimport { beforeEach, describe, expect, it, vi } from \"vitest\";\n\nconst state = vi.hoisted(() => ({\n  writes: [] as Array<{ path: string; content: string }>,\n}));\n\nvi.mock(\"fs\", () => ({\n  default: {\n    mkdirSync: vi.fn(),\n    writeFileSync: vi.fn((filePath: string, content: string) => {\n      state.writes.push({ path: filePath, content: String(content) });\n    }),\n  },\n  mkdirSync: vi.fn(),\n  writeFileSync: vi.fn((filePath: string, content: string) => {\n    state.writes.push({ path: filePath, content: String(content) });\n  }),\n}));\n\nimport { startLlmTraceSession, stopLlmTraceSession, writeLlmTraceRecord } from \"../../testing/llmTraceSession.js\";\n\ndescribe(\"llmTraceSession\", () => {\n  beforeEach(() => {\n    state.writes = [];\n    stopLlmTraceSession();\n  });\n\n  it(\"writes numbered trace files with stable call ids\", () => {\n    startLlmTraceSession(\"/repo/artifacts/integration/case1\");\n\n    const first = writeLlmTraceRecord({\n      caller: \"finalAnswerModule\",\n      query: \"q1\",\n      model: \"test-model\",\n      requestBody: { prompt: \"one\" },\n      promptText: \"one\",\n      inputContext: { query: \"q1\" },\n      startedAt: \"2026-04-01T02:00:00.000+02:00\",\n      finishedAt: \"2026-04-01T02:00:01.500+02:00\",\n      durationMs: 1500,\n      attemptCount: 1,\n    });\n    const second = writeLlmTraceRecord({\n      caller: \"finalAnswerModule\",\n      query: \"q2\",\n      model: \"test-model\",\n      requestBody: { prompt: \"two\" },\n      promptText: \"two\",\n      inputContext: { initContext: {}, task: {} },\n      startedAt: \"2026-04-01T02:00:02.000+02:00\",\n      finishedAt: \"2026-04-01T02:00:05.000+02:00\",\n      durationMs: 3000,\n      attemptCount: 2,\n    });\n\n    expect(first?.callId).toBe(\"llm-0001-finalAnswerModule\");\n    expect(second?.callId).toBe(\"llm-0002-finalAnswerModule\");\n    expect(state.writes[0]?.path).toBe(\"/repo/artifacts/integration/case1/llm_calls/0001_finalAnswerModule.json\");\n    expect(state.writes[1]?.path).toBe(\"/repo/artifacts/integration/case1/llm_calls/0002_finalAnswerModule.json\");\n    expect(state.writes[1]?.content).toContain('\"suspiciousFullContext\": true');\n    expect(state.writes[0]?.content).toContain('\"startedAt\": \"2026-04-01T02:00:00.000+02:00\"');\n    expect(state.writes[1]?.content).toContain('\"durationMs\": 3000');\n    expect(state.writes[1]?.content).toContain('\"attemptCount\": 2');\n  });\n});\n"]}

package/dist/__tests__/testing/registerDevCliCommands.test.js

@@ -0,0 +1,35 @@
+/**
+ * Verifies the dev CLI exposes only the supported test helpers.
+ *
+ * Why this file exists:
+ * - shell wiring and direct CLI wiring can drift apart
+ * - command cleanup should not accidentally remove the remaining test helpers
+ */
+import { Command } from "commander";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+const mockedTestCmd = vi.hoisted(() => ({
+    runAllTestsCommand: vi.fn(),
+    listRegisteredTests: vi.fn(),
+    printTestCommand: vi.fn(),
+    runRegisteredTestCommand: vi.fn(),
+}));
+vi.mock("../../commands/TestCmd.js", () => ({
+    runAllTestsCommand: mockedTestCmd.runAllTestsCommand,
+    listRegisteredTests: mockedTestCmd.listRegisteredTests,
+    printTestCommand: mockedTestCmd.printTestCommand,
+    runRegisteredTestCommand: mockedTestCmd.runRegisteredTestCommand,
+}));
+import { registerDevCliCommands } from "../../testing/registerDevCliCommands.js";
+describe("registerDevCliCommands", () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+    it("registers the batch tests command", async () => {
+        const cmd = new Command();
+        const runQuery = vi.fn().mockResolvedValue(undefined);
+        registerDevCliCommands(cmd, runQuery);
+        await cmd.parseAsync(["node", "scai", "tests"], { from: "node" });
+        expect(mockedTestCmd.runAllTestsCommand).toHaveBeenCalledTimes(1);
+    });
+});
+//# sourceMappingURL=registerDevCliCommands.test.js.map

package/dist/__tests__/testing/registerDevCliCommands.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"registerDevCliCommands.test.js","sourceRoot":"","sources":["../../../src/__tests__/testing/registerDevCliCommands.test.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9D,MAAM,aAAa,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACtC,kBAAkB,EAAE,EAAE,CAAC,EAAE,EAAE;IAC3B,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE;IAC5B,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE;IACzB,wBAAwB,EAAE,EAAE,CAAC,EAAE,EAAE;CAClC,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,2BAA2B,EAAE,GAAG,EAAE,CAAC,CAAC;IAC1C,kBAAkB,EAAE,aAAa,CAAC,kBAAkB;IACpD,mBAAmB,EAAE,aAAa,CAAC,mBAAmB;IACtD,gBAAgB,EAAE,aAAa,CAAC,gBAAgB;IAChD,wBAAwB,EAAE,aAAa,CAAC,wBAAwB;CACjE,CAAC,CAAC,CAAC;AAEJ,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AAEjF,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACtD,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAEtC,MAAM,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAElE,MAAM,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["/**\n * Verifies the dev CLI exposes only the supported test helpers.\n *\n * Why this file exists:\n * - shell wiring and direct CLI wiring can drift apart\n * - command cleanup should not accidentally remove the remaining test helpers\n */\nimport { Command } from \"commander\";\nimport { beforeEach, describe, expect, it, vi } from \"vitest\";\n\nconst mockedTestCmd = vi.hoisted(() => ({\n  runAllTestsCommand: vi.fn(),\n  listRegisteredTests: vi.fn(),\n  printTestCommand: vi.fn(),\n  runRegisteredTestCommand: vi.fn(),\n}));\n\nvi.mock(\"../../commands/TestCmd.js\", () => ({\n  runAllTestsCommand: mockedTestCmd.runAllTestsCommand,\n  listRegisteredTests: mockedTestCmd.listRegisteredTests,\n  printTestCommand: mockedTestCmd.printTestCommand,\n  runRegisteredTestCommand: mockedTestCmd.runRegisteredTestCommand,\n}));\n\nimport { registerDevCliCommands } from \"../../testing/registerDevCliCommands.js\";\n\ndescribe(\"registerDevCliCommands\", () => {\n  beforeEach(() => {\n    vi.clearAllMocks();\n  });\n\n  it(\"registers the batch tests command\", async () => {\n    const cmd = new Command();\n    const runQuery = vi.fn().mockResolvedValue(undefined);\n    registerDevCliCommands(cmd, runQuery);\n\n    await cmd.parseAsync([\"node\", \"scai\", \"tests\"], { from: \"node\" });\n\n    expect(mockedTestCmd.runAllTestsCommand).toHaveBeenCalledTimes(1);\n  });\n});\n"]}