scai 0.1.178 → 1.0.0

package/dist/__tests__/evalCommands.test.js

@@ -0,0 +1,506 @@
+/**
+ * Covers shared eval command actions and shell adapters.
+ *
+ * Why this file exists:
+ * - `/evals` should default to the fast converted new-agent suite
+ * - `/evals-transform` should isolate the slower transform suite
+ * - `/eval-run` and `/eval-list` should stay aligned with the trimmed eval registry
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+const mockState = vi.hoisted(() => ({
+    currentTaskId: undefined,
+    runLog: "",
+    files: {},
+    artifactCalls: [],
+    createHarnessArtifactDir: vi.fn((kind, caseId) => `/tmp/${kind}/${caseId}`),
+    startLlmTraceSession: vi.fn(),
+    stopLlmTraceSession: vi.fn(),
+}));
+vi.mock("fs", () => {
+    const normalize = (value) => String(value);
+    const writeFileSync = vi.fn((_path, data, options) => {
+        const filePath = normalize(_path);
+        mockState.files[filePath] = String(data);
+        if (options?.flag === "w" && filePath === "/tmp/scai-eval-run.log") {
+            mockState.runLog = String(data);
+        }
+    });
+    const appendFileSync = vi.fn((_path, data) => {
+        const filePath = normalize(_path);
+        const next = (mockState.files[filePath] ?? "") + String(data);
+        mockState.files[filePath] = next;
+        if (filePath === "/tmp/scai-eval-run.log") {
+            mockState.runLog = next;
+        }
+    });
+    const readFileSync = vi.fn((filePath) => mockState.files[normalize(filePath)] ?? "");
+    const existsSync = vi.fn((filePath) => {
+        const normalized = normalize(filePath);
+        return normalized in mockState.files
+            || Object.keys(mockState.files).some((candidate) => candidate.startsWith(`${normalized}/`));
+    });
+    const mkdirSync = vi.fn();
+    return {
+        default: { writeFileSync, appendFileSync, readFileSync, existsSync, mkdirSync },
+        writeFileSync,
+        appendFileSync,
+        readFileSync,
+        existsSync,
+        mkdirSync,
+    };
+});
+vi.mock("../constants.js", async (importOriginal) => {
+    const actual = await importOriginal();
+    return {
+        ...actual,
+        RUN_LOG_PATH: "/tmp/scai-eval-run.log",
+    };
+});
+vi.mock("../commands/EvalReportCmd.js", () => ({
+    buildEvalReportMarkdown: vi.fn(() => "report"),
+    getLatestTaskId: vi.fn(() => mockState.currentTaskId),
+    printEvalReport: vi.fn(),
+    runEvalReportCommand: vi.fn(),
+}));
+vi.mock("../testing/harnessArtifacts.js", () => ({
+    createHarnessArtifactDir: mockState.createHarnessArtifactDir,
+    writeHarnessArtifacts: vi.fn((input) => {
+        mockState.artifactCalls.push(input);
+        return String(input.artifactDir ?? `/tmp/artifacts/${String(input.caseId)}`);
+    }),
+}));
+vi.mock("../testing/llmTraceSession.js", () => ({
+    startLlmTraceSession: mockState.startLlmTraceSession,
+    stopLlmTraceSession: mockState.stopLlmTraceSession,
+}));
+vi.mock("../utils/repoIdentity.js", () => ({
+    resolveCanonicalRepoIdentity: vi.fn(() => ({
+        repoRootPath: "/repo",
+        executionPath: "/repo",
+        currentBranch: "main",
+    })),
+}));
+import { buildEvalReportMarkdown } from "../commands/EvalReportCmd.js";
+import { evalQueries, listEvalsCommand, localEvalCases, registerEvalCommands, registerEvalShellCommands, runEvalBatchCommand, runEvalByIdCommand, runEvalSmokeCommand, runEvalTransformBatchCommand, runLocalEvals, } from "../commands/evalCommands.js";
+import { Command } from "commander";
+function getAgentEvalCountForSuite(suite) {
+    return localEvalCases.filter((test) => (test.group ?? "main") === "agent" && (test.suite ?? "default") === suite).length;
+}
+describe("eval shell commands", () => {
+    beforeEach(() => {
+        mockState.currentTaskId = undefined;
+        mockState.runLog = "";
+        mockState.files = {
+            "/repo/playground/test/spatialmap.small.js": "function runBootstrapFlow(context) {}\nrunBootstrapFlow(context)\n",
+            "/repo/playground/test/spatialmap.small.extract.js": "function buildProfileToolUrl(profileName, toolPath) { return \"/rest/profile/\" + profileName + \"/tools/\" + toolPath; }\n" +
+                "function buildPortalHtmlUrl(profileName) { return buildProfileToolUrl(profileName, \"html/client/portal\"); }\n" +
+                "function buildLayerPanelUrl(profileName) { return buildProfileToolUrl(profileName, \"html/client/layer-panel\"); }\n" +
+                "function buildSearchPanelUrl(profileName) { return buildProfileToolUrl(profileName, \"html/client/search-panel\"); }\n",
+            "/repo/playground/test/spatialmap.small.multi.js": "function beginPortalFlow(context) {}\nfunction beginWidgetFlow(context) {}\nfunction beginReadyFlow(context) {}\n",
+            "/repo/playground/test/spatialmap.small.multi.extract.js": "function buildProfileToolRequestUrl(profileName, toolPath) { return \"/rest/profile/\" + profileName + \"/tools/\" + toolPath; }\n" +
+                "function buildPortalHtmlRequestUrl(profileName) { return buildProfileToolRequestUrl(profileName, \"html/client/portal\"); }\n" +
+                "function buildLayerPanelRequestUrl(profileName) { return buildProfileToolRequestUrl(profileName, \"html/client/layer-panel\"); }\n" +
+                "function buildSearchPanelRequestUrl(profileName) { return buildProfileToolRequestUrl(profileName, \"html/client/search-panel\"); }\n" +
+                "function buildPortalSourceRequestUrl(profileName, sessionId) { return \"/rest/profile/\" + profileName + \"/tools/src/client/portal/\" + sessionId; }\n",
+            "/repo/playground/test/spatialmap.medium.js": "document.querySelector(\"body\").insertAdjacentHTML(\"beforeend\", html);\nconst pageBody = document.querySelector(\"body\");\n",
+            "/repo/playground/test/spatialmap.medium.extract.js": "function buildProfileToolUrl(profileName, toolPath) { return \"/rest/profile/\" + profileName + \"/tools/\" + toolPath; }\n" +
+                "function buildPortalHtmlUrl(profileName) { return buildProfileToolUrl(profileName, \"html/client/portal\"); }\n" +
+                "function buildSidebarHtmlUrl(profileName) { return buildProfileToolUrl(profileName, \"html/client/sidebar\"); }\n" +
+                "function buildSearchHtmlUrl(profileName) { return buildProfileToolUrl(profileName, \"html/client/search\"); }\n" +
+                "function buildPortalSourceUrl(profileName, sessionId) { return buildProfileToolUrl(profileName, \"src/client/portal/\" + sessionId); }\n",
+            "/repo/playground/test/spatialmap.medium.multi.js": "document.querySelector(\"body\").insertAdjacentHTML(\"beforeend\", html);\nconst pageBody = document.querySelector(\"body\");\n",
+            "/repo/playground/test/spatialmap.js": "const sessionId = sessionStorage.getItem(\"spatialmapSessionId\");\n" +
+                "sessionId: sessionStorage.getItem(\"spatialmapSessionId\"),\n",
+        };
+        mockState.artifactCalls = [];
+        mockState.createHarnessArtifactDir.mockClear();
+        mockState.startLlmTraceSession.mockClear();
+        mockState.stopLlmTraceSession.mockClear();
+        vi.clearAllMocks();
+        vi.spyOn(console, "log").mockImplementation(() => { });
+    });
+    it("starts eval queries without continuing the current task", async () => {
+        const runQuery = vi.fn().mockResolvedValue(undefined);
+        await runEvalSmokeCommand(runQuery);
+        expect(runQuery).toHaveBeenCalledWith(evalQueries[0], {
+            continueCurrentTask: false,
+            runtime: "agent",
+        });
+    });
+    it("defaults /evals to the fast new-agent suite", async () => {
+        const runQuery = vi.fn().mockImplementation(async () => {
+            mockState.currentTaskId = (mockState.currentTaskId ?? 500) + 1;
+            mockState.runLog += [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "agentTypes.ts",
+            ].join("\n");
+        });
+        await runEvalBatchCommand(runQuery);
+        expect(runQuery).toHaveBeenCalledTimes(getAgentEvalCountForSuite("default"));
+        expect(runQuery.mock.calls.every(([, options]) => options?.runtime === "agent")).toBe(true);
+    });
+    it("runs the slow transform suite only through /evals-transform", async () => {
+        const runQuery = vi.fn().mockImplementation(async () => {
+            mockState.currentTaskId = (mockState.currentTaskId ?? 600) + 1;
+            mockState.files["/repo/playground/test/spatialmap.small.js"] =
+                "function bootstrapPortalFlow(context) {}\nbootstrapPortalFlow(context);\n";
+            mockState.runLog += [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "playground/test/spatialmap.small.js",
+                "action=codeTransform",
+            ].join("\n");
+            mockState.files["/tmp/scai-eval-run.log"] = mockState.runLog;
+        });
+        await runEvalTransformBatchCommand(runQuery);
+        expect(runQuery).toHaveBeenCalledTimes(getAgentEvalCountForSuite("transform"));
+        expect(runQuery.mock.calls.every(([, options]) => options?.runtime === "agent")).toBe(true);
+    });
+    it("keeps the transform suite populated with the expected spatialmap eval ids", () => {
+        const transformIds = localEvalCases
+            .filter((test) => (test.group ?? "main") === "agent" && (test.suite ?? "default") === "transform")
+            .map((test) => test.id);
+        expect(transformIds).toEqual(expect.arrayContaining([
+            "transform_spatialmap_small_rename_run_bootstrap_flow",
+            "transform_spatialmap_small_extract_profile_url_helper",
+            "transform_spatialmap_small_multi_rename_flow_helpers",
+            "transform_spatialmap_small_multi_extract_portal_request_helpers",
+            "transform_spatialmap_medium_document_body_cleanup",
+            "transform_spatialmap_medium_multi_dom_surface_cleanup",
+            "transform_spatialmap_medium_extract_profile_tool_url",
+            "transform_spatialmap_large_cache_session_id",
+        ]));
+        expect(transformIds).toHaveLength(8);
+    });
+    it("eval-run resolves one converted agent eval directly", async () => {
+        const runQuery = vi.fn().mockImplementation(async (query) => {
+            mockState.currentTaskId = (mockState.currentTaskId ?? 200) + 1;
+            mockState.runLog += [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "EvalReportCmd",
+            ].join("\n");
+        });
+        await runEvalByIdCommand(runQuery, ["search_file_without_extension_exact_target"]);
+        expect(runQuery).toHaveBeenCalledTimes(1);
+        expect(runQuery).toHaveBeenCalledWith("Explain EvalReportCmd", { continueCurrentTask: false, resumeTaskId: undefined, bindAsCurrentTask: undefined, runtime: "agent" });
+        expect(mockState.artifactCalls).toHaveLength(1);
+        expect(mockState.artifactCalls[0]?.caseId).toBe("search_file_without_extension_exact_target");
+        expect(mockState.artifactCalls[0]?.artifactDir).toBe("/tmp/evals/search_file_without_extension_exact_target");
+    });
+    it("writes report text for converted agent eval artifacts even without per-eval console reporting", async () => {
+        const runQuery = vi.fn().mockImplementation(async (query) => {
+            mockState.currentTaskId = (mockState.currentTaskId ?? 300) + 1;
+            mockState.runLog += [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "EvalReportCmd",
+            ].join("\n");
+        });
+        await runLocalEvals(runQuery, {
+            includeIds: ["search_file_without_extension_exact_target"],
+        });
+        expect(buildEvalReportMarkdown).toHaveBeenCalled();
+        expect(buildEvalReportMarkdown).toHaveBeenCalledWith(301, expect.any(Object));
+        expect(mockState.artifactCalls).toHaveLength(1);
+        expect(mockState.artifactCalls[0]?.reportText).toBe("report");
+    });
+    it("eval-list omits deleted legacy-only eval ids", async () => {
+        const logSpy = vi.spyOn(console, "log").mockImplementation(() => { });
+        await listEvalsCommand();
+        const output = logSpy.mock.calls.map((args) => args.join(" ")).join("\n");
+        expect(output).toContain("search_file_without_extension_exact_target");
+        expect(output).toContain("suite: default");
+        expect(output).toContain("suite: transform");
+        expect(output).not.toContain("multifile_no_early_complete_regression");
+        expect(output).not.toContain("resume_continuity_answer_now");
+        logSpy.mockRestore();
+    });
+    it("keeps the refreshed stale-default eval checks aligned with the new agent", () => {
+        const exactTargetEval = localEvalCases.find((test) => test.id === "search_file_with_extension_exact_target");
+        const sqliteEval = localEvalCases.find((test) => test.id === "verify_pipeline_sqlite_query");
+        const testEntrypointsEval = localEvalCases.find((test) => test.id === "final_answer_for_test_entrypoints");
+        const naturalLanguageFolderEval = localEvalCases.find((test) => test.id === "search_folder_natural_language_exact_target");
+        expect(exactTargetEval?.query).toBe("Explain loopReview.ts");
+        expect(exactTargetEval?.checks.mustContainAnswer).toEqual(["loopReview.ts"]);
+        expect(sqliteEval?.checks.mustContainAnswer).toEqual([
+            "db/fileIndex.ts",
+            "db/client.ts",
+            "SQLite",
+        ]);
+        expect(testEntrypointsEval?.checks.mustContainAnswer).toEqual([
+            "vitest",
+            "package.json",
+            "vitest.config.ts",
+        ]);
+        expect(naturalLanguageFolderEval?.checks.mustContain).toEqual([
+            "artifact final-answer",
+        ]);
+    });
+    it("eval-run still resolves one transform eval directly", async () => {
+        const runQuery = vi.fn().mockImplementation(async () => {
+            mockState.currentTaskId = (mockState.currentTaskId ?? 700) + 1;
+            mockState.files["/repo/playground/test/spatialmap.small.js"] =
+                "function bootstrapPortalFlow(context) {}\nbootstrapPortalFlow(context);\n";
+            mockState.runLog += [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "playground/test/spatialmap.small.js",
+                "action=codeTransform",
+            ].join("\n");
+            mockState.files["/tmp/scai-eval-run.log"] = mockState.runLog;
+        });
+        await runEvalByIdCommand(runQuery, ["transform_spatialmap_small_rename_run_bootstrap_flow"]);
+        expect(runQuery).toHaveBeenCalledTimes(1);
+        expect(runQuery).toHaveBeenCalledWith("In playground/test/spatialmap.small.js, rename runBootstrapFlow to bootstrapPortalFlow and update all call sites. Keep behavior the same.", { continueCurrentTask: false, resumeTaskId: undefined, bindAsCurrentTask: undefined, runtime: "agent" });
+        expect(mockState.artifactCalls[0]?.caseId).toBe("transform_spatialmap_small_rename_run_bootstrap_flow");
+    });
+    it("captures the direct committed target file in transform eval artifacts", async () => {
+        const runQuery = vi.fn().mockImplementation(async () => {
+            mockState.currentTaskId = 801;
+            mockState.files["/repo/playground/test/spatialmap.small.js"] =
+                "function bootstrapPortalFlow(context) {}\nbootstrapPortalFlow(context);\n";
+            mockState.runLog = [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "playground/test/spatialmap.small.js",
+                "action=codeTransform",
+            ].join("\n");
+            mockState.files["/tmp/scai-eval-run.log"] = mockState.runLog;
+        });
+        await runLocalEvals(runQuery, {
+            includeIds: ["transform_spatialmap_small_rename_run_bootstrap_flow"],
+        });
+        expect(mockState.artifactCalls).toHaveLength(1);
+        expect((mockState.artifactCalls[0]?.outputFiles)[0]?.path)
+            .toBe("playground/test/spatialmap.small.js");
+    });
+    it("fails a transform eval when the output file keeps the old symbol", async () => {
+        const runQuery = vi.fn().mockImplementation(async () => {
+            mockState.currentTaskId = 950;
+            mockState.runLog = [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "playground/test/spatialmap.small.js",
+                "action=codeTransform",
+            ].join("\n");
+            mockState.files["/tmp/scai-eval-run.log"] = mockState.runLog;
+        });
+        await runLocalEvals(runQuery, {
+            includeIds: ["transform_spatialmap_small_rename_run_bootstrap_flow"],
+        });
+        const result = mockState.artifactCalls[0]?.result;
+        expect(result.failures?.some((failure) => failure.includes("forbidden token"))).toBe(true);
+    });
+    it("fails a transform eval when the direct target file is not valid JavaScript", async () => {
+        const runQuery = vi.fn().mockImplementation(async () => {
+            mockState.currentTaskId = 975;
+            mockState.files["/repo/playground/test/spatialmap.small.js"] =
+                "function bootstrapPortalFlow(context) {\n";
+            mockState.runLog = [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "playground/test/spatialmap.small.js",
+                "action=codeTransform",
+            ].join("\n");
+            mockState.files["/tmp/scai-eval-run.log"] = mockState.runLog;
+        });
+        await runLocalEvals(runQuery, {
+            includeIds: ["transform_spatialmap_small_rename_run_bootstrap_flow"],
+        });
+        const result = mockState.artifactCalls[0]?.result;
+        expect(result.failures?.some((failure) => failure.includes("syntax check failed"))).toBe(true);
+    });
+    it("points transform eval queries at direct committed fixtures with one unique target file per eval", () => {
+        const transformCases = localEvalCases
+            .filter((test) => (test.group ?? "main") === "agent" && (test.suite ?? "default") === "transform")
+            .map((test) => ({
+            query: test.query ?? "",
+            path: test.checks.fileMustContain?.[0]?.path ?? test.syntaxCheckPaths?.[0] ?? "",
+        }));
+        expect(transformCases.every((test) => !test.query.includes("playground/test/eval-work/"))).toBe(true);
+        expect(new Set(transformCases.map((test) => test.path)).size).toBe(transformCases.length);
+    });
+    it("accepts helper-call forms that omit the literal /tools prefix in extraction eval definitions", () => {
+        const smallExtractEval = localEvalCases.find((test) => test.id === "transform_spatialmap_small_extract_profile_url_helper");
+        const mediumExtractEval = localEvalCases.find((test) => test.id === "transform_spatialmap_medium_extract_profile_tool_url");
+        expect(smallExtractEval?.checks.fileMustContainAnyOf).toEqual(expect.arrayContaining([
+            expect.objectContaining({
+                anyOf: expect.arrayContaining([
+                    "return buildProfileToolUrl(profileName, \"html/client/portal\")",
+                    "return buildProfileToolUrl(profileName) + \"/html/client/portal\"",
+                ]),
+            }),
+            expect.objectContaining({
+                anyOf: expect.arrayContaining([
+                    "return buildProfileToolUrl(profileName, \"html/client/layer-panel\")",
+                    "return buildProfileToolUrl(profileName) + \"/html/client/layer-panel\"",
+                ]),
+            }),
+            expect.objectContaining({
+                anyOf: expect.arrayContaining([
+                    "return buildProfileToolUrl(profileName, \"html/client/search-panel\")",
+                    "return buildProfileToolUrl(profileName) + \"/html/client/search-panel\"",
+                ]),
+            }),
+        ]));
+        expect(mediumExtractEval?.checks.fileMustContain?.[0]?.tokens).toEqual(expect.arrayContaining([
+            "return buildProfileToolUrl(profileName, \"html/client/portal\")",
+            "return buildProfileToolUrl(profileName, \"html/client/sidebar\")",
+            "return buildProfileToolUrl(profileName, \"html/client/search\")",
+            "return buildProfileToolUrl(profileName, \"src/client/portal/\" + sessionId)",
+        ]));
+    });
+    it("does not require the small multi extraction eval to rewrite the portal source request helper", () => {
+        const smallMultiExtractEval = localEvalCases.find((test) => test.id === "transform_spatialmap_small_multi_extract_portal_request_helpers");
+        const fileMustContain = smallMultiExtractEval?.checks.fileMustContain?.[0]?.tokens ?? [];
+        const fileMustNotContain = smallMultiExtractEval?.checks.fileMustNotContain?.[0]?.tokens ?? [];
+        const query = smallMultiExtractEval?.query ?? "";
+        expect(query).not.toContain("buildPortalSourceRequestUrl");
+        expect(fileMustContain.some((token) => token.includes("portal/\" + sessionId"))).toBe(false);
+        expect(fileMustNotContain.some((token) => token.includes("buildPortalSourceRequestUrl"))).toBe(false);
+    });
+    it("keeps the active transform queries copyable with direct committed target paths", () => {
+        const transformQueries = localEvalCases
+            .filter((test) => (test.group ?? "main") === "agent" && (test.suite ?? "default") === "transform")
+            .map((test) => test.query ?? "");
+        expect(transformQueries).toEqual(expect.arrayContaining([
+            "In playground/test/spatialmap.small.js, rename runBootstrapFlow to bootstrapPortalFlow and update all call sites. Keep behavior the same.",
+            "In playground/test/spatialmap.small.extract.js, extract the repeated /rest/profile/... URL building into a helper named buildProfileToolUrl and use it in buildPortalHtmlUrl, buildLayerPanelUrl, and buildSearchPanelUrl. Keep behavior the same.",
+            "In playground/test/spatialmap.medium.extract.js, extract the repeated /rest/profile/... tool URL building into a helper named buildProfileToolUrl and use it in buildPortalHtmlUrl, buildSidebarHtmlUrl, buildSearchHtmlUrl, and buildPortalSourceUrl. Keep behavior the same.",
+        ]));
+    });
+    it("labels extraction assertion misses with a likely failure kind in saved eval results", async () => {
+        const runQuery = vi.fn().mockImplementation(async () => {
+            mockState.currentTaskId = 990;
+            mockState.runLog = [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "playground/test/spatialmap.medium.extract.js",
+                "action=codeTransform",
+            ].join("\n");
+            mockState.files["/tmp/scai-eval-run.log"] = mockState.runLog;
+            mockState.files["/repo/playground/test/spatialmap.medium.extract.js"] = [
+                "function buildProfileToolUrl(profileName) {",
+                "  return \"/rest/profile/\" + profileName + \"/tools\";",
+                "}",
+                "function buildPortalHtmlUrl(profileName) {",
+                "  return buildProfileToolUrl(profileName) + \"/html/client/portal\";",
+                "}",
+            ].join("\n");
+        });
+        await runLocalEvals(runQuery, {
+            includeIds: ["transform_spatialmap_medium_extract_profile_tool_url"],
+        });
+        const result = mockState.artifactCalls[0]?.result;
+        expect(result.failures?.length).toBeGreaterThan(0);
+        expect(result.likelyFailureKind).toBe("harness-strictness");
+    });
+    it("accepts the small extract eval when the helper is used through string concatenation", async () => {
+        const runQuery = vi.fn().mockImplementation(async () => {
+            mockState.currentTaskId = 991;
+            mockState.runLog = [
+                `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "playground/test/spatialmap.small.extract.js",
+                "action=codeTransform",
+            ].join("\n");
+            mockState.files["/tmp/scai-eval-run.log"] = mockState.runLog;
+            mockState.files["/repo/playground/test/spatialmap.small.extract.js"] = [
+                "function buildProfileToolUrl(profileName) {",
+                "  return \"/rest/profile/\" + profileName + \"/tools\";",
+                "}",
+                "function buildPortalHtmlUrl(profileName) {",
+                "  return buildProfileToolUrl(profileName) + \"/html/client/portal\";",
+                "}",
+                "function buildLayerPanelUrl(profileName) {",
+                "  return buildProfileToolUrl(profileName) + \"/html/client/layer-panel\";",
+                "}",
+                "function buildSearchPanelUrl(profileName) {",
+                "  return buildProfileToolUrl(profileName) + \"/html/client/search-panel\";",
+                "}",
+            ].join("\n");
+        });
+        await runLocalEvals(runQuery, {
+            includeIds: ["transform_spatialmap_small_extract_profile_url_helper"],
+        });
+        const result = mockState.artifactCalls[0]?.result;
+        expect(result.passed).toBe(true);
+        expect(result.failures).toEqual([]);
+    });
+    it("matches eval log tokens after stripping ANSI color codes", async () => {
+        const runQuery = vi.fn().mockImplementation(async (query) => {
+            mockState.currentTaskId = (mockState.currentTaskId ?? 400) + 1;
+            mockState.runLog += [
+                `[2026-05-04 07:36:54 +02:00] [agent] [\x1b[33mtask:created\x1b[39m] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "agentTypes.ts",
+            ].join("\n");
+        });
+        await runLocalEvals(runQuery, {
+            includeIds: ["agent_read_and_answer_simple_target"],
+        });
+        expect(mockState.artifactCalls).toHaveLength(1);
+        expect(mockState.artifactCalls[0]?.caseId).toBe("agent_read_and_answer_simple_target");
+        expect(mockState.artifactCalls[0]?.reportText).toBe("report");
+    });
+    it("does not attach a stale report task when the scenario did not create a new task id", async () => {
+        mockState.currentTaskId = 1547;
+        const runQuery = vi.fn().mockImplementation(async (_query) => {
+            mockState.currentTaskId = 1547;
+            mockState.runLog += [
+                `[2026-05-04 07:36:54 +02:00] [agent] [\x1b[33mtask:created\x1b[39m] status=active run started`,
+                `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,
+                "agentTypes.ts",
+            ].join("\n");
+        });
+        await runLocalEvals(runQuery, {
+            includeIds: ["agent_read_and_answer_simple_target"],
+        });
+        expect(buildEvalReportMarkdown).not.toHaveBeenCalled();
+        expect(mockState.artifactCalls).toHaveLength(1);
+        expect(mockState.artifactCalls[0]?.reportText).toBeUndefined();
+    });
+    it("marks an eval incomplete when the run did not create its own task or owned run log", async () => {
+        mockState.currentTaskId = 2200;
+        mockState.runLog = [
+            `[2026-05-04 07:36:54 +02:00] [shared] [legacy:output] SearchOrchestrator output`,
+            `{"stepName":"SearchOrchestrator","type":"output","content":{"route":"db"}}`,
+        ].join("\n");
+        mockState.files["/tmp/scai-eval-run.log"] = mockState.runLog;
+        const runQuery = vi.fn().mockImplementation(async () => {
+            mockState.currentTaskId = 2200;
+            mockState.files["/tmp/scai-eval-run.log"] = mockState.runLog;
+        });
+        await runLocalEvals(runQuery, {
+            includeIds: ["search_folder_natural_language_exact_target"],
+        });
+        const result = mockState.artifactCalls[0]?.result;
+        expect(result.outcome).toBe("incomplete");
+        expect(result.failures?.[0]).toContain("could not isolate the run log or created task");
+    });
+    it("keeps shell adapters wired to the shared eval actions", async () => {
+        const customCommands = {};
+        const runQuery = vi.fn().mockResolvedValue(undefined);
+        registerEvalShellCommands(customCommands, runQuery);
+        await customCommands.eval();
+        await customCommands["eval-random"]();
+        expect(runQuery).toHaveBeenCalledTimes(2);
+    });
+    it("registers eval CLI commands alongside eval-report", async () => {
+        const cmd = new Command();
+        const runQuery = vi.fn().mockResolvedValue(undefined);
+        registerEvalCommands(cmd, runQuery);
+        expect(cmd.commands.map((command) => command.name())).toEqual(expect.arrayContaining([
+            "eval",
+            "eval-random",
+            "evals",
+            "evals-transform",
+            "eval-run",
+            "eval-list",
+            "eval-report",
+        ]));
+    });
+});
+//# sourceMappingURL=evalCommands.test.js.map

package/dist/__tests__/evalCommands.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalCommands.test.js","sourceRoot":"","sources":["../../src/__tests__/evalCommands.test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9D,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAClC,aAAa,EAAE,SAA+B;IAC9C,MAAM,EAAE,EAAE;IACV,KAAK,EAAE,EAA4B;IACnC,aAAa,EAAE,EAAoC;IACnD,wBAAwB,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,IAAY,EAAE,MAAc,EAAE,EAAE,CAAC,QAAQ,IAAI,IAAI,MAAM,EAAE,CAAC;IAC3F,oBAAoB,EAAE,EAAE,CAAC,EAAE,EAAE;IAC7B,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE;CAC7B,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE;IACjB,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,aAAa,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,KAAa,EAAE,IAAY,EAAE,OAA2B,EAAE,EAAE;QACvF,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,OAAO,EAAE,IAAI,KAAK,GAAG,IAAI,QAAQ,KAAK,wBAAwB,EAAE,CAAC;YACnE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,KAAa,EAAE,IAAY,EAAE,EAAE;QAC3D,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC9D,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;QACjC,IAAI,QAAQ,KAAK,wBAAwB,EAAE,CAAC;YAC1C,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7F,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE;QAC5C,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;QACvC,OAAO,UAAU,IAAI,SAAS,CAAC,KAAK;eAC/B,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC;IAChG,CAAC,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;IAE1B,OAAO;QACL,OAAO,EAAE,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE;QAC/E,aAAa;QACb,cAAc;QACd,YAAY;QACZ,UAAU;QACV,SAAS;KACV,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,EAAC,cAAc,EAAC,EAAE;IAChD,MAAM,MAAM,GAAG,MAAM,cAAc,EAAoC,CAAC;IACxE,OAAO;QACL,GAAG,MAAM;QACT,YAAY,EAAE,wBAAwB;KACvC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,EAAE,CAAC,IAAI,CAAC,8BAA8B,EAAE,GAAG,EAAE,CAAC,CAAC;IAC7C,uBAAuB,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC;IAC9C,eAAe,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,aAAa,CAAC;IACrD,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE;IACxB,oBAAoB,EAAE,EAAE,CAAC,EAAE,EAAE;CAC9B,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,gCAAgC,EAAE,GAAG,EAAE,CAAC,CAAC;IAC/C,wBAAwB,EAAE,SAAS,CAAC,wBAAwB;IAC5D,qBAAqB,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,KAA8B,EAAE,EAAE;QAC9D,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,kBAAkB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC/E,CAAC,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,+BAA+B,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9C,oBAAoB,EAAE,SAAS,CAAC,oBAAoB;IACpD,mBAAmB,EAAE,SAAS,CAAC,mBAAmB;CACnD,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,0BAA0B,EAAE,GAAG,EAAE,CAAC,CAAC;IACzC,4BAA4B,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QACzC,YAAY,EAAE,OAAO;QACrB,aAAa,EAAE,OAAO;QACtB,aAAa,EAAE,MAAM;KACtB,CAAC,CAAC;CACJ,CAAC,CAAC,CAAC;AAEJ,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,cAAc,EACd,oBAAoB,EACpB,yBAAyB,EACzB,mBAAmB,EACnB,kBAAkB,EAElB,mBAAmB,EACnB,4BAA4B,EAC5B,aAAa,GACd,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,SAAS,yBAAyB,CAAC,KAA8B;IAC/D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;AAC3H,CAAC;AAED,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,CAAC,aAAa,GAAG,SAAS,CAAC;QACpC,SAAS,CAAC,MAAM,GAAG,EAAE,CAAC;QACtB,SAAS,CAAC,KAAK,GAAG;YAChB,2CAA2C,EACzC,oEAAoE;YACtE,mDAAmD,EACjD,6HAA6H;gBAC7H,iHAAiH;gBACjH,sHAAsH;gBACtH,wHAAwH;YAC1H,iDAAiD,EAC/C,mHAAmH;YACrH,yDAAyD,EACvD,oIAAoI;gBACpI,+HAA+H;gBAC/H,oIAAoI;gBACpI,sIAAsI;gBACtI,yJAAyJ;YAC3J,4CAA4C,EAC1C,iIAAiI;YACnI,oDAAoD,EAClD,6HAA6H;gBAC7H,iHAAiH;gBACjH,mHAAmH;gBACnH,iHAAiH;gBACjH,0IAA0I;YAC5I,kDAAkD,EAChD,iIAAiI;YACnI,qCAAqC,EACnC,sEAAsE;gBACtE,+DAA+D;SAClE,CAAC;QACF,SAAS,CAAC,aAAa,GAAG,EAAE,CAAC;QAC7B,SAAS,CAAC,wBAAwB,CAAC,SAAS,EAAE,CAAC;QAC/C,SAAS,CAAC,oBAAoB,CAAC,SAAS,EAAE,CAAC;QAC3C,SAAS,CAAC,mBAAmB,CAAC,SAAS,EAAE,CAAC;QAC1C,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEtD,MAAM,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAEpC,MAAM,CAAC,QAAQ,CAAC,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE;YACpD,mBAAmB,EAAE,KAAK;YAC1B,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACrD,SAAS,CAAC,aAAa,GAAG,CAAC,SAAS,CAAC,aAAa,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/D,SAAS,CAAC,MAAM,IAAI;gBAClB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,eAAe;aAChB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,MAAM,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAEpC,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,yBAAyB,CAAC,SAAS,CAAC,CAAC,CAAC;QAC7E,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACrD,SAAS,CAAC,aAAa,GAAG,CAAC,SAAS,CAAC,aAAa,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/D,SAAS,CAAC,KAAK,CAAC,2CAA2C,CAAC;gBAC1D,2EAA2E,CAAC;YAC9E,SAAS,CAAC,MAAM,IAAI;gBAClB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,qCAAqC;gBACrC,sBAAsB;aACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,MAAM,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAE7C,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,yBAAyB,CAAC,WAAW,CAAC,CAAC,CAAC;QAC/E,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,MAAM,YAAY,GAAG,cAAc;aAChC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,WAAW,CAAC;aACjG,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE1B,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;YAClD,sDAAsD;YACtD,uDAAuD;YACvD,sDAAsD;YACtD,iEAAiE;YACjE,mDAAmD;YACnD,uDAAuD;YACvD,sDAAsD;YACtD,6CAA6C;SAC9C,CAAC,CAAC,CAAC;QACJ,MAAM,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAa,EAAE,EAAE;YAClE,SAAS,CAAC,aAAa,GAAG,CAAC,SAAS,CAAC,aAAa,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/D,SAAS,CAAC,MAAM,IAAI;gBAClB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,eAAe;aAChB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,MAAM,kBAAkB,CAAC,QAAQ,EAAE,CAAC,4CAA4C,CAAC,CAAC,CAAC;QAEnF,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,QAAQ,CAAC,CAAC,oBAAoB,CACnC,uBAAuB,EACvB,EAAE,mBAAmB,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,iBAAiB,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CACxG,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;QAC9F,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IAChH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+FAA+F,EAAE,KAAK,IAAI,EAAE;QAC7G,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAa,EAAE,EAAE;YAClE,SAAS,CAAC,aAAa,GAAG,CAAC,SAAS,CAAC,aAAa,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/D,SAAS,CAAC,MAAM,IAAI;gBAClB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,eAAe;aAChB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,UAAU,EAAE,CAAC,4CAA4C,CAAC;SAC3D,CAAC,CAAC;QAEH,MAAM,CAAC,uBAAuB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACnD,MAAM,CAAC,uBAAuB,CAAC,CAAC,oBAAoB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9E,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAErE,MAAM,gBAAgB,EAAE,CAAC;QAEzB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,4CAA4C,CAAC,CAAC;QACvE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,wCAAwC,CAAC,CAAC;QACvE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,8BAA8B,CAAC,CAAC;QAC7D,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0EAA0E,EAAE,GAAG,EAAE;QAClF,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,yCAAyC,CAAC,CAAC;QAC7G,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,8BAA8B,CAAC,CAAC;QAC7F,MAAM,mBAAmB,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,mCAAmC,CAAC,CAAC;QAC3G,MAAM,yBAAyB,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,6CAA6C,CAAC,CAAC;QAE3H,MAAM,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAC7D,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QAE7E,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC;YACnD,iBAAiB;YACjB,cAAc;YACd,QAAQ;SACT,CAAC,CAAC;QAEH,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC;YAC5D,QAAQ;YACR,cAAc;YACd,kBAAkB;SACnB,CAAC,CAAC;QAEH,MAAM,CAAC,yBAAyB,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC;YAC5D,uBAAuB;SACxB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACrD,SAAS,CAAC,aAAa,GAAG,CAAC,SAAS,CAAC,aAAa,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/D,SAAS,CAAC,KAAK,CAAC,2CAA2C,CAAC;gBAC1D,2EAA2E,CAAC;YAC9E,SAAS,CAAC,MAAM,IAAI;gBAClB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,qCAAqC;gBACrC,sBAAsB;aACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,MAAM,kBAAkB,CAAC,QAAQ,EAAE,CAAC,sDAAsD,CAAC,CAAC,CAAC;QAE7F,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,QAAQ,CAAC,CAAC,oBAAoB,CACnC,2IAA2I,EAC3I,EAAE,mBAAmB,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,iBAAiB,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CACxG,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IAC1G,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,KAAK,IAAI,EAAE;QACrF,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACrD,SAAS,CAAC,aAAa,GAAG,GAAG,CAAC;YAC9B,SAAS,CAAC,KAAK,CAAC,2CAA2C,CAAC;gBAC1D,2EAA2E,CAAC;YAC9E,SAAS,CAAC,MAAM,GAAG;gBACjB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,qCAAqC;gBACrC,sBAAsB;aACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,UAAU,EAAE,CAAC,sDAAsD,CAAC;SACrE,CAAC,CAAC;QAEH,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,WAAwD,CAAA,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;aACnG,IAAI,CAAC,qCAAqC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAChF,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACrD,SAAS,CAAC,aAAa,GAAG,GAAG,CAAC;YAC9B,SAAS,CAAC,MAAM,GAAG;gBACjB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,qCAAqC;gBACrC,sBAAsB;aACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,UAAU,EAAE,CAAC,sDAAsD,CAAC;SACrE,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,MAAiC,CAAC;QAC7E,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;QAC1F,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACrD,SAAS,CAAC,aAAa,GAAG,GAAG,CAAC;YAC9B,SAAS,CAAC,KAAK,CAAC,2CAA2C,CAAC;gBAC1D,2CAA2C,CAAC;YAC9C,SAAS,CAAC,MAAM,GAAG;gBACjB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,qCAAqC;gBACrC,sBAAsB;aACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,UAAU,EAAE,CAAC,sDAAsD,CAAC;SACrE,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,MAAiC,CAAC;QAC7E,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iGAAiG,EAAE,GAAG,EAAE;QACzG,MAAM,cAAc,GAAG,cAAc;aAClC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,WAAW,CAAC;aACjG,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACd,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;SACjF,CAAC,CAAC,CAAC;QAEN,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtG,MAAM,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8FAA8F,EAAE,GAAG,EAAE;QACtG,MAAM,gBAAgB,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,uDAAuD,CAAC,CAAC;QAC5H,MAAM,iBAAiB,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,sDAAsD,CAAC,CAAC;QAE5H,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;YACnF,MAAM,CAAC,gBAAgB,CAAC;gBACtB,KAAK,EAAE,MAAM,CAAC,eAAe,CAAC;oBAC5B,iEAAiE;oBACjE,mEAAmE;iBACpE,CAAC;aACH,CAAC;YACF,MAAM,CAAC,gBAAgB,CAAC;gBACtB,KAAK,EAAE,MAAM,CAAC,eAAe,CAAC;oBAC5B,sEAAsE;oBACtE,wEAAwE;iBACzE,CAAC;aACH,CAAC;YACF,MAAM,CAAC,gBAAgB,CAAC;gBACtB,KAAK,EAAE,MAAM,CAAC,eAAe,CAAC;oBAC5B,uEAAuE;oBACvE,yEAAyE;iBAC1E,CAAC;aACH,CAAC;SACH,CAAC,CAAC,CAAC;QACJ,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;YAC5F,iEAAiE;YACjE,kEAAkE;YAClE,iEAAiE;YACjE,6EAA6E;SAC9E,CAAC,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8FAA8F,EAAE,GAAG,EAAE;QACtG,MAAM,qBAAqB,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,iEAAiE,CAAC,CAAC;QAC3I,MAAM,eAAe,GAAG,qBAAqB,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;QACzF,MAAM,kBAAkB,GAAG,qBAAqB,EAAE,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;QAC/F,MAAM,KAAK,GAAG,qBAAqB,EAAE,KAAK,IAAI,EAAE,CAAC;QAEjD,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;QAC3D,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7F,MAAM,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gFAAgF,EAAE,GAAG,EAAE;QACxF,MAAM,gBAAgB,GAAG,cAAc;aACpC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,WAAW,CAAC;aACjG,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAEnC,MAAM,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;YACtD,2IAA2I;YAC3I,oPAAoP;YACpP,gRAAgR;SACjR,CAAC,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qFAAqF,EAAE,KAAK,IAAI,EAAE;QACnG,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACrD,SAAS,CAAC,aAAa,GAAG,GAAG,CAAC;YAC9B,SAAS,CAAC,MAAM,GAAG;gBACjB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,8CAA8C;gBAC9C,sBAAsB;aACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;YAC7D,SAAS,CAAC,KAAK,CAAC,oDAAoD,CAAC,GAAG;gBACtE,6CAA6C;gBAC7C,yDAAyD;gBACzD,GAAG;gBACH,4CAA4C;gBAC5C,sEAAsE;gBACtE,GAAG;aACJ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,UAAU,EAAE,CAAC,sDAAsD,CAAC;SACrE,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,MAA6D,CAAC;QACzG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qFAAqF,EAAE,KAAK,IAAI,EAAE;QACnG,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACrD,SAAS,CAAC,aAAa,GAAG,GAAG,CAAC;YAC9B,SAAS,CAAC,MAAM,GAAG;gBACjB,+EAA+E;gBAC/E,0EAA0E;gBAC1E,6CAA6C;gBAC7C,sBAAsB;aACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;YAC7D,SAAS,CAAC,KAAK,CAAC,mDAAmD,CAAC,GAAG;gBACrE,6CAA6C;gBAC7C,yDAAyD;gBACzD,GAAG;gBACH,4CAA4C;gBAC5C,sEAAsE;gBACtE,GAAG;gBACH,4CAA4C;gBAC5C,2EAA2E;gBAC3E,GAAG;gBACH,6CAA6C;gBAC7C,4EAA4E;gBAC5E,GAAG;aACJ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,UAAU,EAAE,CAAC,uDAAuD,CAAC;SACtE,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,MAAmD,CAAC;QAC/F,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAa,EAAE,EAAE;YAClE,SAAS,CAAC,aAAa,GAAG,CAAC,SAAS,CAAC,aAAa,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/D,SAAS,CAAC,MAAM,IAAI;gBAClB,+FAA+F;gBAC/F,0EAA0E;gBAC1E,eAAe;aAChB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,UAAU,EAAE,CAAC,qCAAqC,CAAC;SACpD,CAAC,CAAC;QAEH,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QACvF,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oFAAoF,EAAE,KAAK,IAAI,EAAE;QAClG,SAAS,CAAC,aAAa,GAAG,IAAI,CAAC;QAC/B,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAc,EAAE,EAAE;YACnE,SAAS,CAAC,aAAa,GAAG,IAAI,CAAC;YAC/B,SAAS,CAAC,MAAM,IAAI;gBAClB,+FAA+F;gBAC/F,0EAA0E;gBAC1E,eAAe;aAChB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,UAAU,EAAE,CAAC,qCAAqC,CAAC;SACpD,CAAC,CAAC;QAEH,MAAM,CAAC,uBAAuB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACvD,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,aAAa,EAAE,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oFAAoF,EAAE,KAAK,IAAI,EAAE;QAClG,SAAS,CAAC,aAAa,GAAG,IAAI,CAAC;QAC/B,SAAS,CAAC,MAAM,GAAG;YACjB,iFAAiF;YACjF,4EAA4E;SAC7E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAE7D,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE;YACrD,SAAS,CAAC,aAAa,GAAG,IAAI,CAAC;YAC/B,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,UAAU,EAAE,CAAC,6CAA6C,CAAC;SAC5D,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,MAAmD,CAAC;QAC/F,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,+CAA+C,CAAC,CAAC;IAC1F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,cAAc,GAAyD,EAAE,CAAC;QAChF,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEtD,yBAAyB,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;QACpD,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,cAAc,CAAC,aAAa,CAAC,EAAE,CAAC;QAEtC,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEtD,oBAAoB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAEpC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;YACnF,MAAM;YACN,aAAa;YACb,OAAO;YACP,iBAAiB;YACjB,UAAU;YACV,WAAW;YACX,aAAa;SACd,CAAC,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["/**\n * Covers shared eval command actions and shell adapters.\n *\n * Why this file exists:\n * - `/evals` should default to the fast converted new-agent suite\n * - `/evals-transform` should isolate the slower transform suite\n * - `/eval-run` and `/eval-list` should stay aligned with the trimmed eval registry\n */\nimport { beforeEach, describe, expect, it, vi } from \"vitest\";\n\nconst mockState = vi.hoisted(() => ({\n  currentTaskId: undefined as number | undefined,\n  runLog: \"\",\n  files: {} as Record<string, string>,\n  artifactCalls: [] as Array<Record<string, unknown>>,\n  createHarnessArtifactDir: vi.fn((kind: string, caseId: string) => `/tmp/${kind}/${caseId}`),\n  startLlmTraceSession: vi.fn(),\n  stopLlmTraceSession: vi.fn(),\n}));\n\nvi.mock(\"fs\", () => {\n  const normalize = (value: string) => String(value);\n  const writeFileSync = vi.fn((_path: string, data: string, options?: { flag?: string }) => {\n    const filePath = normalize(_path);\n    mockState.files[filePath] = String(data);\n    if (options?.flag === \"w\" && filePath === \"/tmp/scai-eval-run.log\") {\n      mockState.runLog = String(data);\n    }\n  });\n  const appendFileSync = vi.fn((_path: string, data: string) => {\n    const filePath = normalize(_path);\n    const next = (mockState.files[filePath] ?? \"\") + String(data);\n    mockState.files[filePath] = next;\n    if (filePath === \"/tmp/scai-eval-run.log\") {\n      mockState.runLog = next;\n    }\n  });\n  const readFileSync = vi.fn((filePath: string) => mockState.files[normalize(filePath)] ?? \"\");\n  const existsSync = vi.fn((filePath: string) => {\n    const normalized = normalize(filePath);\n    return normalized in mockState.files\n      || Object.keys(mockState.files).some((candidate) => candidate.startsWith(`${normalized}/`));\n  });\n  const mkdirSync = vi.fn();\n\n  return {\n    default: { writeFileSync, appendFileSync, readFileSync, existsSync, mkdirSync },\n    writeFileSync,\n    appendFileSync,\n    readFileSync,\n    existsSync,\n    mkdirSync,\n  };\n});\n\nvi.mock(\"../constants.js\", async importOriginal => {\n  const actual = await importOriginal<typeof import(\"../constants.js\")>();\n  return {\n    ...actual,\n    RUN_LOG_PATH: \"/tmp/scai-eval-run.log\",\n  };\n});\n\nvi.mock(\"../commands/EvalReportCmd.js\", () => ({\n  buildEvalReportMarkdown: vi.fn(() => \"report\"),\n  getLatestTaskId: vi.fn(() => mockState.currentTaskId),\n  printEvalReport: vi.fn(),\n  runEvalReportCommand: vi.fn(),\n}));\n\nvi.mock(\"../testing/harnessArtifacts.js\", () => ({\n  createHarnessArtifactDir: mockState.createHarnessArtifactDir,\n  writeHarnessArtifacts: vi.fn((input: Record<string, unknown>) => {\n    mockState.artifactCalls.push(input);\n    return String(input.artifactDir ?? `/tmp/artifacts/${String(input.caseId)}`);\n  }),\n}));\n\nvi.mock(\"../testing/llmTraceSession.js\", () => ({\n  startLlmTraceSession: mockState.startLlmTraceSession,\n  stopLlmTraceSession: mockState.stopLlmTraceSession,\n}));\n\nvi.mock(\"../utils/repoIdentity.js\", () => ({\n  resolveCanonicalRepoIdentity: vi.fn(() => ({\n    repoRootPath: \"/repo\",\n    executionPath: \"/repo\",\n    currentBranch: \"main\",\n  })),\n}));\n\nimport { buildEvalReportMarkdown } from \"../commands/EvalReportCmd.js\";\nimport {\n  evalQueries,\n  listEvalsCommand,\n  localEvalCases,\n  registerEvalCommands,\n  registerEvalShellCommands,\n  runEvalBatchCommand,\n  runEvalByIdCommand,\n  runEvalRandomCommand,\n  runEvalSmokeCommand,\n  runEvalTransformBatchCommand,\n  runLocalEvals,\n} from \"../commands/evalCommands.js\";\nimport { Command } from \"commander\";\n\nfunction getAgentEvalCountForSuite(suite: \"default\" | \"transform\"): number {\n  return localEvalCases.filter((test) => (test.group ?? \"main\") === \"agent\" && (test.suite ?? \"default\") === suite).length;\n}\n\ndescribe(\"eval shell commands\", () => {\n  beforeEach(() => {\n    mockState.currentTaskId = undefined;\n    mockState.runLog = \"\";\n    mockState.files = {\n      \"/repo/playground/test/spatialmap.small.js\":\n        \"function runBootstrapFlow(context) {}\\nrunBootstrapFlow(context)\\n\",\n      \"/repo/playground/test/spatialmap.small.extract.js\":\n        \"function buildProfileToolUrl(profileName, toolPath) { return \\\"/rest/profile/\\\" + profileName + \\\"/tools/\\\" + toolPath; }\\n\" +\n        \"function buildPortalHtmlUrl(profileName) { return buildProfileToolUrl(profileName, \\\"html/client/portal\\\"); }\\n\" +\n        \"function buildLayerPanelUrl(profileName) { return buildProfileToolUrl(profileName, \\\"html/client/layer-panel\\\"); }\\n\" +\n        \"function buildSearchPanelUrl(profileName) { return buildProfileToolUrl(profileName, \\\"html/client/search-panel\\\"); }\\n\",\n      \"/repo/playground/test/spatialmap.small.multi.js\":\n        \"function beginPortalFlow(context) {}\\nfunction beginWidgetFlow(context) {}\\nfunction beginReadyFlow(context) {}\\n\",\n      \"/repo/playground/test/spatialmap.small.multi.extract.js\":\n        \"function buildProfileToolRequestUrl(profileName, toolPath) { return \\\"/rest/profile/\\\" + profileName + \\\"/tools/\\\" + toolPath; }\\n\" +\n        \"function buildPortalHtmlRequestUrl(profileName) { return buildProfileToolRequestUrl(profileName, \\\"html/client/portal\\\"); }\\n\" +\n        \"function buildLayerPanelRequestUrl(profileName) { return buildProfileToolRequestUrl(profileName, \\\"html/client/layer-panel\\\"); }\\n\" +\n        \"function buildSearchPanelRequestUrl(profileName) { return buildProfileToolRequestUrl(profileName, \\\"html/client/search-panel\\\"); }\\n\" +\n        \"function buildPortalSourceRequestUrl(profileName, sessionId) { return \\\"/rest/profile/\\\" + profileName + \\\"/tools/src/client/portal/\\\" + sessionId; }\\n\",\n      \"/repo/playground/test/spatialmap.medium.js\":\n        \"document.querySelector(\\\"body\\\").insertAdjacentHTML(\\\"beforeend\\\", html);\\nconst pageBody = document.querySelector(\\\"body\\\");\\n\",\n      \"/repo/playground/test/spatialmap.medium.extract.js\":\n        \"function buildProfileToolUrl(profileName, toolPath) { return \\\"/rest/profile/\\\" + profileName + \\\"/tools/\\\" + toolPath; }\\n\" +\n        \"function buildPortalHtmlUrl(profileName) { return buildProfileToolUrl(profileName, \\\"html/client/portal\\\"); }\\n\" +\n        \"function buildSidebarHtmlUrl(profileName) { return buildProfileToolUrl(profileName, \\\"html/client/sidebar\\\"); }\\n\" +\n        \"function buildSearchHtmlUrl(profileName) { return buildProfileToolUrl(profileName, \\\"html/client/search\\\"); }\\n\" +\n        \"function buildPortalSourceUrl(profileName, sessionId) { return buildProfileToolUrl(profileName, \\\"src/client/portal/\\\" + sessionId); }\\n\",\n      \"/repo/playground/test/spatialmap.medium.multi.js\":\n        \"document.querySelector(\\\"body\\\").insertAdjacentHTML(\\\"beforeend\\\", html);\\nconst pageBody = document.querySelector(\\\"body\\\");\\n\",\n      \"/repo/playground/test/spatialmap.js\":\n        \"const sessionId = sessionStorage.getItem(\\\"spatialmapSessionId\\\");\\n\" +\n        \"sessionId: sessionStorage.getItem(\\\"spatialmapSessionId\\\"),\\n\",\n    };\n    mockState.artifactCalls = [];\n    mockState.createHarnessArtifactDir.mockClear();\n    mockState.startLlmTraceSession.mockClear();\n    mockState.stopLlmTraceSession.mockClear();\n    vi.clearAllMocks();\n    vi.spyOn(console, \"log\").mockImplementation(() => {});\n  });\n\n  it(\"starts eval queries without continuing the current task\", async () => {\n    const runQuery = vi.fn().mockResolvedValue(undefined);\n\n    await runEvalSmokeCommand(runQuery);\n\n    expect(runQuery).toHaveBeenCalledWith(evalQueries[0], {\n      continueCurrentTask: false,\n      runtime: \"agent\",\n    });\n  });\n\n  it(\"defaults /evals to the fast new-agent suite\", async () => {\n    const runQuery = vi.fn().mockImplementation(async () => {\n      mockState.currentTaskId = (mockState.currentTaskId ?? 500) + 1;\n      mockState.runLog += [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"agentTypes.ts\",\n      ].join(\"\\n\");\n    });\n\n    await runEvalBatchCommand(runQuery);\n\n    expect(runQuery).toHaveBeenCalledTimes(getAgentEvalCountForSuite(\"default\"));\n    expect(runQuery.mock.calls.every(([, options]) => options?.runtime === \"agent\")).toBe(true);\n  });\n\n  it(\"runs the slow transform suite only through /evals-transform\", async () => {\n    const runQuery = vi.fn().mockImplementation(async () => {\n      mockState.currentTaskId = (mockState.currentTaskId ?? 600) + 1;\n      mockState.files[\"/repo/playground/test/spatialmap.small.js\"] =\n        \"function bootstrapPortalFlow(context) {}\\nbootstrapPortalFlow(context);\\n\";\n      mockState.runLog += [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"playground/test/spatialmap.small.js\",\n        \"action=codeTransform\",\n      ].join(\"\\n\");\n      mockState.files[\"/tmp/scai-eval-run.log\"] = mockState.runLog;\n    });\n\n    await runEvalTransformBatchCommand(runQuery);\n\n    expect(runQuery).toHaveBeenCalledTimes(getAgentEvalCountForSuite(\"transform\"));\n    expect(runQuery.mock.calls.every(([, options]) => options?.runtime === \"agent\")).toBe(true);\n  });\n\n  it(\"keeps the transform suite populated with the expected spatialmap eval ids\", () => {\n    const transformIds = localEvalCases\n      .filter((test) => (test.group ?? \"main\") === \"agent\" && (test.suite ?? \"default\") === \"transform\")\n      .map((test) => test.id);\n\n    expect(transformIds).toEqual(expect.arrayContaining([\n      \"transform_spatialmap_small_rename_run_bootstrap_flow\",\n      \"transform_spatialmap_small_extract_profile_url_helper\",\n      \"transform_spatialmap_small_multi_rename_flow_helpers\",\n      \"transform_spatialmap_small_multi_extract_portal_request_helpers\",\n      \"transform_spatialmap_medium_document_body_cleanup\",\n      \"transform_spatialmap_medium_multi_dom_surface_cleanup\",\n      \"transform_spatialmap_medium_extract_profile_tool_url\",\n      \"transform_spatialmap_large_cache_session_id\",\n    ]));\n    expect(transformIds).toHaveLength(8);\n  });\n\n  it(\"eval-run resolves one converted agent eval directly\", async () => {\n    const runQuery = vi.fn().mockImplementation(async (query: string) => {\n      mockState.currentTaskId = (mockState.currentTaskId ?? 200) + 1;\n      mockState.runLog += [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"EvalReportCmd\",\n      ].join(\"\\n\");\n    });\n\n    await runEvalByIdCommand(runQuery, [\"search_file_without_extension_exact_target\"]);\n\n    expect(runQuery).toHaveBeenCalledTimes(1);\n    expect(runQuery).toHaveBeenCalledWith(\n      \"Explain EvalReportCmd\",\n      { continueCurrentTask: false, resumeTaskId: undefined, bindAsCurrentTask: undefined, runtime: \"agent\" }\n    );\n    expect(mockState.artifactCalls).toHaveLength(1);\n    expect(mockState.artifactCalls[0]?.caseId).toBe(\"search_file_without_extension_exact_target\");\n    expect(mockState.artifactCalls[0]?.artifactDir).toBe(\"/tmp/evals/search_file_without_extension_exact_target\");\n  });\n\n  it(\"writes report text for converted agent eval artifacts even without per-eval console reporting\", async () => {\n    const runQuery = vi.fn().mockImplementation(async (query: string) => {\n      mockState.currentTaskId = (mockState.currentTaskId ?? 300) + 1;\n      mockState.runLog += [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"EvalReportCmd\",\n      ].join(\"\\n\");\n    });\n\n    await runLocalEvals(runQuery, {\n      includeIds: [\"search_file_without_extension_exact_target\"],\n    });\n\n    expect(buildEvalReportMarkdown).toHaveBeenCalled();\n    expect(buildEvalReportMarkdown).toHaveBeenCalledWith(301, expect.any(Object));\n    expect(mockState.artifactCalls).toHaveLength(1);\n    expect(mockState.artifactCalls[0]?.reportText).toBe(\"report\");\n  });\n\n  it(\"eval-list omits deleted legacy-only eval ids\", async () => {\n    const logSpy = vi.spyOn(console, \"log\").mockImplementation(() => {});\n\n    await listEvalsCommand();\n\n    const output = logSpy.mock.calls.map((args) => args.join(\" \")).join(\"\\n\");\n    expect(output).toContain(\"search_file_without_extension_exact_target\");\n    expect(output).toContain(\"suite: default\");\n    expect(output).toContain(\"suite: transform\");\n    expect(output).not.toContain(\"multifile_no_early_complete_regression\");\n    expect(output).not.toContain(\"resume_continuity_answer_now\");\n    logSpy.mockRestore();\n  });\n\n  it(\"keeps the refreshed stale-default eval checks aligned with the new agent\", () => {\n    const exactTargetEval = localEvalCases.find((test) => test.id === \"search_file_with_extension_exact_target\");\n    const sqliteEval = localEvalCases.find((test) => test.id === \"verify_pipeline_sqlite_query\");\n    const testEntrypointsEval = localEvalCases.find((test) => test.id === \"final_answer_for_test_entrypoints\");\n    const naturalLanguageFolderEval = localEvalCases.find((test) => test.id === \"search_folder_natural_language_exact_target\");\n\n    expect(exactTargetEval?.query).toBe(\"Explain loopReview.ts\");\n    expect(exactTargetEval?.checks.mustContainAnswer).toEqual([\"loopReview.ts\"]);\n\n    expect(sqliteEval?.checks.mustContainAnswer).toEqual([\n      \"db/fileIndex.ts\",\n      \"db/client.ts\",\n      \"SQLite\",\n    ]);\n\n    expect(testEntrypointsEval?.checks.mustContainAnswer).toEqual([\n      \"vitest\",\n      \"package.json\",\n      \"vitest.config.ts\",\n    ]);\n\n    expect(naturalLanguageFolderEval?.checks.mustContain).toEqual([\n      \"artifact final-answer\",\n    ]);\n  });\n\n  it(\"eval-run still resolves one transform eval directly\", async () => {\n    const runQuery = vi.fn().mockImplementation(async () => {\n      mockState.currentTaskId = (mockState.currentTaskId ?? 700) + 1;\n      mockState.files[\"/repo/playground/test/spatialmap.small.js\"] =\n        \"function bootstrapPortalFlow(context) {}\\nbootstrapPortalFlow(context);\\n\";\n      mockState.runLog += [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"playground/test/spatialmap.small.js\",\n        \"action=codeTransform\",\n      ].join(\"\\n\");\n      mockState.files[\"/tmp/scai-eval-run.log\"] = mockState.runLog;\n    });\n\n    await runEvalByIdCommand(runQuery, [\"transform_spatialmap_small_rename_run_bootstrap_flow\"]);\n\n    expect(runQuery).toHaveBeenCalledTimes(1);\n    expect(runQuery).toHaveBeenCalledWith(\n      \"In playground/test/spatialmap.small.js, rename runBootstrapFlow to bootstrapPortalFlow and update all call sites. Keep behavior the same.\",\n      { continueCurrentTask: false, resumeTaskId: undefined, bindAsCurrentTask: undefined, runtime: \"agent\" }\n    );\n    expect(mockState.artifactCalls[0]?.caseId).toBe(\"transform_spatialmap_small_rename_run_bootstrap_flow\");\n  });\n\n  it(\"captures the direct committed target file in transform eval artifacts\", async () => {\n    const runQuery = vi.fn().mockImplementation(async () => {\n      mockState.currentTaskId = 801;\n      mockState.files[\"/repo/playground/test/spatialmap.small.js\"] =\n        \"function bootstrapPortalFlow(context) {}\\nbootstrapPortalFlow(context);\\n\";\n      mockState.runLog = [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"playground/test/spatialmap.small.js\",\n        \"action=codeTransform\",\n      ].join(\"\\n\");\n      mockState.files[\"/tmp/scai-eval-run.log\"] = mockState.runLog;\n    });\n\n    await runLocalEvals(runQuery, {\n      includeIds: [\"transform_spatialmap_small_rename_run_bootstrap_flow\"],\n    });\n\n    expect(mockState.artifactCalls).toHaveLength(1);\n    expect((mockState.artifactCalls[0]?.outputFiles as Array<{ path: string; content: string }>)[0]?.path)\n      .toBe(\"playground/test/spatialmap.small.js\");\n  });\n\n  it(\"fails a transform eval when the output file keeps the old symbol\", async () => {\n    const runQuery = vi.fn().mockImplementation(async () => {\n      mockState.currentTaskId = 950;\n      mockState.runLog = [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"playground/test/spatialmap.small.js\",\n        \"action=codeTransform\",\n      ].join(\"\\n\");\n      mockState.files[\"/tmp/scai-eval-run.log\"] = mockState.runLog;\n    });\n\n    await runLocalEvals(runQuery, {\n      includeIds: [\"transform_spatialmap_small_rename_run_bootstrap_flow\"],\n    });\n\n    const result = mockState.artifactCalls[0]?.result as { failures?: string[] };\n    expect(result.failures?.some((failure) => failure.includes(\"forbidden token\"))).toBe(true);\n  });\n\n  it(\"fails a transform eval when the direct target file is not valid JavaScript\", async () => {\n    const runQuery = vi.fn().mockImplementation(async () => {\n      mockState.currentTaskId = 975;\n      mockState.files[\"/repo/playground/test/spatialmap.small.js\"] =\n        \"function bootstrapPortalFlow(context) {\\n\";\n      mockState.runLog = [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"playground/test/spatialmap.small.js\",\n        \"action=codeTransform\",\n      ].join(\"\\n\");\n      mockState.files[\"/tmp/scai-eval-run.log\"] = mockState.runLog;\n    });\n\n    await runLocalEvals(runQuery, {\n      includeIds: [\"transform_spatialmap_small_rename_run_bootstrap_flow\"],\n    });\n\n    const result = mockState.artifactCalls[0]?.result as { failures?: string[] };\n    expect(result.failures?.some((failure) => failure.includes(\"syntax check failed\"))).toBe(true);\n  });\n\n  it(\"points transform eval queries at direct committed fixtures with one unique target file per eval\", () => {\n    const transformCases = localEvalCases\n      .filter((test) => (test.group ?? \"main\") === \"agent\" && (test.suite ?? \"default\") === \"transform\")\n      .map((test) => ({\n        query: test.query ?? \"\",\n        path: test.checks.fileMustContain?.[0]?.path ?? test.syntaxCheckPaths?.[0] ?? \"\",\n      }));\n\n    expect(transformCases.every((test) => !test.query.includes(\"playground/test/eval-work/\"))).toBe(true);\n    expect(new Set(transformCases.map((test) => test.path)).size).toBe(transformCases.length);\n  });\n\n  it(\"accepts helper-call forms that omit the literal /tools prefix in extraction eval definitions\", () => {\n    const smallExtractEval = localEvalCases.find((test) => test.id === \"transform_spatialmap_small_extract_profile_url_helper\");\n    const mediumExtractEval = localEvalCases.find((test) => test.id === \"transform_spatialmap_medium_extract_profile_tool_url\");\n\n    expect(smallExtractEval?.checks.fileMustContainAnyOf).toEqual(expect.arrayContaining([\n      expect.objectContaining({\n        anyOf: expect.arrayContaining([\n          \"return buildProfileToolUrl(profileName, \\\"html/client/portal\\\")\",\n          \"return buildProfileToolUrl(profileName) + \\\"/html/client/portal\\\"\",\n        ]),\n      }),\n      expect.objectContaining({\n        anyOf: expect.arrayContaining([\n          \"return buildProfileToolUrl(profileName, \\\"html/client/layer-panel\\\")\",\n          \"return buildProfileToolUrl(profileName) + \\\"/html/client/layer-panel\\\"\",\n        ]),\n      }),\n      expect.objectContaining({\n        anyOf: expect.arrayContaining([\n          \"return buildProfileToolUrl(profileName, \\\"html/client/search-panel\\\")\",\n          \"return buildProfileToolUrl(profileName) + \\\"/html/client/search-panel\\\"\",\n        ]),\n      }),\n    ]));\n    expect(mediumExtractEval?.checks.fileMustContain?.[0]?.tokens).toEqual(expect.arrayContaining([\n      \"return buildProfileToolUrl(profileName, \\\"html/client/portal\\\")\",\n      \"return buildProfileToolUrl(profileName, \\\"html/client/sidebar\\\")\",\n      \"return buildProfileToolUrl(profileName, \\\"html/client/search\\\")\",\n      \"return buildProfileToolUrl(profileName, \\\"src/client/portal/\\\" + sessionId)\",\n    ]));\n  });\n\n  it(\"does not require the small multi extraction eval to rewrite the portal source request helper\", () => {\n    const smallMultiExtractEval = localEvalCases.find((test) => test.id === \"transform_spatialmap_small_multi_extract_portal_request_helpers\");\n    const fileMustContain = smallMultiExtractEval?.checks.fileMustContain?.[0]?.tokens ?? [];\n    const fileMustNotContain = smallMultiExtractEval?.checks.fileMustNotContain?.[0]?.tokens ?? [];\n    const query = smallMultiExtractEval?.query ?? \"\";\n\n    expect(query).not.toContain(\"buildPortalSourceRequestUrl\");\n    expect(fileMustContain.some((token) => token.includes(\"portal/\\\" + sessionId\"))).toBe(false);\n    expect(fileMustNotContain.some((token) => token.includes(\"buildPortalSourceRequestUrl\"))).toBe(false);\n  });\n\n  it(\"keeps the active transform queries copyable with direct committed target paths\", () => {\n    const transformQueries = localEvalCases\n      .filter((test) => (test.group ?? \"main\") === \"agent\" && (test.suite ?? \"default\") === \"transform\")\n      .map((test) => test.query ?? \"\");\n\n    expect(transformQueries).toEqual(expect.arrayContaining([\n      \"In playground/test/spatialmap.small.js, rename runBootstrapFlow to bootstrapPortalFlow and update all call sites. Keep behavior the same.\",\n      \"In playground/test/spatialmap.small.extract.js, extract the repeated /rest/profile/... URL building into a helper named buildProfileToolUrl and use it in buildPortalHtmlUrl, buildLayerPanelUrl, and buildSearchPanelUrl. Keep behavior the same.\",\n      \"In playground/test/spatialmap.medium.extract.js, extract the repeated /rest/profile/... tool URL building into a helper named buildProfileToolUrl and use it in buildPortalHtmlUrl, buildSidebarHtmlUrl, buildSearchHtmlUrl, and buildPortalSourceUrl. Keep behavior the same.\",\n    ]));\n  });\n\n  it(\"labels extraction assertion misses with a likely failure kind in saved eval results\", async () => {\n    const runQuery = vi.fn().mockImplementation(async () => {\n      mockState.currentTaskId = 990;\n      mockState.runLog = [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"playground/test/spatialmap.medium.extract.js\",\n        \"action=codeTransform\",\n      ].join(\"\\n\");\n      mockState.files[\"/tmp/scai-eval-run.log\"] = mockState.runLog;\n      mockState.files[\"/repo/playground/test/spatialmap.medium.extract.js\"] = [\n        \"function buildProfileToolUrl(profileName) {\",\n        \"  return \\\"/rest/profile/\\\" + profileName + \\\"/tools\\\";\",\n        \"}\",\n        \"function buildPortalHtmlUrl(profileName) {\",\n        \"  return buildProfileToolUrl(profileName) + \\\"/html/client/portal\\\";\",\n        \"}\",\n      ].join(\"\\n\");\n    });\n\n    await runLocalEvals(runQuery, {\n      includeIds: [\"transform_spatialmap_medium_extract_profile_tool_url\"],\n    });\n\n    const result = mockState.artifactCalls[0]?.result as { likelyFailureKind?: string; failures?: string[] };\n    expect(result.failures?.length).toBeGreaterThan(0);\n    expect(result.likelyFailureKind).toBe(\"harness-strictness\");\n  });\n\n  it(\"accepts the small extract eval when the helper is used through string concatenation\", async () => {\n    const runQuery = vi.fn().mockImplementation(async () => {\n      mockState.currentTaskId = 991;\n      mockState.runLog = [\n        `[2026-05-04 07:36:54 +02:00] [agent] [task:created] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"playground/test/spatialmap.small.extract.js\",\n        \"action=codeTransform\",\n      ].join(\"\\n\");\n      mockState.files[\"/tmp/scai-eval-run.log\"] = mockState.runLog;\n      mockState.files[\"/repo/playground/test/spatialmap.small.extract.js\"] = [\n        \"function buildProfileToolUrl(profileName) {\",\n        \"  return \\\"/rest/profile/\\\" + profileName + \\\"/tools\\\";\",\n        \"}\",\n        \"function buildPortalHtmlUrl(profileName) {\",\n        \"  return buildProfileToolUrl(profileName) + \\\"/html/client/portal\\\";\",\n        \"}\",\n        \"function buildLayerPanelUrl(profileName) {\",\n        \"  return buildProfileToolUrl(profileName) + \\\"/html/client/layer-panel\\\";\",\n        \"}\",\n        \"function buildSearchPanelUrl(profileName) {\",\n        \"  return buildProfileToolUrl(profileName) + \\\"/html/client/search-panel\\\";\",\n        \"}\",\n      ].join(\"\\n\");\n    });\n\n    await runLocalEvals(runQuery, {\n      includeIds: [\"transform_spatialmap_small_extract_profile_url_helper\"],\n    });\n\n    const result = mockState.artifactCalls[0]?.result as { passed?: boolean; failures?: string[] };\n    expect(result.passed).toBe(true);\n    expect(result.failures).toEqual([]);\n  });\n\n  it(\"matches eval log tokens after stripping ANSI color codes\", async () => {\n    const runQuery = vi.fn().mockImplementation(async (query: string) => {\n      mockState.currentTaskId = (mockState.currentTaskId ?? 400) + 1;\n      mockState.runLog += [\n        `[2026-05-04 07:36:54 +02:00] [agent] [\\x1b[33mtask:created\\x1b[39m] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"agentTypes.ts\",\n      ].join(\"\\n\");\n    });\n\n    await runLocalEvals(runQuery, {\n      includeIds: [\"agent_read_and_answer_simple_target\"],\n    });\n\n    expect(mockState.artifactCalls).toHaveLength(1);\n    expect(mockState.artifactCalls[0]?.caseId).toBe(\"agent_read_and_answer_simple_target\");\n    expect(mockState.artifactCalls[0]?.reportText).toBe(\"report\");\n  });\n\n  it(\"does not attach a stale report task when the scenario did not create a new task id\", async () => {\n    mockState.currentTaskId = 1547;\n    const runQuery = vi.fn().mockImplementation(async (_query: string) => {\n      mockState.currentTaskId = 1547;\n      mockState.runLog += [\n        `[2026-05-04 07:36:54 +02:00] [agent] [\\x1b[33mtask:created\\x1b[39m] status=active run started`,\n        `[2026-05-04 07:37:25 +02:00] [agent] [step:output] artifact final-answer`,\n        \"agentTypes.ts\",\n      ].join(\"\\n\");\n    });\n\n    await runLocalEvals(runQuery, {\n      includeIds: [\"agent_read_and_answer_simple_target\"],\n    });\n\n    expect(buildEvalReportMarkdown).not.toHaveBeenCalled();\n    expect(mockState.artifactCalls).toHaveLength(1);\n    expect(mockState.artifactCalls[0]?.reportText).toBeUndefined();\n  });\n\n  it(\"marks an eval incomplete when the run did not create its own task or owned run log\", async () => {\n    mockState.currentTaskId = 2200;\n    mockState.runLog = [\n      `[2026-05-04 07:36:54 +02:00] [shared] [legacy:output] SearchOrchestrator output`,\n      `{\"stepName\":\"SearchOrchestrator\",\"type\":\"output\",\"content\":{\"route\":\"db\"}}`,\n    ].join(\"\\n\");\n    mockState.files[\"/tmp/scai-eval-run.log\"] = mockState.runLog;\n\n    const runQuery = vi.fn().mockImplementation(async () => {\n      mockState.currentTaskId = 2200;\n      mockState.files[\"/tmp/scai-eval-run.log\"] = mockState.runLog;\n    });\n\n    await runLocalEvals(runQuery, {\n      includeIds: [\"search_folder_natural_language_exact_target\"],\n    });\n\n    const result = mockState.artifactCalls[0]?.result as { outcome?: string; failures?: string[] };\n    expect(result.outcome).toBe(\"incomplete\");\n    expect(result.failures?.[0]).toContain(\"could not isolate the run log or created task\");\n  });\n\n  it(\"keeps shell adapters wired to the shared eval actions\", async () => {\n    const customCommands: Record<string, (...args: string[]) => Promise<void>> = {};\n    const runQuery = vi.fn().mockResolvedValue(undefined);\n\n    registerEvalShellCommands(customCommands, runQuery);\n    await customCommands.eval();\n    await customCommands[\"eval-random\"]();\n\n    expect(runQuery).toHaveBeenCalledTimes(2);\n  });\n\n  it(\"registers eval CLI commands alongside eval-report\", async () => {\n    const cmd = new Command();\n    const runQuery = vi.fn().mockResolvedValue(undefined);\n\n    registerEvalCommands(cmd, runQuery);\n\n    expect(cmd.commands.map((command) => command.name())).toEqual(expect.arrayContaining([\n      \"eval\",\n      \"eval-random\",\n      \"evals\",\n      \"evals-transform\",\n      \"eval-run\",\n      \"eval-list\",\n      \"eval-report\",\n    ]));\n  });\n});\n"]}