scai 0.1.178 → 1.0.0

package/dist/commands/evalCommands.js

@@ -0,0 +1,1337 @@
+/**
+ * Defines local eval cases and shared CLI/shell eval command actions.
+ *
+ * Why this file exists:
+ * - local evals catch new-agent regressions without needing a full benchmark harness
+ * - checks should prefer user-visible output and stable shared runtime-log tokens
+ * - stale architecture names should not stay in the default regression suite after cleanup
+ * - `evalQueries` stay as smoke prompts, while `localEvalCases` stay the real regression suite
+ * - CLI and shell eval commands should call one shared behavior owner
+ * - slower transform coverage should live behind one explicit eval command
+ *
+ * Example eval shapes:
+ * - explicit target -> answer mentions the right file or symbol
+ * - search-heavy question -> shared logs still show the new runtime start and final answer
+ * - repo-wide analysis -> answer stays grounded without relying on legacy trace phrases
+ */
+import fs from "fs";
+import path from "path";
+import { Script } from "node:vm";
+import chalk from "chalk";
+import { RUN_LOG_PATH } from "../constants.js";
+import { buildEvalReportMarkdown, getLatestTaskId, printEvalReport, runEvalReportCommand } from "./EvalReportCmd.js";
+import { createHarnessArtifactDir, writeHarnessArtifacts } from "../testing/harnessArtifacts.js";
+import { startLlmTraceSession, stopLlmTraceSession } from "../testing/llmTraceSession.js";
+import { extractFinalAnswerTextFromRuntimeLog, normalizeRuntimeLogText } from "../testing/runtimeLogReader.js";
+import { resolveCanonicalRepoIdentity } from "../utils/repoIdentity.js";
+const SUPPRESS_FINAL_CONTEXT_ENV = "SCAI_SUPPRESS_FINAL_CONTEXT";
+function createInitialEvalOutcomes(evalIds) {
+    return new Map(evalIds.map((id) => [id, {
+            kind: "incomplete",
+            applicable: true,
+            failures: ["eval did not finish"],
+        }]));
+}
+function recordEvalOutcome(outcomesById, evalId, record) {
+    outcomesById.set(evalId, record);
+}
+export function buildEvalBatchSummary(outcomesById) {
+    let applicable = 0;
+    let passed = 0;
+    let failed = 0;
+    let skipped = 0;
+    let incomplete = 0;
+    for (const outcome of outcomesById.values()) {
+        if (outcome.applicable) {
+            applicable += 1;
+        }
+        switch (outcome.kind) {
+            case "passed":
+                passed += 1;
+                break;
+            case "failed":
+                failed += 1;
+                break;
+            case "skipped":
+                skipped += 1;
+                break;
+            case "incomplete":
+            default:
+                incomplete += 1;
+                break;
+        }
+    }
+    return {
+        total: outcomesById.size,
+        applicable,
+        completed: passed + failed,
+        passed,
+        failed,
+        skipped,
+        incomplete,
+    };
+}
+export function assertEvalSummaryInvariant(summary) {
+    const classifiedTotal = summary.passed + summary.failed + summary.skipped + summary.incomplete;
+    const classifiedCompleted = summary.passed + summary.failed;
+    if (summary.total !== classifiedTotal
+        || summary.completed !== classifiedCompleted
+        || summary.completed > summary.applicable
+        || summary.applicable > summary.total) {
+        throw new Error([
+            "Eval summary totals do not add up.",
+            `total=${summary.total} classified=${classifiedTotal}`,
+            `completed=${summary.completed} classifiedCompleted=${classifiedCompleted}`,
+            `applicable=${summary.applicable} total=${summary.total}`,
+        ].join(" "));
+    }
+}
+export function formatEvalSummaryLine(summary, durationSec) {
+    return `\n[EVAL SUMMARY] total=${summary.total}, applicable=${summary.applicable}, completed=${summary.completed}, passed=${summary.passed}, failed=${summary.failed}, skipped=${summary.skipped}, incomplete=${summary.incomplete}, duration=${durationSec}s`;
+}
+export function writeEvalBatchSummaryArtifact(artifactDir, input) {
+    fs.mkdirSync(artifactDir, { recursive: true });
+    const outcomes = Object.fromEntries(Array.from(input.outcomesById.entries()).map(([evalId, outcome]) => [evalId, outcome]));
+    const outputPath = path.join(artifactDir, "batch_summary.json");
+    fs.writeFileSync(outputPath, JSON.stringify({
+        selectedEvalIds: input.selectedEvalIds,
+        durationSec: input.durationSec,
+        summary: input.summary,
+        outcomes,
+    }, null, 2));
+    return outputPath;
+}
+export const evalQueries = [
+    "Explain how agentMemory, agentLoop, agentActions, and agentStateMachine divide responsibilities.",
+    "Explain how the simple agent chooses between plan, search-db, search-regex, search-list-directory, triage-file, read-file, and final-answer.",
+    "Trace one simple-agent run from understand through final answer and show where state transitions happen.",
+    "How do runtime.log, runtime.ndjson, and llm_calls differ, and where is each written?",
+    "Trace how generate() timing reaches structured logs and the console presentation layer.",
+    "How does scai config dev-output affect console timing for both runtimes?",
+    "Where is console presentation centralized, and which normal runtime messages flow through it?",
+    "How are local-time timestamps formatted across runtime.log, runtime.ndjson, debug logs, and llm trace files?",
+    "How does /resume for the simple agent work, and when does it reuse in-memory state?",
+    "Where are readable runtime-local ids like session-001 and task-001 assigned, and where do they appear?",
+    "Which simple-agent files under cli/src/agents are missing targeted test coverage?",
+    "Summarize CLI architecture from index.ts, commands/factory.ts, and runQueryWithDaemonControl.ts.",
+    "How does this repo run tests and where are the test entry points configured?",
+    "Are there flaky-test signals in __tests__ or test scripts, and where would retries be added?",
+    "Where are SQLite queries defined in db/fileIndex.ts and db/client.ts?",
+    "Map the full run lifecycle in MainAgent and point out where routing decisions are made.",
+    "For repo-wide analysis questions, which method decides verify wave budget and why?",
+    "Compare runSearch, runVerify, and runResearch responsibilities and identify overlap risks.",
+    "List all gate checks (phase/scope/route/readiness/research) and the exact stop conditions.",
+    "Explain why a repo-wide question might still end up analyzing only a few files.",
+];
+export const localEvalScenarios = [];
+function createTransformEvalCase(options) {
+    return {
+        id: options.id,
+        group: "agent",
+        suite: "transform",
+        query: options.queryTemplate.split("$TARGET").join(options.targetPath),
+        runOptions: {
+            runtime: "agent",
+        },
+        when: {
+            mustContainInRunLog: [
+                "action=codeTransform",
+            ],
+        },
+        syntaxCheckPaths: [options.targetPath],
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                options.targetPath,
+            ],
+            fileMustContain: options.fileMustContain?.length
+                ? [{ path: options.targetPath, tokens: options.fileMustContain }]
+                : undefined,
+            fileMustNotContain: options.fileMustNotContain?.length
+                ? [{ path: options.targetPath, tokens: options.fileMustNotContain }]
+                : undefined,
+            fileMustContainAnyOf: options.fileMustContainAnyOf,
+        },
+    };
+}
+export const localEvalCases = [
+    {
+        id: "agent_read_and_answer_simple_target",
+        group: "agent",
+        query: "Explain what cli/src/agents/agentTypes.ts is for.",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "agentTypes.ts",
+            ],
+        },
+        when: {
+            mustContainInRunLog: [
+                "action=read-file",
+            ],
+            mustNotContainInRunLog: [
+                "action=triage-file",
+            ],
+        },
+    },
+    {
+        id: "agent_broad_triage_then_full_read",
+        group: "agent",
+        query: "Explain how the modular agent chooses between triage-file and read-file.",
+        runOptions: {
+            runtime: "agent",
+        },
+        when: {
+            mustContainInRunLog: [
+                "action=triage-file",
+                "action=read-file",
+            ],
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "triage-file",
+                "read-file",
+            ],
+        },
+    },
+    {
+        id: "agent_ranked_candidate_promotion",
+        group: "agent",
+        query: "Explain how agentMemory, agentLoop, agentActions, and agentStateMachine divide responsibilities.",
+        runOptions: {
+            runtime: "agent",
+        },
+        when: {
+            mustContainInRunLog: [
+                "action=read-file",
+            ],
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "agentLoop",
+                "agentActions",
+            ],
+        },
+    },
+    {
+        id: "agent_list_plan_sections",
+        group: "agent",
+        query: "List the main sections in plan.md.",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "plan.md",
+            ],
+        },
+    },
+    {
+        id: "agent_cli_architecture_summary",
+        group: "agent",
+        query: "Summarize CLI architecture from index.ts, commands/factory.ts, and runQueryWithDaemonControl.ts.",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "index.ts",
+                "factory.ts",
+                "runQueryWithDaemonControl.ts",
+            ],
+        },
+    },
+    {
+        id: "routing_repo_wide_question",
+        group: "agent",
+        query: "summarize this repo architecture and identify weak coupling points",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "module",
+                "architecture",
+            ],
+        },
+    },
+    {
+        id: "verify_pipeline_sqlite_query",
+        group: "agent",
+        query: "Where are SQLite queries defined in db/fileIndex.ts and db/client.ts?",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "db/fileIndex.ts",
+                "db/client.ts",
+                "SQLite",
+            ],
+        },
+    },
+    {
+        id: "search_folder_basename_exact_target",
+        group: "agent",
+        query: "check guards",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "executionPolicyResolver.ts",
+                "guardState.ts",
+                "resolveProgressState.ts",
+            ],
+        },
+    },
+    {
+        id: "search_folder_path_exact_target",
+        group: "agent",
+        query: "check agents/guards",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "executionPolicyResolver.ts",
+                "guardState.ts",
+                "resolveProgressState.ts",
+            ],
+        },
+    },
+    {
+        id: "search_folder_natural_language_exact_target",
+        group: "agent",
+        query: "please tell me what the three files in the guards folder do?",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "executionPolicyResolver.ts",
+                "guardState.ts",
+                "resolveProgressState.ts",
+            ],
+        },
+    },
+    {
+        id: "search_folder_absolute_path_exact_target",
+        group: "agent",
+        query: "/Users/rzs/dev/repos/scai/cli/src/agents/guards",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "executionPolicyResolver.ts",
+            ],
+        },
+    },
+    {
+        id: "search_file_without_extension_exact_target",
+        group: "agent",
+        query: "Explain EvalReportCmd",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "EvalReportCmd",
+            ],
+        },
+    },
+    {
+        id: "search_file_with_extension_exact_target",
+        group: "agent",
+        query: "Explain loopReview.ts",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "loopReview.ts",
+            ],
+        },
+    },
+    {
+        id: "search_filename_with_spaces_between_words",
+        group: "agent",
+        query: "Explain final answer module",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "Final Answer Module",
+                "buildFinalAnswer",
+            ],
+        },
+    },
+    {
+        id: "search_snippet_without_filename",
+        group: "agent",
+        query: "\"returns raw operator-pattern matches without semantic guessing\"",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "SearchOrchestrator.test.ts",
+            ],
+        },
+    },
+    {
+        id: "search_identical_snippet_multiple_files",
+        group: "agent",
+        query: "\"const currentQuestionId = currentStep.questionId ?? (orderedQuestions.length === 1 ? orderedQuestions[0].id : undefined);\"",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "analysisPlanGenStep.ts",
+                "transformPlanGenStep.ts",
+            ],
+        },
+    },
+    {
+        id: "search_operator_pattern_query",
+        group: "agent",
+        query: "runSearch|semanticSearchFiles|plannerSearchFiles",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "runSearch",
+            ],
+        },
+    },
+    {
+        id: "search_unresolved_folder_fallback",
+        group: "agent",
+        query: "check nonexistent-folder-name",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+            ],
+        },
+    },
+    {
+        id: "analysis_plan_and_semantic_pass",
+        group: "agent",
+        query: "Where does semanticAnalysis merge prior state, and what fields should remain semantic-only?",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+                "semanticAnalysisModule.ts",
+            ],
+        },
+    },
+    {
+        id: "final_answer_for_test_entrypoints",
+        group: "agent",
+        query: "How does this repo run tests and where are the test entry points configured?",
+        runOptions: {
+            runtime: "agent",
+        },
+        checks: {
+            mustContain: [
+                "[agent] [task:created]",
+                "artifact final-answer",
+            ],
+            mustContainAnswer: [
+                "vitest",
+                "package.json",
+                "vitest.config.ts",
+            ],
+        },
+    },
+    /**
+     * Transform evals for the spatialmap practice files.
+     *
+     * These stay in a separate slow suite so the default `/evals` run stays fast.
+     */
+    createTransformEvalCase({
+        id: "transform_spatialmap_small_rename_run_bootstrap_flow",
+        targetPath: "playground/test/spatialmap.small.js",
+        queryTemplate: "In $TARGET, rename runBootstrapFlow to bootstrapPortalFlow and update all call sites. Keep behavior the same.",
+        fileMustContain: [
+            "function bootstrapPortalFlow(",
+            "bootstrapPortalFlow(context)",
+        ],
+        fileMustNotContain: [
+            "runBootstrapFlow",
+        ],
+    }),
+    createTransformEvalCase({
+        id: "transform_spatialmap_small_extract_profile_url_helper",
+        targetPath: "playground/test/spatialmap.small.extract.js",
+        queryTemplate: "In $TARGET, extract the repeated /rest/profile/... URL building into a helper named buildProfileToolUrl and use it in buildPortalHtmlUrl, buildLayerPanelUrl, and buildSearchPanelUrl. Keep behavior the same.",
+        fileMustContain: [
+            "function buildProfileToolUrl(",
+        ],
+        fileMustContainAnyOf: [
+            {
+                path: "playground/test/spatialmap.small.extract.js",
+                anyOf: [
+                    "return buildProfileToolUrl(profileName, \"html/client/portal\")",
+                    "return buildProfileToolUrl(profileName) + \"/html/client/portal\"",
+                ],
+            },
+            {
+                path: "playground/test/spatialmap.small.extract.js",
+                anyOf: [
+                    "return buildProfileToolUrl(profileName, \"html/client/layer-panel\")",
+                    "return buildProfileToolUrl(profileName) + \"/html/client/layer-panel\"",
+                ],
+            },
+            {
+                path: "playground/test/spatialmap.small.extract.js",
+                anyOf: [
+                    "return buildProfileToolUrl(profileName, \"html/client/search-panel\")",
+                    "return buildProfileToolUrl(profileName) + \"/html/client/search-panel\"",
+                ],
+            },
+        ],
+        fileMustNotContain: [
+            "return \"/rest/profile/\" + profileName + \"/tools/html/client/portal\";",
+            "return \"/rest/profile/\" + profileName + \"/tools/html/client/layer-panel\";",
+            "return \"/rest/profile/\" + profileName + \"/tools/html/client/search-panel\";",
+        ],
+    }),
+    createTransformEvalCase({
+        id: "transform_spatialmap_small_multi_rename_flow_helpers",
+        targetPath: "playground/test/spatialmap.small.multi.js",
+        queryTemplate: "In $TARGET, rename the beginPortalFlow, beginWidgetFlow, and beginReadyFlow helper family to startPortalFlow, startWidgetFlow, and startReadyFlow, and update all call sites. Keep behavior the same.",
+        fileMustContain: [
+            "function startPortalFlow(",
+            "function startWidgetFlow(",
+            "function startReadyFlow(",
+        ],
+        fileMustNotContain: [
+            "beginPortalFlow",
+            "beginWidgetFlow",
+            "beginReadyFlow",
+        ],
+    }),
+    createTransformEvalCase({
+        id: "transform_spatialmap_small_multi_extract_portal_request_helpers",
+        targetPath: "playground/test/spatialmap.small.multi.extract.js",
+        queryTemplate: "In $TARGET, extract the repeated /rest/profile/... portal request URL building into a helper named buildProfileToolRequestUrl and use it in buildPortalHtmlRequestUrl, buildLayerPanelRequestUrl, and buildSearchPanelRequestUrl. Keep behavior the same.",
+        fileMustContain: [
+            "function buildProfileToolRequestUrl(",
+            "return buildProfileToolRequestUrl(profileName, \"html/client/portal\")",
+            "return buildProfileToolRequestUrl(profileName, \"html/client/layer-panel\")",
+            "return buildProfileToolRequestUrl(profileName, \"html/client/search-panel\")",
+        ],
+        fileMustNotContain: [
+            "function buildPortalHtmlRequestUrl(profileName) { return \"/rest/profile/\" + profileName + \"/tools/html/client/portal\"; }",
+            "function buildLayerPanelRequestUrl(profileName) { return \"/rest/profile/\" + profileName + \"/tools/html/client/layer-panel\"; }",
+            "function buildSearchPanelRequestUrl(profileName) { return \"/rest/profile/\" + profileName + \"/tools/html/client/search-panel\"; }",
+        ],
+    }),
+    createTransformEvalCase({
+        id: "transform_spatialmap_medium_document_body_cleanup",
+        targetPath: "playground/test/spatialmap.medium.js",
+        queryTemplate: "In $TARGET, add a helper named getPageBody that returns document.body, then use it in prepareSurface, loadPortalMarkup, loadSidebarMarkup, loadSearchMarkup, syncSurfaceShell, syncSurfacePanels, syncSurfaceMessages, syncSurfaceLayout, and syncSurfaceTheme. Keep behavior the same.",
+        fileMustContain: [
+            "function getPageBody()",
+            "document.body",
+        ],
+        fileMustNotContain: [
+            "document.querySelector(\"body\").insertAdjacentHTML(",
+            "const pageBody = document.querySelector(\"body\");",
+        ],
+    }),
+    createTransformEvalCase({
+        id: "transform_spatialmap_medium_multi_dom_surface_cleanup",
+        targetPath: "playground/test/spatialmap.medium.multi.js",
+        queryTemplate: "In $TARGET, add a helper named getPageBody that returns document.body, then use it in the markup-loading and syncSurface helpers instead of repeating document.querySelector(\"body\"). Keep behavior the same.",
+        fileMustContain: [
+            "function getPageBody()",
+            "return document.body;",
+        ],
+        fileMustNotContain: [
+            "document.querySelector(\"body\")",
+            "context.widget.body",
+        ],
+    }),
+    createTransformEvalCase({
+        id: "transform_spatialmap_medium_extract_profile_tool_url",
+        targetPath: "playground/test/spatialmap.medium.extract.js",
+        queryTemplate: "In $TARGET, extract the repeated /rest/profile/... tool URL building into a helper named buildProfileToolUrl and use it in buildPortalHtmlUrl, buildSidebarHtmlUrl, buildSearchHtmlUrl, and buildPortalSourceUrl. Keep behavior the same.",
+        fileMustContain: [
+            "function buildProfileToolUrl(",
+            "return buildProfileToolUrl(profileName, \"html/client/portal\")",
+            "return buildProfileToolUrl(profileName, \"html/client/sidebar\")",
+            "return buildProfileToolUrl(profileName, \"html/client/search\")",
+            "return buildProfileToolUrl(profileName, \"src/client/portal/\" + sessionId)",
+        ],
+        fileMustNotContain: [
+            "return \"/rest/profile/\" + profileName + \"/tools/html/client/portal\";",
+            "return \"/rest/profile/\" + profileName + \"/tools/html/client/sidebar\";",
+            "return \"/rest/profile/\" + profileName + \"/tools/html/client/search\";",
+            "return \"/rest/profile/\" + profileName + \"/tools/src/client/portal/\" + sessionId;",
+        ],
+    }),
+    createTransformEvalCase({
+        id: "transform_spatialmap_large_cache_session_id",
+        targetPath: "playground/test/spatialmap.js",
+        queryTemplate: "In $TARGET, cache sessionStorage.getItem(\"spatialmapSessionId\") in a const named sessionId inside doLoadNow and reuse it in loadToolsConfig, loadToolRequires, and initSpatialmapWidget. Keep behavior the same.",
+        fileMustContain: [
+            "const sessionId = sessionStorage.getItem(\"spatialmapSessionId\");",
+            "sessionId: sessionId,",
+        ],
+        fileMustNotContain: [
+            "sessionStorage.getItem(\"spatialmapSessionId\"),",
+        ],
+    }),
+];
+export async function runEvalSmokeCommand(runQuery) {
+    const query = evalQueries[0];
+    console.log(chalk.bold(`[EVAL QUERY] ${query}`));
+    await runQuery(query, { continueCurrentTask: false, runtime: "agent" });
+}
+export async function runEvalRandomCommand(runQuery) {
+    const query = pickRandom(evalQueries);
+    console.log(`\n[eval-random] Selected query:\n-> ${query}\n`);
+    await runQuery(query, { continueCurrentTask: false, runtime: "agent" });
+}
+export async function runEvalBatchCommand(runQuery) {
+    await runLocalEvals(runQuery, { group: "agent", suite: "default" });
+}
+export async function runEvalTransformBatchCommand(runQuery) {
+    await runLocalEvals(runQuery, { group: "agent", suite: "transform" });
+}
+export async function runEvalByIdCommand(runQuery, evalIds) {
+    if (evalIds.length === 0) {
+        console.log(chalk.yellow("Usage: eval-run <id> [id...]"));
+        console.log(chalk.yellow(`Available ids: ${localEvalCases.map(test => test.id).join(", ")}`));
+        return;
+    }
+    await runLocalEvals(runQuery, { includeIds: evalIds });
+}
+export async function listEvalsCommand() {
+    console.log(chalk.cyan("\nAvailable evals:"));
+    for (const test of localEvalCases) {
+        const scenario = resolveEvalScenario(test);
+        console.log(chalk.bold(`- ${test.id}`));
+        console.log(`  group: ${test.group ?? "main"}`);
+        console.log(`  suite: ${test.suite ?? "default"}`);
+        console.log(`  query: ${scenario.query}`);
+    }
+    console.log("");
+}
+export async function runEvalReportShellCommand(runQuery) {
+    await runLocalEvals(runQuery, { reportEachEval: true, group: "agent", suite: "default" });
+    const latestTaskId = getLatestTaskId();
+    if (typeof latestTaskId === "number") {
+        printEvalReport(latestTaskId, {
+            title: "Eval Report | latest task",
+            mermaid: true,
+            evaluate: true,
+        });
+    }
+}
+export function registerEvalCommands(cmd, runQuery) {
+    cmd
+        .command("eval")
+        .description("Run the default smoke eval query")
+        .action(async () => {
+        await runEvalSmokeCommand(runQuery);
+    });
+    cmd
+        .command("eval-random")
+        .description("Run one random smoke eval query")
+        .action(async () => {
+        await runEvalRandomCommand(runQuery);
+    });
+    cmd
+        .command("evals")
+        .description("Run the default local eval suite")
+        .action(async () => {
+        await runEvalBatchCommand(runQuery);
+    });
+    cmd
+        .command("evals-transform")
+        .description("Run the transform-only local eval suite")
+        .action(async () => {
+        await runEvalTransformBatchCommand(runQuery);
+    });
+    cmd
+        .command("eval-run <id> [ids...]")
+        .description("Run one or more local evals by id")
+        .action(async (id, ids) => {
+        await runEvalByIdCommand(runQuery, [id, ...(ids ?? [])]);
+    });
+    cmd
+        .command("eval-list")
+        .description("List local eval ids and queries")
+        .action(async () => {
+        await listEvalsCommand();
+    });
+    cmd
+        .command("eval-report [taskId]")
+        .description("Show task flow timeline, eval checks, and Mermaid graph (defaults to latest task)")
+        .option("--no-mermaid", "Skip Mermaid graph output")
+        .option("--no-evaluate", "Skip flow eval checks")
+        .action((taskId, opts) => {
+        runEvalReportCommand(taskId, {
+            mermaid: opts.mermaid,
+            evaluate: opts.evaluate,
+        });
+    });
+}
+export function registerEvalShellCommands(customCommands, runQuery) {
+    customCommands.eval = async () => runEvalSmokeCommand(runQuery);
+    customCommands["eval-random"] = async () => runEvalRandomCommand(runQuery);
+    customCommands.evals = async () => runEvalBatchCommand(runQuery);
+    customCommands["evals-transform"] = async () => runEvalTransformBatchCommand(runQuery);
+    customCommands["eval-run"] = async (...args) => runEvalByIdCommand(runQuery, args.map((arg) => arg.trim()).filter(Boolean));
+    customCommands["eval-list"] = async () => listEvalsCommand();
+    customCommands["eval-report"] = async () => runEvalReportShellCommand(runQuery);
+}
+function pickRandom(items) {
+    return items[Math.floor(Math.random() * items.length)];
+}
+function countOccurrences(haystack, needle) {
+    if (!needle)
+        return 0;
+    return haystack.split(needle).length - 1;
+}
+function resolveRepoRelativePath(repoRootPath, repoRelativePath) {
+    return path.join(repoRootPath, repoRelativePath);
+}
+/**
+ * Captures the real mutated fixture files that a transform eval asserted on.
+ *
+ * Example:
+ * - syntax check path `playground/test/spatialmap.small.js`
+ * - saved artifact snapshot uses that same committed path
+ */
+function collectOutputFilesForAssertions(assertions, repoRootPath) {
+    const filePaths = new Set();
+    for (const test of assertions) {
+        for (const filePath of test.syntaxCheckPaths ?? []) {
+            filePaths.add(filePath);
+        }
+        for (const check of test.checks.fileMustContain ?? []) {
+            filePaths.add(check.path);
+        }
+        for (const check of test.checks.fileMustNotContain ?? []) {
+            filePaths.add(check.path);
+        }
+    }
+    const files = [];
+    for (const filePath of filePaths) {
+        const absolutePath = resolveRepoRelativePath(repoRootPath, filePath);
+        if (!fs.existsSync(absolutePath)) {
+            continue;
+        }
+        files.push({
+            path: filePath,
+            content: fs.readFileSync(absolutePath, "utf-8"),
+        });
+    }
+    return files;
+}
+function collectFileCheckFailures(checks, repoRootPath, mode) {
+    const failures = [];
+    for (const check of checks ?? []) {
+        const absolutePath = resolveRepoRelativePath(repoRootPath, check.path);
+        if (!fs.existsSync(absolutePath)) {
+            failures.push(`file missing: ${check.path}`);
+            continue;
+        }
+        const content = fs.readFileSync(absolutePath, "utf-8");
+        for (const token of check.tokens) {
+            const hasToken = content.includes(token);
+            if (mode === "must-contain" && !hasToken) {
+                failures.push(`file missing token: ${check.path} -> ${token}`);
+            }
+            if (mode === "must-not-contain" && hasToken) {
+                failures.push(`file contains forbidden token: ${check.path} -> ${token}`);
+            }
+        }
+    }
+    return failures;
+}
+function collectFileAnyTokenFailures(checks, repoRootPath) {
+    const failures = [];
+    for (const check of checks ?? []) {
+        const absolutePath = resolveRepoRelativePath(repoRootPath, check.path);
+        if (!fs.existsSync(absolutePath)) {
+            failures.push(`file missing: ${check.path}`);
+            continue;
+        }
+        const content = fs.readFileSync(absolutePath, "utf-8");
+        if (!check.anyOf.some((token) => content.includes(token))) {
+            failures.push(`file missing any accepted token: ${check.path} -> ${check.anyOf.join(" || ")}`);
+        }
+    }
+    return failures;
+}
+function collectSyntaxFailures(pathsToCheck, repoRootPath) {
+    const failures = [];
+    for (const filePath of pathsToCheck ?? []) {
+        const absolutePath = resolveRepoRelativePath(repoRootPath, filePath);
+        if (!fs.existsSync(absolutePath)) {
+            failures.push(`syntax check file missing: ${filePath}`);
+            continue;
+        }
+        const content = fs.readFileSync(absolutePath, "utf-8");
+        try {
+            new Script(content, { filename: filePath });
+        }
+        catch (error) {
+            failures.push(`syntax check failed: ${filePath} -> ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    return failures;
+}
+/**
+ * Saves one small metadata file for eval scenario runs that feed several assertions.
+ */
+function writeEvalScenarioMetadata(artifactDir, scenario, assertions) {
+    fs.writeFileSync(`${artifactDir}/scenario.json`, JSON.stringify({
+        scenarioId: scenario.id,
+        query: scenario.query,
+        assertionIds: assertions.map(test => test.id),
+    }, null, 2), "utf-8");
+}
+/**
+ * Resolves the execution unit for one assertion.
+ * Example:
+ * - direct eval keeps its own query/setup/runOptions
+ * - grouped mutating eval maps to one shared scenario and runs once
+ */
+function resolveEvalScenario(test) {
+    if (!test.scenarioId) {
+        return {
+            id: test.id,
+            query: test.query ?? "",
+            setupSteps: test.setupSteps,
+            runOptions: test.runOptions,
+        };
+    }
+    const sharedScenario = localEvalScenarios.find(scenario => scenario.id === test.scenarioId);
+    if (!sharedScenario) {
+        throw new Error(`Unknown eval scenario: ${test.scenarioId}`);
+    }
+    return sharedScenario;
+}
+function groupEvalCasesByScenario(selectedEvals) {
+    const grouped = new Map();
+    for (const test of selectedEvals) {
+        const scenario = resolveEvalScenario(test);
+        const existing = grouped.get(scenario.id);
+        if (existing) {
+            existing.assertions.push(test);
+            continue;
+        }
+        grouped.set(scenario.id, {
+            scenario,
+            assertions: [test],
+            runLog: "",
+            finalAnswerText: "",
+        });
+    }
+    return Array.from(grouped.values());
+}
+function extractFinalAnswerText(runLog) {
+    return extractFinalAnswerTextFromRuntimeLog(runLog);
+}
+/**
+ * Keeps only the latest visible run block for the current query when one exists.
+ * Example: mixed shared output + `Query: check guards` -> slice from that query block onward.
+ */
+function isolateRunLogForQuery(runLog, query) {
+    const normalizedRunLog = normalizeRuntimeLogText(runLog);
+    const marker = `Query: ${query.trim()}`;
+    const queryIndex = normalizedRunLog.lastIndexOf(marker);
+    if (queryIndex < 0) {
+        return normalizedRunLog;
+    }
+    const runStartIndex = normalizedRunLog.lastIndexOf("== ", queryIndex);
+    return (runStartIndex >= 0 ? normalizedRunLog.slice(runStartIndex) : normalizedRunLog.slice(queryIndex)).trim();
+}
+/**
+ * Detects whether the captured run log looks like the run that this eval just created.
+ * Example: a log with `Query: Explain index.ts` or `[agent] [task:created]` is owned; stale shared output is not.
+ */
+function hasOwnedEvalRunLog(runLog, query) {
+    if (!runLog.trim()) {
+        return false;
+    }
+    return runLog.includes(`Query: ${query.trim()}`)
+        || runLog.includes("[agent] [task:created]")
+        || runLog.includes("[main] [task:created]");
+}
+function isEvalApplicable(test, runLog) {
+    const normalizedRunLog = normalizeRuntimeLogText(runLog);
+    const mustContain = test.when?.mustContainInRunLog ?? [];
+    for (const token of mustContain) {
+        if (!normalizedRunLog.includes(token)) {
+            return { ok: false, reason: `condition unmet: missing "${token}"` };
+        }
+    }
+    const mustNotContain = test.when?.mustNotContainInRunLog ?? [];
+    for (const token of mustNotContain) {
+        if (normalizedRunLog.includes(token)) {
+            return { ok: false, reason: `condition unmet: found forbidden "${token}"` };
+        }
+    }
+    return { ok: true };
+}
+function deriveTuningHints(failures, runLog) {
+    const hints = new Set();
+    const joined = failures.join(" ").toLowerCase();
+    const runLogLower = runLog.toLowerCase();
+    if (joined.includes("targetsymbols") || joined.includes("\"can\"")) {
+        hints.add("Symbol extraction is too permissive. Tune sentence-starter/stopword filtering in symbol heuristics.");
+    }
+    if (joined.includes("domain-anchors:") || joined.includes("missing token: domain-anchors:")) {
+        hints.add("Domain anchor recall is low. Tune phrase expansions or generic-token exclusions in query anchor extraction.");
+    }
+    if (joined.includes("searchorchestrator") || joined.includes("resolvedtargetfiles")) {
+        hints.add("Exact-target retrieval did not leave a visible search packet. Inspect orchestrator logging and downstream focus derivation.");
+    }
+    if (joined.includes("output | infoplangen") || joined.includes("output | filesearch")) {
+        hints.add("Expansion did not trigger as expected. Tune readiness weak-alignment guard or regex escalation thresholds.");
+    }
+    if (runLogLower.includes("weakly aligned to domain anchors") || runLogLower.includes("weak domain-anchor alignment")) {
+        hints.add("Weak-alignment gate fired. Consider lowering anchor strictness or adding a missing phrase synonym.");
+    }
+    if (hints.size === 0 && failures.length > 0) {
+        hints.add("Inspect failed tokens and tune the nearest retrieval gate (symbol, anchors, readiness, or expansion) with a minimal change.");
+    }
+    return Array.from(hints);
+}
+function buildEvalArtifactReport(taskId, testId, artifactDir, options) {
+    if (typeof taskId !== "number") {
+        return undefined;
+    }
+    try {
+        const reportText = buildEvalReportMarkdown(taskId, {
+            title: `Eval Report | ${testId}`,
+            mermaid: true,
+            evaluate: true,
+            artifactDir,
+        }) ?? undefined;
+        if (options?.reportEachEval) {
+            printEvalReport(taskId, {
+                title: `Eval Report | ${testId}`,
+                mermaid: true,
+                evaluate: true,
+                artifactDir,
+            });
+        }
+        return reportText;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.log(chalk.yellow(`  - report error: ${message}`));
+        return undefined;
+    }
+}
+function resolveEvalQueryOptions(options, latestTaskId) {
+    if (options?.resumeLatestTask) {
+        return {
+            resumeTaskId: latestTaskId,
+            bindAsCurrentTask: options.bindAsCurrentTask,
+            continueCurrentTask: false,
+            runtime: options.runtime,
+        };
+    }
+    return {
+        resumeTaskId: options?.resumeTaskId,
+        continueCurrentTask: options?.continueCurrentTask ?? false,
+        bindAsCurrentTask: options?.bindAsCurrentTask,
+        runtime: options?.runtime,
+    };
+}
+function inferLikelyFailureKind(test, failures, runtimeError) {
+    if ((test.suite ?? "default") !== "transform") {
+        return undefined;
+    }
+    if (runtimeError || failures.some((failure) => failure.includes("runtime error") || failure.includes("syntax check failed"))) {
+        return "runtime-failure";
+    }
+    if (test.id === "transform_spatialmap_small_multi_extract_portal_request_helpers") {
+        return failures.length > 0 ? "partial-transform-miss" : undefined;
+    }
+    const onlyTokenShapeFailures = failures.length > 0 && failures.every((failure) => failure.includes("file missing token:")
+        || failure.includes("file contains forbidden token:"));
+    if (onlyTokenShapeFailures && (test.id === "transform_spatialmap_small_extract_profile_url_helper"
+        || test.id === "transform_spatialmap_medium_extract_profile_tool_url")) {
+        return "harness-strictness";
+    }
+    return failures.length > 0 ? "partial-transform-miss" : undefined;
+}
+export async function runLocalEvals(runQuery, options) {
+    const previousSuppress = process.env[SUPPRESS_FINAL_CONTEXT_ENV];
+    process.env[SUPPRESS_FINAL_CONTEXT_ENV] = "1";
+    const requestedIds = new Set(options?.includeIds ?? []);
+    const selectedEvalsBase = requestedIds.size > 0
+        ? localEvalCases.filter(test => requestedIds.has(test.id))
+        : localEvalCases;
+    const selectedEvals = options?.group
+        ? selectedEvalsBase.filter((test) => (test.group ?? "main") === options.group)
+        : selectedEvalsBase;
+    const suiteFilteredEvals = options?.suite
+        ? selectedEvals.filter((test) => (test.suite ?? "default") === options.suite)
+        : selectedEvals;
+    if (requestedIds.size > 0 && suiteFilteredEvals.length === 0) {
+        console.log(chalk.yellow("[EVAL] No matching eval ids."));
+        console.log(chalk.yellow(`Available ids: ${localEvalCases.map(test => test.id).join(", ")}`));
+        return;
+    }
+    if (!requestedIds.size && (options?.group || options?.suite) && suiteFilteredEvals.length === 0) {
+        const groupLabel = options?.group ? `group "${options.group}"` : undefined;
+        const suiteLabel = options?.suite ? `suite "${options.suite}"` : undefined;
+        const filterSummary = [groupLabel, suiteLabel].filter(Boolean).join(" and ");
+        console.log(chalk.yellow(`[EVAL] No matching evals for ${filterSummary}.`));
+        return;
+    }
+    const startedAt = Date.now();
+    const selectedScenarioRuns = groupEvalCasesByScenario(suiteFilteredEvals);
+    const repoRootPath = resolveCanonicalRepoIdentity().repoRootPath;
+    const outcomesById = createInitialEvalOutcomes(suiteFilteredEvals.map((test) => test.id));
+    let batchRunError;
+    try {
+        for (const scenarioRun of selectedScenarioRuns) {
+            const { scenario, assertions } = scenarioRun;
+            scenarioRun.artifactDir = createHarnessArtifactDir("evals", scenario.id);
+            writeEvalScenarioMetadata(scenarioRun.artifactDir, scenario, assertions);
+            fs.mkdirSync(requireRunLogDir(), { recursive: true });
+            fs.writeFileSync(RUN_LOG_PATH, "", { flag: "w" });
+            let latestTaskId = getLatestTaskId() ?? undefined;
+            startLlmTraceSession(scenarioRun.artifactDir);
+            try {
+                for (const step of scenario.setupSteps ?? []) {
+                    const beforeSetupTaskId = getLatestTaskId();
+                    await runQuery(step.query, resolveEvalQueryOptions(step.options, latestTaskId));
+                    const afterSetupTaskId = getLatestTaskId();
+                    latestTaskId =
+                        typeof afterSetupTaskId === "number" &&
+                            (beforeSetupTaskId == null || afterSetupTaskId >= beforeSetupTaskId)
+                            ? afterSetupTaskId
+                            : latestTaskId;
+                }
+                const assertionIds = assertions.map(test => test.id).join(", ");
+                console.log(chalk.cyan(`\n[EVAL] ${scenario.id}`));
+                console.log(chalk.dim(`  assertions: ${assertionIds}`));
+                console.log(chalk.bold(`[EVAL QUERY] ${scenario.query}`));
+                fs.appendFileSync(RUN_LOG_PATH, `\n[EVAL] ${scenario.id}\n[EVAL ASSERTIONS] ${assertionIds}\n[EVAL QUERY] ${scenario.query}\n`, "utf-8");
+                const beforeTaskId = getLatestTaskId();
+                try {
+                    await runQuery(scenario.query, resolveEvalQueryOptions(scenario.runOptions, latestTaskId));
+                }
+                catch (err) {
+                    const message = err instanceof Error ? err.message : String(err);
+                    scenarioRun.runtimeError = message;
+                }
+                const afterTaskId = getLatestTaskId();
+                scenarioRun.createdTaskId =
+                    typeof afterTaskId === "number" &&
+                        (beforeTaskId == null || afterTaskId > beforeTaskId)
+                        ? afterTaskId
+                        : undefined;
+                const rawRunLog = fs.existsSync(RUN_LOG_PATH)
+                    ? fs.readFileSync(RUN_LOG_PATH, "utf-8")
+                    : "";
+                scenarioRun.runLog = isolateRunLogForQuery(rawRunLog, scenario.query);
+                scenarioRun.finalAnswerText = extractFinalAnswerText(scenarioRun.runLog);
+                scenarioRun.outputFiles = collectOutputFilesForAssertions(assertions, repoRootPath);
+                if (!scenarioRun.createdTaskId && !hasOwnedEvalRunLog(scenarioRun.runLog, scenario.query)) {
+                    scenarioRun.harnessError = "eval harness could not isolate the run log or created task for this scenario";
+                }
+            }
+            finally {
+                stopLlmTraceSession();
+            }
+            for (const test of assertions) {
+                let reportText;
+                const testQuery = test.query ?? scenario.query;
+                const normalizedRunLog = normalizeRuntimeLogText(scenarioRun.runLog);
+                if (scenarioRun.harnessError) {
+                    console.log(chalk.yellow(`[INCOMPLETE] ${test.id}`));
+                    console.log(chalk.yellow(`  - ${scenarioRun.harnessError}`));
+                    const artifactDir = writeHarnessArtifacts({
+                        kind: "evals",
+                        caseId: test.id,
+                        query: testQuery,
+                        taskId: scenarioRun.createdTaskId ?? undefined,
+                        artifactDir: scenarioRun.artifactDir,
+                        runLog: scenarioRun.runLog,
+                        finalAnswerText: scenarioRun.finalAnswerText,
+                        result: {
+                            id: test.id,
+                            applicable: true,
+                            passed: false,
+                            outcome: "incomplete",
+                            failures: [scenarioRun.harnessError],
+                        },
+                        outputFiles: scenarioRun.outputFiles,
+                    });
+                    recordEvalOutcome(outcomesById, test.id, {
+                        kind: "incomplete",
+                        applicable: true,
+                        failures: [scenarioRun.harnessError],
+                        artifactDir,
+                    });
+                    console.log(chalk.dim(`  artifacts: ${artifactDir}`));
+                    continue;
+                }
+                if (scenarioRun.runtimeError) {
+                    recordEvalOutcome(outcomesById, test.id, {
+                        kind: "failed",
+                        applicable: true,
+                        failures: [`runtime error: ${scenarioRun.runtimeError}`],
+                    });
+                    console.log(chalk.red(`[FAIL] ${test.id}`));
+                    console.log(chalk.red(`  - runtime error: ${scenarioRun.runtimeError}`));
+                    const artifactDir = writeHarnessArtifacts({
+                        kind: "evals",
+                        caseId: test.id,
+                        query: testQuery,
+                        artifactDir: scenarioRun.artifactDir,
+                        runLog: scenarioRun.runLog,
+                        finalAnswerText: scenarioRun.finalAnswerText,
+                        result: {
+                            id: test.id,
+                            applicable: true,
+                            passed: false,
+                            failures: [`runtime error: ${scenarioRun.runtimeError}`],
+                            likelyFailureKind: inferLikelyFailureKind(test, [`runtime error: ${scenarioRun.runtimeError}`], scenarioRun.runtimeError),
+                        },
+                        outputFiles: scenarioRun.outputFiles,
+                    });
+                    recordEvalOutcome(outcomesById, test.id, {
+                        kind: "failed",
+                        applicable: true,
+                        failures: [`runtime error: ${scenarioRun.runtimeError}`],
+                        artifactDir,
+                    });
+                    continue;
+                }
+                const applicability = isEvalApplicable(test, scenarioRun.runLog);
+                if (!applicability.ok) {
+                    reportText = buildEvalArtifactReport(scenarioRun.createdTaskId, test.id, scenarioRun.artifactDir, options);
+                    console.log(chalk.blue(`[SKIP] ${test.id} (${applicability.reason})`));
+                    const artifactDir = writeHarnessArtifacts({
+                        kind: "evals",
+                        caseId: test.id,
+                        query: testQuery,
+                        taskId: scenarioRun.createdTaskId ?? undefined,
+                        artifactDir: scenarioRun.artifactDir,
+                        runLog: scenarioRun.runLog,
+                        finalAnswerText: scenarioRun.finalAnswerText,
+                        reportText,
+                        result: {
+                            id: test.id,
+                            applicable: false,
+                            passed: false,
+                            failures: [applicability.reason ?? "skipped"],
+                            likelyFailureKind: inferLikelyFailureKind(test, [applicability.reason ?? "skipped"], undefined),
+                        },
+                        outputFiles: scenarioRun.outputFiles,
+                    });
+                    recordEvalOutcome(outcomesById, test.id, {
+                        kind: "skipped",
+                        applicable: false,
+                        failures: [applicability.reason ?? "skipped"],
+                        artifactDir,
+                    });
+                    console.log(chalk.dim(`  artifacts: ${artifactDir}`));
+                    continue;
+                }
+                const failures = [];
+                for (const token of test.checks.mustContain ?? []) {
+                    if (!normalizedRunLog.includes(token)) {
+                        failures.push(`missing token: ${token}`);
+                    }
+                }
+                for (const token of test.checks.mustNotContain ?? []) {
+                    if (normalizedRunLog.includes(token)) {
+                        failures.push(`unexpected token: ${token}`);
+                    }
+                }
+                for (const [token, max] of Object.entries(test.checks.maxOccurrences ?? {})) {
+                    const count = countOccurrences(normalizedRunLog, token);
+                    if (count > max) {
+                        failures.push(`too many occurrences: "${token}" => ${count} (max ${max})`);
+                    }
+                }
+                for (const token of test.checks.mustContainAnswer ?? []) {
+                    if (!scenarioRun.finalAnswerText.toLowerCase().includes(token.toLowerCase())) {
+                        failures.push(`final answer missing token: ${token}`);
+                    }
+                }
+                for (const token of test.checks.mustNotContainAnswer ?? []) {
+                    if (scenarioRun.finalAnswerText.toLowerCase().includes(token.toLowerCase())) {
+                        failures.push(`final answer contains forbidden token: ${token}`);
+                    }
+                }
+                failures.push(...collectFileCheckFailures(test.checks.fileMustContain, repoRootPath, "must-contain"), ...collectFileCheckFailures(test.checks.fileMustNotContain, repoRootPath, "must-not-contain"), ...collectFileAnyTokenFailures(test.checks.fileMustContainAnyOf, repoRootPath), ...collectSyntaxFailures(test.syntaxCheckPaths, repoRootPath));
+                if (failures.length === 0) {
+                    console.log(chalk.green(`[PASS] ${test.id}`));
+                }
+                else {
+                    console.log(chalk.red(`[FAIL] ${test.id}`));
+                    for (const failure of failures) {
+                        console.log(chalk.red(`  - ${failure}`));
+                    }
+                    const tuningHints = deriveTuningHints(failures, scenarioRun.runLog);
+                    if (tuningHints.length > 0) {
+                        console.log(chalk.yellow("  tuning hints:"));
+                        for (const hint of tuningHints) {
+                            console.log(chalk.yellow(`  - ${hint}`));
+                        }
+                    }
+                }
+                reportText = buildEvalArtifactReport(scenarioRun.createdTaskId, test.id, scenarioRun.artifactDir, options);
+                const artifactDir = writeHarnessArtifacts({
+                    kind: "evals",
+                    caseId: test.id,
+                    query: testQuery,
+                    taskId: scenarioRun.createdTaskId ?? undefined,
+                    artifactDir: scenarioRun.artifactDir,
+                    runLog: scenarioRun.runLog,
+                    finalAnswerText: scenarioRun.finalAnswerText,
+                    reportText,
+                    result: {
+                        id: test.id,
+                        applicable: applicability.ok,
+                        passed: failures.length === 0,
+                        failures,
+                        likelyFailureKind: inferLikelyFailureKind(test, failures, scenarioRun.runtimeError),
+                    },
+                    outputFiles: scenarioRun.outputFiles,
+                });
+                recordEvalOutcome(outcomesById, test.id, {
+                    kind: failures.length === 0 ? "passed" : "failed",
+                    applicable: true,
+                    failures,
+                    artifactDir,
+                });
+                console.log(chalk.dim(`  artifacts: ${artifactDir}`));
+            }
+        }
+    }
+    catch (err) {
+        batchRunError = err instanceof Error ? err : new Error(String(err));
+    }
+    finally {
+        if (previousSuppress === undefined) {
+            delete process.env[SUPPRESS_FINAL_CONTEXT_ENV];
+        }
+        else {
+            process.env[SUPPRESS_FINAL_CONTEXT_ENV] = previousSuppress;
+        }
+    }
+    const durationSec = ((Date.now() - startedAt) / 1000).toFixed(1);
+    const summary = buildEvalBatchSummary(outcomesById);
+    assertEvalSummaryInvariant(summary);
+    const batchSummaryArtifactDir = createHarnessArtifactDir("evals", "batch-summary");
+    const batchSummaryArtifact = writeEvalBatchSummaryArtifact(batchSummaryArtifactDir, {
+        selectedEvalIds: suiteFilteredEvals.map((test) => test.id),
+        summary,
+        outcomesById,
+        durationSec,
+    });
+    const color = summary.failed === 0 && summary.incomplete === 0 ? chalk.green : chalk.yellow;
+    console.log(color(formatEvalSummaryLine(summary, durationSec)));
+    console.log(chalk.dim(`  batch summary: ${batchSummaryArtifact}`));
+    if (batchRunError) {
+        throw batchRunError;
+    }
+}
+function requireRunLogDir() {
+    const idx = RUN_LOG_PATH.lastIndexOf("/");
+    if (idx <= 0)
+        return ".";
+    return RUN_LOG_PATH.slice(0, idx);
+}
+//# sourceMappingURL=evalCommands.js.map