scai 0.1.170 → 0.1.172

package/dist/agents/MainAgent.js

@@ -10,6 +10,7 @@ import { fileCheckStep } from "./fileCheckStep.js";
 import { analysisPlanGenStep } from "./analysisPlanGenStep.js";
 import { readinessGateStep } from "./readinessGateStep.js";
 import { scopeClassificationStep } from "./scopeClassificationStep.js";
+import { routingDecisionStep } from "./routingDecisionStep.js";
 import { evidenceVerifierStep } from "./evidenceVerifierStep.js";
 import { validateChangesStep } from './validateChangesStep.js';
 import { reasonNextTaskStep } from './reasonNextTaskStep.js';
@@ -19,8 +20,14 @@ import { selectRelevantSourcesStep } from "./selectRelevantSourcesStep.js";
 import { iterationFileSelector } from "./iterationFileSelector.js";
 import { finalAnswerModule } from "../pipeline/modules/finalAnswerModule.js";
 import { reasonNextStep } from "./reasonNextStep.js";
+import { buildLightContext } from "../utils/buildContextualPrompt.js";
+import { semanticSearchFiles } from "../db/fileIndex.js";
+import { NUM_TOPFILES, RELATED_FILES_LIMIT } from "../constants.js";
 import { structuralPreloadStep } from "./structuralPreloadStep.js";
+import { extractFileReferences } from "../utils/extractFileReferences.js";
+import { PREFILTER_STOP_WORDS } from "../fileRules/stopWords.js";
 import chalk from "chalk";
+import path from "path";
 /* ───────────────────────── registry ───────────────────────── */
 const MODULE_REGISTRY = Object.fromEntries(Object.entries(builtInModules).map(([name, mod]) => [name, mod]));
 function resolveModuleForAction(action) {
@@ -49,7 +56,6 @@ export class MainAgent {
      */
     constructor(context, ui) {
         this.runCount = 0;
-        this.maxRuns = 2;
         this.context = context;
         this.query = context.initContext?.userQuery ?? "";
         this.ui = ui;
@@ -60,6 +66,7 @@ export class MainAgent {
             this.runCount = 0;
             await this.runBoot();
             await this.runScope();
+            await this.runInitialRetrieval();
             await this.runGrounding();
             await this.runWorkLoop();
             await this.runFinalize();
@@ -72,7 +79,6 @@ export class MainAgent {
     async runBoot() {
         var _a;
         await understandIntentStep.run({ context: this.context });
-        await resolveExecutionModeStep.run(this.context);
         // Boot the task and get the real DB taskId
         this.taskId = bootTaskForRepo(this.context, getDbForRepo(), (phase, step, ms, desc) => this.logLine(phase, step, ms, desc, { highlight: true }));
         (_a = this.context).task || (_a.task = {
@@ -90,11 +96,38 @@ export class MainAgent {
     /* ───────────── scope ───────────── */
     async runScope() {
         await scopeClassificationStep.run(this.context);
+        await resolveExecutionModeStep.run(this.context);
+        await routingDecisionStep.run(this.context);
+        const routing = this.context.analysis?.routingDecision;
+        if (routing) {
+            this.logLine("TASK", "Routing decision", undefined, `${routing.decision} | search=${routing.allowSearch} | transform=${routing.allowTransform} | scopeLocked=${routing.scopeLocked}`);
+        }
         this.logLine("TASK", "Scope classification complete");
     }
+    /* ───────────── initial retrieval ───────────── */
+    async runInitialRetrieval() {
+        const { rawUserQuery, retrievalQuery } = this.resolveInitialRetrievalQueries();
+        const t = this.startTimer();
+        try {
+            const results = await this.fetchInitialRetrievalResults(retrievalQuery);
+            const promptArgs = this.buildInitialRetrievalPromptArgs(results, retrievalQuery);
+            const seededContext = await buildLightContext(promptArgs);
+            const mergedRelatedCount = this.mergeSeededInitialContext(rawUserQuery, seededContext);
+            const prefilter = this.applyDeterministicPreGroundingPrefilter(retrievalQuery);
+            this.logLine("ANALYSIS", "initialRetrieval", t(), `${results.length} result(s), ${mergedRelatedCount} candidate file(s), prefilter ${prefilter.before} -> ${prefilter.after}`);
+        }
+        catch (err) {
+            this.logLine("ANALYSIS", "initialRetrieval", t(), `failed: ${String(err)}`);
+        }
+    }
+    /* ───────────── grounding ───────────── */
     async runGrounding() {
         let ready = false;
-        while (this.runCount < this.maxRuns) {
+        const maxGroundingWaves = this.getGroundingWaveBudget();
+        let groundingWave = 0;
+        while (groundingWave < maxGroundingWaves) {
+            groundingWave++;
+            this.logLine("ANALYSIS", "groundingWave", undefined, `wave ${groundingWave}/${maxGroundingWaves}`);
             // ---------------- EVIDENCE PIPELINE ----------------
             // -------- STRUCTURAL PRELOAD --------
             const t0 = this.startTimer();
@@ -112,7 +145,7 @@ export class MainAgent {
             // ---------------- READINESS GATE ----------------
             const t4 = this.startTimer();
             await readinessGateStep.run(this.context);
-            this.logLine("HASINFO", "readinessGate", t4());
+            this.logLine("ANALYSIS", "readinessGate", t4());
             ready = this.context.analysis?.readiness?.decision === "ready";
             if (ready) {
                 break;
@@ -135,82 +168,67 @@ export class MainAgent {
                 }
                 this.logLine("PLAN", "infoPlanGen", t(), undefined, { highlight: false });
             }
-            this.runCount++;
             this.logLine("HASINFO", "Not ready — looping back to evidence collection", undefined, undefined, { highlight: false });
         }
+        const selectedFiles = this.context.analysis?.focus?.selectedFiles ?? [];
+        if (selectedFiles.length > 0) {
+            this.context.analysis.readiness.decision = "ready";
+            this.logLine("ANALYSIS", "readinessOverrideFromSelectedFiles", undefined, `${selectedFiles.length} selected file(s) available after grounding`);
+        }
     }
-    /* ───────────── finalize ───────────── */
-    async runFinalize() {
-        await finalAnswerModule.run({ query: this.query, context: this.context });
-        persistTaskData(this.context, this.taskId, getDbForRepo(), this.logLine.bind(this));
-        this.logLine("TASK", "Finalize complete", undefined, undefined, { highlight: false });
+    /**
+     * Resolves grounding wave budget from current routing metadata.
+     * Example: scope=repo-wide + decision=needs-info => 4 waves.
+     */
+    getGroundingWaveBudget() {
+        const scope = this.context.analysis?.scopeType ?? "repo-wide";
+        const decision = this.context.analysis?.routingDecision?.decision ?? "has-info";
+        const allowSearch = this.context.analysis?.routingDecision?.allowSearch ?? true;
+        let budget = 2;
+        if (!allowSearch || scope === "none")
+            budget = 1;
+        else if (scope === "single-file" && decision === "has-info")
+            budget = 2;
+        else if (scope === "multi-file")
+            budget = 3;
+        else if (scope === "repo-wide" && decision === "needs-info")
+            budget = 4;
+        this.logLine("ANALYSIS", "groundingBudget", undefined, `scope=${scope}, decision=${decision}, search=${allowSearch}, maxWaves=${budget}`);
+        return budget;
     }
     /* ───────────── work loop ───────────── */
     async runWorkLoop() {
-        const readinessDecision = this.context.analysis?.readiness?.decision;
-        const readinessConfidence = this.context.analysis?.readiness?.confidence ?? 0;
-        if (readinessDecision !== "ready") {
-            // ❌ Graceful fallback instead of throwing
-            this.context.task.status = "deferred";
-            this.context.task.reason = `Readiness not achieved (decision=${readinessDecision}, confidence=${readinessConfidence})`;
-            persistTaskData(this.context, this.taskId, getDbForRepo(), this.logLine.bind(this));
-            this.logLine("TASK", `Cannot start work loop — agent needs more evidence to safely proceed`, undefined, `Readiness: ${readinessDecision}, Confidence: ${readinessConfidence}`, { highlight: true });
+        if (!this.isWorkLoopReady())
             return;
-        }
-        if (!this.context.task) {
-            throw new Error("runWorkLoop: missing task");
-        }
+        this.ensureTaskForWorkLoop();
         const MAX_TASK_STEPS = 5;
         let stepCount = 0;
         while (stepCount < MAX_TASK_STEPS &&
             this.context.task.status === "active") {
-            await reasonNextTaskStep.run(this.context);
-            const nextAction = this.context.analysis?.iterationReasoning?.nextAction;
-            // 🟡 Pause for user clarification
+            const nextAction = await this.resolveNextTaskAction();
             if (nextAction === "request-feedback") {
-                this.context.task.status = "paused";
-                persistTaskData(this.context, this.taskId, getDbForRepo(), this.logLine.bind(this));
+                this.persistTaskStatus("paused");
                 this.logLine("TASK", "Execution paused — awaiting user clarification", undefined, undefined, { highlight: false });
                 return;
             }
-            // 🟢 Completed task
             if (nextAction === "complete") {
-                this.context.task.status = "completed";
-                persistTaskData(this.context, this.taskId, getDbForRepo(), this.logLine.bind(this));
+                this.persistTaskStatus("completed");
                 this.logLine("TASK", "All selected files processed — task complete", undefined, undefined, { highlight: false });
                 return;
             }
             const taskStep = await iterationFileSelector.run(this.context);
             if (!taskStep) {
-                this.context.task.status = "completed";
-                persistTaskData(this.context, this.taskId, getDbForRepo(), this.logLine.bind(this));
+                this.persistTaskStatus("completed");
                 this.logLine("TASK", "No eligible taskStep found — task complete", undefined, undefined, { highlight: false });
                 return;
             }
-            this.context.task.currentStep = taskStep;
             stepCount++;
-            taskStep.taskId = this.taskId;
-            taskStep.stepIndex = stepCount;
-            taskStep.status = "pending";
-            persistTaskStepInsert(taskStep, getDbForRepo());
-            this.logLine("NEW STEP", `Processing taskStep ${stepCount}`, undefined, taskStep.filePath, { highlight: true });
-            taskStep.startTime = Date.now();
-            persistTaskStepStart(taskStep, getDbForRepo());
+            this.startTaskStep(taskStep, stepCount);
             // ---------------------------
             // Step-level iterations
             // ---------------------------
             const stepAction = await this.runStepIterations(taskStep);
-            if (stepAction === "complete") {
-                taskStep.status = "completed";
-                taskStep.endTime = Date.now();
-                persistTaskStepCompletion(taskStep, getDbForRepo());
-                this.logLine("STEP-DONE", `Completed taskStep ${stepCount}`, undefined, taskStep.filePath, { highlight: false });
-            }
-            else {
-                taskStep.status = "pending";
-                persistTaskStepCompletion(taskStep, getDbForRepo());
-                this.logLine("STEP", `Pending taskStep ${stepCount}`, undefined, taskStep.filePath);
-            }
+            this.finishTaskStep(taskStep, stepCount, stepAction);
         }
         this.logLine("TASK", "Max task step limit reached — stopping work loop", undefined, undefined, { highlight: false });
     }
@@ -219,16 +237,17 @@ export class MainAgent {
         const MAX_ITERATIONS = 5;
         let loopCount = 0;
         const getNextIterationAction = () => {
-            const nextAction = this.context.analysis?.iterationReasoning?.nextAction;
-            if (!["continue", "redo-step", "expand-scope", "request-feedback", "complete"].includes(nextAction ?? ""))
-                return "complete";
+            const nextAction = taskStep.result?.stepReasoning?.nextAction;
+            if (!["continue", "redo-step", "expand-scope", "request-feedback", "complete"].includes(nextAction ?? "")) {
+                return "continue";
+            }
             return nextAction;
         };
         while (loopCount < MAX_ITERATIONS) {
             this.runCount++;
             loopCount++;
-            if (this.context.analysis?.iterationReasoning)
-                this.context.analysis.iterationReasoning.nextAction = undefined;
+            if (taskStep.result?.stepReasoning)
+                taskStep.result.stepReasoning.nextAction = undefined;
             await this.runWorkIteration(taskStep);
             const nextAction = getNextIterationAction();
             this.logLine("STEP-LOOP", `nextAction = ${nextAction}`);
@@ -236,6 +255,8 @@ export class MainAgent {
                 return "complete";
             if (nextAction === "request-feedback")
                 return "request-feedback";
+            if (nextAction === "redo-step")
+                continue;
         }
         return "continue";
     }
@@ -307,14 +328,189 @@ export class MainAgent {
         }
         try {
             this.ui.update(`Running step: ${step.action}`);
-            await mod.run({ query: step.description ?? input.query, content: input.data ?? input.content, context: this.context });
-            return { query: step.description ?? input.query, data: {} };
+            const output = await mod.run({ query: step.description ?? input.query, content: input.data ?? input.content, context: this.context });
+            const errors = Array.isArray(output.data?.errors)
+                ? output.data.errors.filter((e) => typeof e === "string" && e.trim().length > 0)
+                : [];
+            if (errors.length > 0) {
+                const detail = errors.slice(0, 2).join(" | ");
+                this.logLine("EXECUTE", step.action, stop(), `completed with errors: ${detail}`);
+                console.error(`[${step.action}] ${errors.join(" | ")}`);
+            }
+            return output;
         }
         catch (err) {
             this.logLine("EXECUTE", step.action, stop(), "failed");
             throw err;
         }
     }
+    /* ───────────── finalize ───────────── */
+    async runFinalize() {
+        await finalAnswerModule.run({ query: this.query, context: this.context });
+        persistTaskData(this.context, this.taskId, getDbForRepo(), this.logLine.bind(this));
+        this.logLine("TASK", "Finalize complete", undefined, undefined, { highlight: false });
+    }
+    /* ───────────── extracted from runInitialRetrieval ───────────── */
+    resolveInitialRetrievalQueries() {
+        const rawUserQuery = this.context.initContext?.userQuery ?? this.query;
+        const retrievalQuery = this.context.analysis?.intent?.normalizedQuery?.trim() || rawUserQuery;
+        return { rawUserQuery, retrievalQuery };
+    }
+    async fetchInitialRetrievalResults(retrievalQuery) {
+        return semanticSearchFiles(retrievalQuery, RELATED_FILES_LIMIT, this.context.analysis?.intent ?? {});
+    }
+    mapSearchResultToTopFile(result) {
+        return {
+            id: result.id,
+            path: result.path,
+            summary: result.summary ?? undefined,
+            bm25Score: result.bm25Score,
+        };
+    }
+    mapSearchResultToRelatedFile(result) {
+        return {
+            id: result.id,
+            path: result.path,
+            summary: result.summary ?? undefined,
+            bm25Score: result.bm25Score,
+        };
+    }
+    buildInitialRetrievalPromptArgs(results, retrievalQuery) {
+        const topFiles = results
+            .slice(0, NUM_TOPFILES)
+            .map(result => this.mapSearchResultToTopFile(result));
+        const relatedFiles = results
+            .slice(NUM_TOPFILES)
+            .map(result => this.mapSearchResultToRelatedFile(result));
+        const queryExpansionTerms = results.find(result => Array.isArray(result.queryExpansionTerms))?.queryExpansionTerms;
+        return {
+            topFiles,
+            relatedFiles,
+            query: retrievalQuery,
+            queryExpansionTerms,
+        };
+    }
+    mergeSeededInitialContext(rawUserQuery, seededContext) {
+        const existingInit = this.context.initContext ?? { userQuery: rawUserQuery };
+        const seededInit = seededContext.initContext;
+        const mergedRelatedFiles = Array.from(new Set([
+            ...(existingInit.relatedFiles ?? []),
+            ...(seededInit?.relatedFiles ?? []),
+        ]));
+        const mergedScores = {
+            ...(existingInit.relatedFileScores ?? {}),
+            ...(seededInit?.relatedFileScores ?? {}),
+        };
+        const mergedQueryExpansionTerms = Array.from(new Set([
+            ...(existingInit.queryExpansionTerms ?? []),
+            ...(seededInit?.queryExpansionTerms ?? []),
+        ]));
+        this.context.initContext = {
+            ...existingInit,
+            ...(seededInit ?? {}),
+            userQuery: rawUserQuery,
+            relatedFiles: mergedRelatedFiles,
+            relatedFileScores: mergedScores,
+            queryExpansionTerms: mergedQueryExpansionTerms,
+            folderCapsules: (seededInit?.folderCapsules?.length
+                ? seededInit.folderCapsules
+                : existingInit.folderCapsules) ?? [],
+        };
+        return mergedRelatedFiles.length;
+    }
+    applyDeterministicPreGroundingPrefilter(retrievalQuery) {
+        const init = this.context.initContext;
+        if (!init?.relatedFiles?.length)
+            return { before: 0, after: 0 };
+        const before = init.relatedFiles.length;
+        const scored = scoreCandidateFiles(init.relatedFiles, init.relatedFileScores ?? {}, retrievalQuery);
+        if (scored.length === 0)
+            return { before, after: before };
+        const scope = this.context.analysis?.scopeType ?? "repo-wide";
+        const baseKeepCount = scope === "single-file" ? 6 : 10;
+        const keepCount = Math.min(Math.max(3, baseKeepCount), scored.length);
+        const selected = new Set(scored
+            .slice(0, keepCount)
+            .map(item => item.filePath));
+        for (const item of scored) {
+            if (item.reasons.includes("exact-filename") || item.reasons.includes("path-anchor")) {
+                selected.add(item.filePath);
+            }
+        }
+        init.relatedFiles = scored
+            .filter(item => selected.has(item.filePath))
+            .map(item => item.filePath);
+        init.relatedFileScores = Object.fromEntries(Object.entries(init.relatedFileScores ?? {})
+            .filter(([filePath]) => selected.has(filePath)));
+        logInputOutput("deterministicPreGroundingPrefilter", "output", {
+            retrievalQuery,
+            scope,
+            before,
+            after: init.relatedFiles.length,
+            keepCount,
+            files: scored.map(item => ({
+                file: item.filePath,
+                score: Number(item.score.toFixed(2)),
+                bm25Raw: item.bm25Raw,
+                kept: selected.has(item.filePath),
+                reasons: item.reasons,
+            })),
+        });
+        return { before, after: init.relatedFiles.length };
+    }
+    /* ───────────── extracted from runWorkLoop ───────────── */
+    isWorkLoopReady() {
+        const readinessDecision = this.context.analysis?.readiness?.decision;
+        const readinessConfidence = this.context.analysis?.readiness?.confidence ?? 0;
+        if (readinessDecision === "ready")
+            return true;
+        this.context.task.status = "deferred";
+        this.context.task.reason = `Readiness not achieved (decision=${readinessDecision}, confidence=${readinessConfidence})`;
+        this.persistTaskDataForRun();
+        this.logLine("TASK", "Cannot start work loop — agent needs more evidence to safely proceed", undefined, `Readiness: ${readinessDecision}, Confidence: ${readinessConfidence}`, { highlight: true });
+        return false;
+    }
+    ensureTaskForWorkLoop() {
+        if (!this.context.task) {
+            throw new Error("runWorkLoop: missing task");
+        }
+    }
+    async resolveNextTaskAction() {
+        await reasonNextTaskStep.run(this.context);
+        const nextAction = this.context.analysis?.iterationReasoning?.nextAction;
+        if (nextAction === "request-feedback" || nextAction === "complete")
+            return nextAction;
+        return "continue";
+    }
+    persistTaskDataForRun() {
+        persistTaskData(this.context, this.taskId, getDbForRepo(), this.logLine.bind(this));
+    }
+    persistTaskStatus(status) {
+        this.context.task.status = status;
+        this.persistTaskDataForRun();
+    }
+    startTaskStep(taskStep, stepCount) {
+        this.context.task.currentStep = taskStep;
+        taskStep.taskId = this.taskId;
+        taskStep.stepIndex = stepCount;
+        taskStep.status = "pending";
+        persistTaskStepInsert(taskStep, getDbForRepo());
+        this.logLine("NEW STEP", `Processing taskStep ${stepCount}`, undefined, taskStep.filePath, { highlight: true });
+        taskStep.startTime = Date.now();
+        persistTaskStepStart(taskStep, getDbForRepo());
+    }
+    finishTaskStep(taskStep, stepCount, stepAction) {
+        taskStep.endTime = Date.now();
+        if (stepAction === "complete") {
+            taskStep.status = "completed";
+            persistTaskStepCompletion(taskStep, getDbForRepo());
+            this.logLine("STEP-DONE", `Completed taskStep ${stepCount}`, undefined, taskStep.filePath, { highlight: false });
+            return;
+        }
+        taskStep.status = "pending";
+        persistTaskStepCompletion(taskStep, getDbForRepo());
+        this.logLine("STEP", `Pending taskStep ${stepCount}`, undefined, taskStep.filePath);
+    }
     /* ───────────── execution gates ───────────── */
     /**
      * Determines whether a phase can be executed based on execution mode and constraints.
@@ -403,6 +599,66 @@ export class MainAgent {
         });
     }
 }
+function scoreCandidateFiles(filePaths, relatedFileScores, retrievalQuery) {
+    const explicitRefs = extractFileReferences(retrievalQuery, { lowercase: true });
+    const explicitBasenames = new Set(explicitRefs.map(ref => path.basename(ref)));
+    const symbolAnchors = extractSymbolAnchors(retrievalQuery);
+    const scored = filePaths.map(filePath => {
+        const fileLower = filePath.toLowerCase();
+        const basename = path.basename(filePath).toLowerCase();
+        let score = 0;
+        const reasons = [];
+        if (explicitBasenames.has(basename)) {
+            score += 100;
+            reasons.push("exact-filename");
+        }
+        if (explicitRefs.some(ref => fileLower.includes(ref))) {
+            score += 60;
+            reasons.push("path-anchor");
+        }
+        for (const anchor of symbolAnchors) {
+            if (basename.includes(anchor)) {
+                score += 40;
+                reasons.push(`symbol:${anchor}:filename`);
+            }
+            else if (fileLower.includes(anchor)) {
+                score += 20;
+                reasons.push(`symbol:${anchor}:path`);
+            }
+        }
+        const bm25Raw = relatedFileScores[filePath];
+        if (typeof bm25Raw === "number" && Number.isFinite(bm25Raw)) {
+            const prior = Math.max(0, Math.min(20, -bm25Raw));
+            score += prior;
+            reasons.push(`bm25-prior:${prior.toFixed(2)}`);
+        }
+        return { filePath, score, bm25Raw, reasons };
+    });
+    return scored.sort((a, b) => {
+        if (b.score !== a.score)
+            return b.score - a.score;
+        const aBm25 = typeof a.bm25Raw === "number" ? a.bm25Raw : Number.POSITIVE_INFINITY;
+        const bBm25 = typeof b.bm25Raw === "number" ? b.bm25Raw : Number.POSITIVE_INFINITY;
+        return aBm25 - bBm25;
+    });
+}
+function extractSymbolAnchors(query) {
+    const matches = query.match(/[A-Za-z_][A-Za-z0-9_]{2,}/g) ?? [];
+    const out = new Set();
+    for (const token of matches) {
+        const lowered = token.toLowerCase();
+        const looksLikeSymbol = /[A-Z]/.test(token) ||
+            token.includes("_") ||
+            token.endsWith("Step") ||
+            token.endsWith("Module");
+        if (!looksLikeSymbol)
+            continue;
+        if (PREFILTER_STOP_WORDS.has(lowered))
+            continue;
+        out.add(lowered);
+    }
+    return Array.from(out);
+}
 // All helper functions (persistTaskData, bootTaskForRepo, persistTaskStep*) remain unchanged
 /* ───────────── FOLDER CAPSULES SUMMARY HELPER ───────────── */
 /**