scai 0.1.117 → 0.1.119

package/dist/pipeline/modules/commitSuggesterModule.js

@@ -1,10 +1,12 @@
-import { generate } from '../../lib/generate.js';
+// File: src/pipeline/modules/commitSuggesterModule.ts
 import { Config } from '../../config.js';
+import { generate } from '../../lib/generate.js';
 export const commitSuggesterModule = {
     name: 'commitSuggester',
     description: 'Suggests conventional commit messages from Git diff',
-    async run({ content }) {
+    run: async (input) => {
         const model = Config.getModel();
+        const diffContent = typeof input.content === 'string' ? input.content : '';
         const prompt = `
 Suggest ALWAYS 3 concise, conventional Git commit messages based on the input code diff.
 
@@ -20,17 +22,37 @@ Format your response exactly as:
 3. <type>: <message>
 
 Here is the diff:
-${content}
-`.trim();
-        const response = await generate({ content: prompt });
-        const lines = response.content
-            .split('\n')
-            .map(line => line.trim())
-            .filter(line => /^\d+\.\s+/.test(line));
-        const suggestions = lines.map(line => line.replace(/^\d+\.\s+/, '').replace(/^"(.*)"$/, '$1').trim());
-        return {
-            content,
-            suggestions
-        };
+${diffContent}
+    `.trim();
+        try {
+            // === INTERACTION #1: generate.ts (via ModuleIO) ===
+            const genInput = {
+                query: 'Suggest 3 conventional commit messages',
+                content: prompt
+            };
+            const genOutput = await generate(genInput);
+            const rawText = typeof genOutput.data === 'string' ? genOutput.data : '';
+            // === Parse lines ===
+            const lines = rawText
+                .split('\n')
+                .map(line => line.trim())
+                .filter(line => /^[1-3][.)]\s+/.test(line));
+            const suggestions = lines.map(line => line.replace(/^[1-3][.)]\s+/, '').replace(/^"(.*)"$/, '$1').trim());
+            return {
+                query: input.query,
+                content: diffContent,
+                data: genOutput.data,
+                suggestions
+            };
+        }
+        catch (err) {
+            console.warn('⚠️ commitSuggesterModule failed:', err);
+            return {
+                query: input.query,
+                content: diffContent,
+                data: {},
+                suggestions: []
+            };
+        }
     }
 };

package/dist/pipeline/modules/contextReviewModule.js

@@ -0,0 +1,52 @@
+import { generate } from "../../lib/generate.js";
+import { logInputOutput } from "../../utils/promptLogHelper.js";
+export const contextReviewModule = {
+    name: "contextReview",
+    description: "Analyzes summaries and global context to decide if reading full files is needed before answering.",
+    groups: ["analysis"],
+    run: async (input) => {
+        const query = input.query;
+        // Convert content to a text blob for the model
+        const contextText = typeof input.content === "string"
+            ? input.content
+            : JSON.stringify(input.content ?? "", null, 2);
+        const promptText = `
+You are an AI reasoning agent reviewing file summaries and contextual notes for a software repository.
+
+User query:
+${query}
+
+Context:
+${contextText}
+
+Your task:
+1. Summarize what this context reveals about the codebase.
+2. Identify what is still missing or unclear.
+3. Decide if reading full file contents is required ("needFullRead") or if current summaries are enough ("enoughForAnswer").
+Respond concisely but analytically.
+`.trim();
+        // ✅ Call generate() with ModuleIO-compliant input
+        const aiResponse = await generate({
+            query,
+            content: promptText,
+        });
+        // ✅ Model output is in aiResponse.data
+        const modelText = typeof aiResponse.data === "string"
+            ? aiResponse.data.trim()
+            : JSON.stringify(aiResponse.data);
+        const lower = modelText.toLowerCase();
+        const decision = lower.includes("needfullread") || lower.includes("read full")
+            ? "needFullRead"
+            : "enoughForAnswer";
+        // ✅ Return fully ModuleIO-compliant output
+        const output = {
+            query,
+            data: {
+                decision,
+                understanding: modelText,
+            },
+        };
+        logInputOutput("contextReview", "output", output.data);
+        return output;
+    },
+};

package/dist/pipeline/modules/fileReaderModule.js

@@ -0,0 +1,72 @@
+import * as fs from "fs/promises";
+import * as path from "path";
+import { logInputOutput } from "../../utils/promptLogHelper.js";
+import { getDbForRepo } from "../../db/client.js";
+import { updateFileWithSummary, upsertFileFtsTemplate, } from "../../db/sqlTemplates.js";
+export const fileReaderModule = {
+    name: "fileReader",
+    description: "Reads files from disk only if they are not available in the DB, updates DB and FTS.",
+    groups: ["analysis"],
+    run: async (input) => {
+        let filePaths = [];
+        logInputOutput('filereader', 'input', input.content);
+        if (Array.isArray(input.content)) {
+            filePaths = input.content;
+        }
+        else if (typeof input.content === "object" && input.content !== null) {
+            const obj = input.content;
+            if (Array.isArray(obj.filePaths))
+                filePaths = obj.filePaths;
+        }
+        if (filePaths.length === 0) {
+            const emptyOutput = {
+                query: input.query,
+                data: { files: [] },
+            };
+            logInputOutput("fileReader", "output", emptyOutput.data);
+            return emptyOutput;
+        }
+        const db = getDbForRepo();
+        const files = [];
+        for (const filePath of filePaths) {
+            const fullPath = path.resolve(filePath);
+            // 1️⃣ Check DB for existing summary/content
+            const row = db
+                .prepare("SELECT path, content_text, summary FROM files WHERE path = ?")
+                .get(fullPath);
+            if (row?.content_text) {
+                files.push({ filePath: fullPath, content: row.content_text });
+                continue; // already in DB, no need to read disk
+            }
+            // 2️⃣ Read file from disk
+            let content = "";
+            try {
+                content = await fs.readFile(fullPath, "utf-8");
+            }
+            catch (err) {
+                content = `ERROR: ${err.message}`;
+            }
+            files.push({ filePath: fullPath, content });
+            // 3️⃣ Update DB and FTS if content successfully read
+            if (!content.startsWith("ERROR")) {
+                db.prepare(updateFileWithSummary).run({
+                    path: fullPath,
+                    summary: null, // optionally leave null; summary can be generated later
+                    contentText: content,
+                });
+                db.prepare(upsertFileFtsTemplate).run({
+                    path: fullPath,
+                    filename: path.basename(fullPath),
+                    summary: null,
+                    contentText: content,
+                });
+            }
+        }
+        const output = {
+            query: input.query,
+            data: { files },
+        };
+        logInputOutput("fileReader", "output", output.data);
+        return output;
+    },
+};

package/dist/pipeline/modules/fileSearchModule.js

@@ -0,0 +1,136 @@
+// File: src/modules/fileSearchModule.ts
+import { execSync } from "child_process";
+import path from "path";
+import fs from "fs";
+import { searchFiles } from "../../db/fileIndex.js";
+import { Config } from "../../config.js";
+import { getDbForRepo } from "../../db/client.js";
+import { IGNORED_EXTENSIONS } from "../../fileRules/ignoredExtensions.js";
+import { logInputOutput } from "../../utils/promptLogHelper.js";
+import { generate } from "../../lib/generate.js";
+import { cleanupModule } from "./cleanupModule.js";
+async function fetchSummariesForPaths(paths) {
+    if (paths.length === 0)
+        return {};
+    const db = getDbForRepo();
+    const placeholders = paths.map(() => "?").join(", ");
+    const rows = db.prepare(`SELECT path, summary FROM summaries 
+         WHERE path IN (${placeholders}) AND summary IS NOT NULL;`).all(...paths);
+    const map = {};
+    for (const row of rows)
+        map[row.path] = row.summary;
+    return map;
+}
+export const fileSearchModule = {
+    name: "fileSearch",
+    description: "Searches the indexed repo for files matching a generated query.",
+    groups: ["analysis"],
+    run: async (input) => {
+        const query = input.query ?? "";
+        const ctx = input.context;
+        if (!ctx) {
+            throw new Error("[fileSearch] StructuredContext is required.");
+        }
+        // -------------------------------------------------
+        // STEP 0: Repo root resolution
+        // -------------------------------------------------
+        let repoRoot = Config.getIndexDir() ?? process.cwd();
+        repoRoot = path.resolve(repoRoot);
+        if (!fs.existsSync(repoRoot)) {
+            return { query, data: { files: [] } };
+        }
+        // -------------------------------------------------
+        // STEP 1: LLM query generation
+        // -------------------------------------------------
+        const llmPrompt = `
+You are a search query generator. Given the following user input,
+produce a concise, meaningful query suitable for searching code files.
+
+Input:
+${typeof query === "string" ? query : JSON.stringify(query, null, 2)}
+
+Return only the improved query text.
+        `.trim();
+        const llmResponse = await generate({
+            content: llmPrompt,
+            query
+        });
+        const cleaned = await cleanupModule.run({
+            query,
+            content: llmResponse.data
+        });
+        const searchQuery = typeof cleaned.data === "string" ? cleaned.data.trim() : "";
+        if (!searchQuery) {
+            return { query, data: { files: [] } };
+        }
+        // -------------------------------------------------
+        // STEP 2: Semantic search
+        // -------------------------------------------------
+        let results = [];
+        try {
+            results = await searchFiles(searchQuery, 5);
+        }
+        catch (err) {
+            console.warn("❌ [fileSearch] Semantic search failed:", err);
+        }
+        // -------------------------------------------------
+        // STEP 3: Grep fallback
+        // -------------------------------------------------
+        if (results.length === 0) {
+            try {
+                const exclude = IGNORED_EXTENSIONS.map(ext => `--exclude=*${ext}`).join(" ");
+                const stdout = execSync(`grep -ril ${exclude} "${query}" "${repoRoot}"`, { encoding: "utf8" });
+                results = stdout
+                    .split("\n")
+                    .filter(Boolean)
+                    .map(f => ({ path: f }));
+            }
+            catch (err) {
+                if (err?.status !== 1) {
+                    console.warn("⚠️ [fileSearch] Grep fallback failed:", err);
+                }
+            }
+        }
+        // -------------------------------------------------
+        // STEP 4: DB Summary enrichment
+        // -------------------------------------------------
+        const missingPaths = results.filter(f => !f.summary).map(f => f.path);
+        if (missingPaths.length > 0) {
+            try {
+                const map = await fetchSummariesForPaths(missingPaths);
+                results.forEach(f => {
+                    if (!f.summary && map[f.path]) {
+                        f.summary = map[f.path];
+                    }
+                });
+            }
+            catch (err) {
+                console.warn("⚠️ [fileSearch] Summary enrichment failed:", err);
+            }
+        }
+        // -------------------------------------------------
+        // STEP 5: Persist discovered files to initContext.relatedFiles
+        // -------------------------------------------------
+        if (!ctx.initContext) {
+            ctx.initContext = {
+                userQuery: query
+            };
+        }
+        const existing = new Set(ctx.initContext.relatedFiles ?? []);
+        const discoveredPaths = results.map(r => r.path);
+        // Append + deduplicate
+        ctx.initContext.relatedFiles = [
+            ...(ctx.initContext.relatedFiles ?? []),
+            ...discoveredPaths.filter(p => !existing.has(p))
+        ];
+        // -------------------------------------------------
+        // STEP 6: Log and return ModuleIO output
+        // -------------------------------------------------
+        const output = {
+            query,
+            data: { files: results }
+        };
+        logInputOutput("fileSearch", "output", output);
+        return output;
+    }
+};

package/dist/pipeline/modules/finalAnswerModule.js

@@ -0,0 +1,53 @@
+import { logInputOutput } from "../../utils/promptLogHelper.js";
+import { generate } from "../../lib/generate.js";
+export const finalAnswerModule = {
+    name: "finalAnswer",
+    description: "Generates a final answer using structured context and relevant working files.",
+    groups: ["finalize"],
+    run: async (input) => {
+        const query = input.query;
+        const context = input.context;
+        if (!context)
+            throw new Error("[finalAnswerModule] No context provided");
+        // Only consider authoritative target files
+        const targetFiles = context.plan?.targetFiles ?? [];
+        const workingFiles = (context.workingFiles ?? []).filter(f => targetFiles.includes(f.path));
+        const filesSummary = workingFiles.map(f => ({
+            path: f.path,
+            hasCode: !!f.code,
+            summary: f.summary ?? "",
+            codeSnippet: f.code ? f.code.slice(0, 500) : undefined, // optional snippet, avoid huge content
+        }));
+        const analysisSummary = {
+            intent: context.analysis?.intent,
+            fileAnalysis: context.analysis?.fileAnalysis,
+            combinedAnalysis: context.analysis?.combinedAnalysis,
+        };
+        const promptText = `
+You are an AI assistant answering a developer query.
+
+Query:
+${query}
+
+Analysis summary:
+${JSON.stringify(analysisSummary, null, 2)}
+
+Target files and content:
+${JSON.stringify(filesSummary, null, 2)}
+
+Instructions:
+- Provide a clear, concise answer based only on the available files and analysis.
+- Highlight risks or important notes from fileAnalysis or combinedAnalysis.
+- Do not ask for additional files; assume the provided code is authoritative.
+`;
+        const aiResponse = await generate({ query, content: promptText });
+        const output = {
+            query,
+            content: '',
+            data: aiResponse.data,
+        };
+        logInputOutput("finalAnswer", "output", output.data);
+        console.log('\n--> Answer: ', output.data);
+        return output;
+    },
+};

package/dist/pipeline/modules/gatherInfoModule.js

@@ -0,0 +1,176 @@
+// src/pipeline/modules/gatherInfoModule.ts
+import { getDbForRepo } from "../../db/client.js";
+import chalk from "chalk";
+import { sanitizeQueryForFts } from "../../utils/sanitizeQuery.js";
+import { logInputOutput } from "../../utils/promptLogHelper.js"; // ✅ import logger
+/** Escape % and _ for LIKE queries */
+function sanitizeForLike(input) {
+    return input.replace(/[%_]/g, "\\$&");
+}
+/** ✅ Strip embeddings before logging */
+function stripEmbeddings(output) {
+    if (output?.data?.files) {
+        output.data.files = output.data.files.map((f) => {
+            const { embedding, ...rest } = f;
+            return rest;
+        });
+    }
+    return output;
+}
+export const gatherInfoModule = {
+    name: "gatherInfo",
+    description: "Collects relevant code summaries, functions, classes, and related graph info for the current query.",
+    run: async function (input) {
+        const db = getDbForRepo();
+        const query = input.metadata?.query?.trim() ?? "";
+        const maxFiles = input.metadata?.maxFiles ?? 10;
+        if (!query) {
+            const emptyOutput = {
+                content: "⚠️ No query provided to gatherInfoModule.",
+                data: { files: [], functions: [], classes: [], summaries: [], tags: [] },
+            };
+            logInputOutput("gatherInfo", "output", stripEmbeddings(emptyOutput)); // ✅
+            return emptyOutput;
+        }
+        const sanitizedFts = sanitizeQueryForFts(query);
+        const likeQuery = `%${sanitizeForLike(query)}%`;
+        // 🩹 Handle legacy DBs that might not have `functions_extracted`
+        let files = [];
+        try {
+            files = db
+                .prepare(`
+          SELECT f.id, f.path, f.type, f.summary, f.embedding,
+                 f.last_modified, f.indexed_at,
+                 COALESCE(f.functions_extracted, 0) AS functions_extracted,
+                 f.functions_extracted_at, f.processing_status
+          FROM files f
+          JOIN files_fts fts ON fts.rowid = f.id
+          WHERE files_fts MATCH '${sanitizedFts}'
+          ORDER BY f.path ASC
+          LIMIT ?;
+        `)
+                .all(maxFiles);
+        }
+        catch (err) {
+            console.warn(chalk.yellow("⚠️ 'functions_extracted' column missing in files table, running fallback query..."));
+            files = db
+                .prepare(`
+          SELECT f.id, f.path, f.type, f.summary, f.embedding,
+                 f.last_modified, f.indexed_at, f.processing_status
+          FROM files f
+          JOIN files_fts fts ON fts.rowid = f.id
+          WHERE files_fts MATCH '${sanitizedFts}'
+          ORDER BY f.path ASC
+          LIMIT ?;
+        `)
+                .all(maxFiles);
+        }
+        const functions = db
+            .prepare(`
+        SELECT fn.id, fn.name, fn.start_line, fn.end_line,
+               substr(fn.content, 1, 400) AS content,
+               f.path AS file_path
+        FROM functions fn
+        JOIN files f ON f.id = fn.file_id
+        WHERE fn.name LIKE ? ESCAPE '\\'
+           OR fn.content LIKE ? ESCAPE '\\'
+        ORDER BY f.path ASC
+        LIMIT ?;
+      `)
+            .all(likeQuery, likeQuery, maxFiles);
+        const classes = db
+            .prepare(`
+        SELECT c.name, f.path AS file_path, substr(c.content, 1, 400) AS snippet
+        FROM graph_classes c
+        JOIN files f ON f.id = c.file_id
+        WHERE c.name LIKE ? ESCAPE '\\'
+           OR c.content LIKE ? ESCAPE '\\'
+        ORDER BY f.path ASC
+        LIMIT ?;
+      `)
+            .all(likeQuery, likeQuery, maxFiles);
+        const tagRows = db
+            .prepare(`
+        SELECT DISTINCT gtm.name
+        FROM graph_tags_master gtm
+        WHERE gtm.name LIKE ? ESCAPE '\\'
+        ORDER BY gtm.name ASC;
+      `)
+            .all(likeQuery);
+        const tags = tagRows.map((t) => t.name);
+        const summaries = db
+            .prepare(`
+        SELECT path, type, summary
+        FROM summaries
+        WHERE summary LIKE ? ESCAPE '\\'
+        ORDER BY CASE WHEN type='project' THEN 0 ELSE 1 END, path ASC
+        LIMIT 5;
+      `)
+            .all(likeQuery);
+        // --- Gather common project config files
+        let configFiles = [];
+        try {
+            configFiles = db
+                .prepare(`
+          SELECT f.id, f.path, f.type, f.summary, f.embedding,
+                 f.last_modified, f.indexed_at,
+                 COALESCE(f.functions_extracted, 0) AS functions_extracted,
+                 f.functions_extracted_at, f.processing_status
+          FROM files f
+          WHERE path LIKE '%package.json%'
+             OR path LIKE '%tsconfig.json%'
+             OR path LIKE '%.eslintrc%'
+             OR path LIKE '%.env%'
+             OR path LIKE '%README.md%'
+          ORDER BY path ASC;
+        `)
+                .all();
+        }
+        catch (err) {
+            configFiles = db
+                .prepare(`
+          SELECT f.id, f.path, f.type, f.summary, f.embedding,
+                 f.last_modified, f.indexed_at, f.processing_status
+          FROM files f
+          WHERE path LIKE '%package.json%'
+             OR path LIKE '%tsconfig.json%'
+             OR path LIKE '%.eslintrc%'
+             OR path LIKE '%.env%'
+             OR path LIKE '%README.md%'
+          ORDER BY path ASC;
+        `)
+                .all();
+        }
+        // Merge & deduplicate
+        const uniqueFiles = [
+            ...files,
+            ...configFiles.filter((c) => !files.find((f) => f.path === c.path)),
+        ];
+        if (!uniqueFiles.length &&
+            !functions.length &&
+            !classes.length &&
+            !summaries.length &&
+            !tags.length) {
+            const emptyOutput = {
+                content: `⚠️ No relevant information found for query: "${query}"`,
+                data: { files: [], functions: [], classes: [], summaries: [], tags: [] },
+            };
+            logInputOutput("gatherInfo", "output", stripEmbeddings(emptyOutput)); // ✅
+            return emptyOutput;
+        }
+        const output = {
+            content: `Gathered ${uniqueFiles.length} files, ${functions.length} functions, ${classes.length} classes, and ${summaries.length} summaries.`,
+            data: {
+                query, // ✅ ensure query passes forward
+                files: uniqueFiles,
+                functions,
+                classes,
+                summaries,
+                tags,
+            },
+        };
+        // ✅ Log sanitized module output (no embeddings)
+        logInputOutput("gatherInfo", "output", stripEmbeddings(output));
+        return output;
+    },
+};