scai 0.1.116 → 0.1.118

package/dist/db/fileIndex.js

@@ -1,78 +1,183 @@
-import fs from 'fs';
+// indexCmd.ts
+import fg from 'fast-glob';
 import path from 'path';
-import { generateEmbedding } from '../lib/generateEmbedding.js';
+import lockfile from 'proper-lockfile';
+import { initSchema } from '../db/schema.js';
+import { getDbForRepo, getDbPathForRepo } from '../db/client.js';
+import { upsertFileTemplate } from '../db/sqlTemplates.js';
+import { detectFileType } from '../fileRules/detectFileType.js';
+import { classifyFile } from '../fileRules/classifyFile.js';
+import { IGNORED_FOLDER_GLOBS } from '../fileRules/ignoredPaths.js';
+import { Config } from '../config.js';
+import { log } from '../utils/log.js';
+import { startDaemon } from '../commands/DaemonCmd.js';
 import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
-import * as sqlTemplates from './sqlTemplates.js';
-import { CANDIDATE_LIMIT } from '../constants.js';
-import { getDbForRepo } from './client.js';
-import { scoreFiles } from '../fileRules/scoreFiles.js';
-import chalk from 'chalk';
-export function indexFile(filePath, summary, type) {
-    const stats = fs.statSync(filePath);
-    const lastModified = stats.mtime.toISOString();
-    const indexedAt = new Date().toISOString();
-    const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
-    const fileName = path.basename(normalizedPath);
-    const db = getDbForRepo();
-    db.prepare(sqlTemplates.upsertFileTemplate).run({
-        path: normalizedPath,
-        filename: fileName,
-        summary,
-        type,
-        lastModified,
-        indexedAt,
-        embedding: null
-    });
-    db.prepare(sqlTemplates.upsertFileFtsTemplate).run({
-        path: normalizedPath,
-        filename: fileName,
-        summary,
+import * as sqlTemplates from '../db/sqlTemplates.js';
+import { RELATED_FILES_LIMIT } from '../constants.js';
+import { generate } from '../lib/generate.js';
+import { cleanupModule } from '../pipeline/modules/cleanupModule.js';
+async function lockDb() {
+    try {
+        return await lockfile.lock(getDbPathForRepo());
+    }
+    catch (err) {
+        log('❌ Failed to acquire DB lock: ' + err);
+        throw err;
+    }
+}
+export async function runIndexCommand() {
+    try {
+        initSchema();
+    }
+    catch (err) {
+        console.error('❌ Failed to initialize schema:', err);
+        process.exit(1);
+    }
+    const indexDir = Config.getIndexDir() || process.cwd();
+    Config.setIndexDir(indexDir);
+    log(`📂 Scanning files in: ${indexDir}`);
+    const files = await fg('**/*.*', {
+        cwd: indexDir,
+        ignore: IGNORED_FOLDER_GLOBS,
+        absolute: true,
     });
-    console.log(`📄 Indexed: ${normalizedPath}`);
+    const db = getDbForRepo();
+    const release = await lockDb();
+    const countByExt = {};
+    let count = 0;
+    try {
+        for (const file of files) {
+            const classification = classifyFile(file);
+            if (classification !== 'valid') {
+                log(`⏭️ Skipping (${classification}): ${file}`);
+                continue;
+            }
+            try {
+                const type = detectFileType(file);
+                const normalizedPath = path.normalize(file).replace(/\\/g, '/');
+                const filename = path.basename(normalizedPath);
+                // --------------------------------------------------
+                // Enqueue file for daemon processing
+                // --------------------------------------------------
+                db.prepare(upsertFileTemplate).run({
+                    path: normalizedPath,
+                    filename,
+                    summary: null,
+                    type,
+                    lastModified: null,
+                    indexedAt: null,
+                });
+                const ext = path.extname(file);
+                countByExt[ext] = (countByExt[ext] || 0) + 1;
+                count++;
+            }
+            catch (err) {
+                log(`⚠️ Skipped in indexCmd ${file}: ${err instanceof Error ? err.message : err}`);
+            }
+        }
+    }
+    finally {
+        await release();
+    }
+    log('📊 Discovered files by extension:', JSON.stringify(countByExt, null, 2));
+    log(`✅ Done. Enqueued ${count} files for indexing.`);
+    // Kick the daemon — it now owns all processing
+    startDaemon();
 }
+// --------------------------------------------------
+// QUERY API (read-only, used by CLI / search)
+// --------------------------------------------------
 export function queryFiles(safeQuery, limit = 10) {
-    console.log(`Executing search query: ${safeQuery}`);
     const db = getDbForRepo();
-    return db.prepare(sqlTemplates.queryFilesTemplate).all(safeQuery, limit);
+    return db
+        .prepare(sqlTemplates.queryFilesTemplate)
+        .all(safeQuery, limit);
 }
+// --------------------------------------------------
+// searchFiles with semantic fallback
+// --------------------------------------------------
 export async function searchFiles(query, topK = 5) {
-    console.log(chalk.yellow(`🧠 Searching for query: "${query}"`));
-    const embedding = await generateEmbedding(query);
-    if (!embedding) {
-        console.log('⚠️ Failed to generate embedding for query');
+    const db = getDbForRepo();
+    // -----------------------------
+    // Primary FTS search
+    // -----------------------------
+    const safeQuery = sanitizeQueryForFts(query);
+    const primaryResults = db
+        .prepare(sqlTemplates.searchFilesTemplate)
+        .all(safeQuery, RELATED_FILES_LIMIT);
+    if (primaryResults.length > 0) {
+        return mapFtsResults(primaryResults, topK);
+    }
+    // -----------------------------
+    // Fallback: model-assisted expansion
+    // -----------------------------
+    const expandedTerms = await expandQueryWithModel(query);
+    if (expandedTerms.length === 0)
         return [];
+    log(`🔁 [searchFiles] Fallback used for "${query}". Expanded terms: ${expandedTerms.join(", ")}`);
+    const seen = new Map();
+    for (const term of expandedTerms) {
+        const safeTerm = sanitizeQueryForFts(term);
+        const rows = db
+            .prepare(sqlTemplates.searchFilesTemplate)
+            .all(safeTerm, RELATED_FILES_LIMIT);
+        for (const row of rows) {
+            if (!seen.has(row.id)) {
+                seen.set(row.id, row);
+            }
+        }
     }
-    const safeQuery = sanitizeQueryForFts(query);
-    console.log(`Executing search query in FTS5: ${safeQuery}`);
-    const db = getDbForRepo();
-    const ftsResults = db.prepare(sqlTemplates.searchFilesTemplate).all(safeQuery, CANDIDATE_LIMIT);
-    console.log(`FTS search returned ${ftsResults.length} results`);
-    if (ftsResults.length === 0)
+    if (seen.size === 0)
         return [];
-    const scored = scoreFiles(query, embedding, ftsResults);
-    return scored.slice(0, topK);
+    const merged = Array.from(seen.values()).sort((a, b) => a.bm25Score - b.bm25Score // lower = more relevant
+    );
+    return mapFtsResults(merged, topK);
 }
-export function getFunctionsForFiles(fileIds) {
-    if (!fileIds.length)
-        return {};
-    const placeholders = fileIds.map(() => '?').join(',');
-    const db = getDbForRepo();
-    const stmt = db.prepare(`
-    SELECT f.file_id, f.name, f.start_line, f.end_line, f.content
-    FROM functions f
-    WHERE f.file_id IN (${placeholders})
-  `);
-    const rows = stmt.all(...fileIds);
-    const grouped = {};
-    for (const row of rows) {
-        if (!grouped[row.file_id])
-            grouped[row.file_id] = [];
-        grouped[row.file_id].push({
-            name: row.name,
-            start_line: row.start_line,
-            end_line: row.end_line,
-            content: row.content,
+// --------------------------------------------------
+// Helpers
+// --------------------------------------------------
+function mapFtsResults(rows, topK) {
+    return rows.slice(0, topK).map(r => ({
+        id: r.id,
+        path: r.path,
+        filename: r.filename,
+        summary: r.summary,
+        type: r.type,
+        lastModified: r.lastModified,
+        bm25Score: r.bm25Score,
+    }));
+}
+async function expandQueryWithModel(query) {
+    const prompt = `
+You are assisting a code search system.
+
+Given a natural-language question about a codebase, return a JSON array
+of 3–8 concrete search terms that are likely to appear literally in source code.
+
+Rules:
+- Return ONLY a JSON array of strings
+- No explanations
+- Prefer filenames, function names, symbols, library names
+
+Question:
+"${query}"
+`.trim();
+    try {
+        const response = await generate({
+            content: prompt,
+            query: "",
+        });
+        const cleaned = await cleanupModule.run({
+            query,
+            content: response.data,
         });
+        const terms = Array.isArray(cleaned.data)
+            ? cleaned.data.filter((t) => typeof t === "string")
+            : [];
+        return terms;
+    }
+    catch (err) {
+        log(`⚠️ [searchFiles] Failed to expand query "${query}": ${String(err)}`);
+        return [];
     }
-    return grouped;
 }

package/dist/db/functionExtractors/extractFromJava.js

@@ -1,9 +1,213 @@
+import { parse } from 'java-parser';
+import path from 'path';
+import { log } from '../../utils/log.js';
 import { getDbForRepo } from '../client.js';
-import { markFileAsSkippedTemplate } from '../sqlTemplates.js';
-export async function extractFromJava(filePath, _content, fileId) {
-    console.warn(`⛔️ Java extraction not implemented: ${filePath}`);
-    // Mark the file as skipped with the relevant status update
+import { markFileAsSkippedTemplate, markFileAsExtractedTemplate, markFileAsFailedTemplate, insertFunctionTemplate, insertGraphClassTemplate, insertEdgeTemplate, insertGraphEntityTagTemplate, insertGraphTagTemplate, selectGraphTagIdTemplate, } from '../sqlTemplates.js';
+import { kgModule } from '../../pipeline/modules/kgModule.js';
+import { BUILTINS } from '../../fileRules/builtins.js';
+import { getUniqueId } from '../../utils/sharedUtils.js';
+export async function extractFromJava(filePath, content, fileId) {
     const db = getDbForRepo();
-    db.prepare(markFileAsSkippedTemplate).run({ id: fileId });
-    return false;
+    const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
+    try {
+        const cst = parse(content);
+        log(`🧩 First-level CST node types:`, cst.children ? Object.keys(cst.children) : []);
+        const functions = [];
+        const classes = [];
+        // --- Traverse CST to collect functions and classes ---
+        function traverse(node, parentClassName) {
+            if (!node)
+                return;
+            // Classes
+            if (node.node === 'normalClassDeclaration') {
+                const name = node.children?.Identifier?.[0]?.image ?? `${path.basename(filePath)}:<anon-class>`;
+                const superClass = node.children?.superclass?.[0]?.children?.Identifier?.[0]?.image ?? null;
+                const startLine = node.location?.startLine ?? 0;
+                const endLine = node.location?.endLine ?? 0;
+                const code = content.split('\n').slice(startLine - 1, endLine).join('\n');
+                const unique_id = getUniqueId(name, filePath, startLine, node.location?.startOffset ?? 0, code);
+                classes.push({ name, start_line: startLine, end_line: endLine, content: code, superClass, unique_id });
+                const classBody = node.children?.classBody?.[0]?.children;
+                if (classBody)
+                    classBody.forEach((child) => traverse(child, name));
+                return;
+            }
+            // Methods/Constructors
+            if (node.node === 'methodDeclaration' || node.node === 'constructorDeclaration') {
+                const name = node.children?.Identifier?.[0]?.image ?? '<anon>';
+                const startLine = node.location?.startLine ?? 0;
+                const endLine = node.location?.endLine ?? 0;
+                const code = content.split('\n').slice(startLine - 1, endLine).join('\n');
+                const unique_id = getUniqueId(name, filePath, startLine, node.location?.startOffset ?? 0, code);
+                functions.push({ name, start_line: startLine, end_line: endLine, content: code, unique_id });
+                return;
+            }
+            // Traverse children
+            if (node.children) {
+                Object.values(node.children).forEach((child) => {
+                    if (Array.isArray(child))
+                        child.forEach(c => traverse(c, parentClassName));
+                    else
+                        traverse(child, parentClassName);
+                });
+            }
+        }
+        traverse(cst);
+        if (functions.length === 0 && classes.length === 0) {
+            log(`⚠️ No functions/classes found in Java file: ${filePath}`);
+            try {
+                db.prepare(markFileAsSkippedTemplate).run({ id: fileId });
+            }
+            catch { }
+            return false;
+        }
+        log(`🔍 Found ${functions.length} methods and ${classes.length} classes in ${filePath}`);
+        // --- Knowledge Graph tagging ---
+        try {
+            const kgInput = { fileId, filepath: filePath, summary: undefined };
+            const kgResult = await kgModule.run(kgInput, content);
+            if (kgResult.entities?.length) {
+                const insertTagStmt = db.prepare(insertGraphTagTemplate);
+                const getTagIdStmt = db.prepare(selectGraphTagIdTemplate);
+                const insertEntityTagStmt = db.prepare(insertGraphEntityTagTemplate);
+                const persistTag = (tag) => {
+                    try {
+                        insertTagStmt.run({ name: tag });
+                        const tagRow = getTagIdStmt.get({ name: tag });
+                        return tagRow?.id;
+                    }
+                    catch {
+                        return undefined;
+                    }
+                };
+                const persistEntityTags = (entity) => {
+                    if (!entity.type || !Array.isArray(entity.tags) || !entity.tags.length)
+                        return;
+                    for (const tag of entity.tags) {
+                        if (!tag)
+                            continue;
+                        const tagId = persistTag(tag);
+                        if (!tagId)
+                            continue;
+                        const matchedUniqueId = functions.find(f => f.name === entity.name)?.unique_id ||
+                            classes.find(c => c.name === entity.name)?.unique_id ||
+                            `${entity.name}@${filePath}`;
+                        try {
+                            insertEntityTagStmt.run({ entity_type: entity.type, entity_unique_id: matchedUniqueId, tag_id: tagId });
+                        }
+                        catch { }
+                    }
+                };
+                kgResult.entities.forEach(persistEntityTags);
+            }
+        }
+        catch (kgErr) {
+            console.warn(`⚠️ KG tagging failed for ${filePath}:`, kgErr instanceof Error ? kgErr.message : kgErr);
+        }
+        const seenEdges = new Set();
+        const javaBuiltins = BUILTINS.java;
+        // --- Insert functions with call edges ---
+        for (const fn of functions) {
+            db.prepare(insertFunctionTemplate).run({
+                file_id: fileId,
+                name: fn.name ?? '<anon>',
+                start_line: fn.start_line ?? -1,
+                end_line: fn.end_line ?? -1,
+                content: fn.content ?? '',
+                embedding: null,
+                lang: 'java',
+                unique_id: fn.unique_id,
+            });
+            const containsEdgeKey = `file->${fn.unique_id}`;
+            if (!seenEdges.has(containsEdgeKey)) {
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'file',
+                    source_unique_id: normalizedPath,
+                    target_type: 'function',
+                    target_unique_id: fn.unique_id,
+                    relation: 'contains',
+                });
+                seenEdges.add(containsEdgeKey);
+            }
+            // --- Function call edges ---
+            const callRegex = /([A-Za-z_$][A-Za-z0-9_$]*)\s*\(/g;
+            const callSet = new Set();
+            let match;
+            while ((match = callRegex.exec(fn.content)) !== null) {
+                const callee = match[1];
+                if (!callee || javaBuiltins.has(callee) || callSet.has(callee))
+                    continue;
+                callSet.add(callee);
+                const targetUniqueId = `${callee}@${normalizedPath}`;
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'function',
+                    source_unique_id: fn.unique_id,
+                    target_type: 'function',
+                    target_unique_id: targetUniqueId,
+                    relation: 'calls',
+                });
+            }
+        }
+        // --- Insert classes with inheritance edges ---
+        for (const cls of classes) {
+            db.prepare(insertGraphClassTemplate).run({
+                file_id: fileId,
+                name: cls.name,
+                start_line: cls.start_line ?? -1,
+                end_line: cls.end_line ?? -1,
+                content: cls.content ?? '',
+                embedding: null,
+                lang: 'java',
+                unique_id: cls.unique_id,
+            });
+            const containsEdgeKey = `file->${cls.unique_id}`;
+            if (!seenEdges.has(containsEdgeKey)) {
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'file',
+                    source_unique_id: normalizedPath,
+                    target_type: 'class',
+                    target_unique_id: cls.unique_id,
+                    relation: 'contains',
+                });
+                seenEdges.add(containsEdgeKey);
+            }
+            if (cls.superClass) {
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'class',
+                    source_unique_id: cls.unique_id,
+                    target_type: 'class',
+                    target_unique_id: `unresolved:${cls.superClass}`,
+                    relation: 'inherits',
+                });
+            }
+        }
+        // --- Java imports as edges ---
+        const importRegex = /^\s*import\s+([A-Za-z0-9_.]+);/gm;
+        let impMatch;
+        while ((impMatch = importRegex.exec(content)) !== null) {
+            const importedClass = impMatch[1];
+            if (!importedClass)
+                continue;
+            const targetUniqueId = `imported:${importedClass}`;
+            db.prepare(insertEdgeTemplate).run({
+                source_type: 'file',
+                source_unique_id: normalizedPath,
+                target_type: 'class',
+                target_unique_id: targetUniqueId,
+                relation: 'imports',
+            });
+        }
+        db.prepare(markFileAsExtractedTemplate).run({ id: fileId });
+        log(`📊 Extraction summary for ${filePath}: ${functions.length} methods, ${classes.length} classes`);
+        return true;
+    }
+    catch (err) {
+        log(`❌ Failed to extract from Java file: ${filePath}`);
+        log(`   ↳ ${err.message}`);
+        try {
+            db.prepare(markFileAsFailedTemplate).run({ id: fileId });
+        }
+        catch { }
+        return false;
+    }
 }