scai 0.1.117 → 0.1.118

package/dist/db/functionExtractors/extractFromTs.js

@@ -1,6 +1,5 @@
 import { Project, SyntaxKind, } from 'ts-morph';
 import path from 'path';
-import { generateEmbedding } from '../../lib/generateEmbedding.js';
 import { log } from '../../utils/log.js';
 import { getDbForRepo } from '../client.js';
 import { markFileAsSkippedTemplate, markFileAsExtractedTemplate, markFileAsFailedTemplate, insertFunctionTemplate, insertGraphClassTemplate, insertEdgeTemplate, insertGraphEntityTagTemplate, insertGraphTagTemplate, selectGraphTagIdTemplate, } from '../sqlTemplates.js';
@@ -13,104 +12,94 @@ export async function extractFromTS(filePath, content, fileId) {
     try {
         const project = new Project({ useInMemoryFileSystem: true });
         const sourceFile = project.createSourceFile(filePath, content);
-        // DTO arrays used to store into functions / graph_classes later (FTS index)
         const functions = [];
         const classes = [];
-        // --- Gather all function AST nodes ---
+        // --- Gather AST nodes ---
         const allFuncNodes = [
             ...sourceFile.getDescendantsOfKind(SyntaxKind.FunctionDeclaration),
             ...sourceFile.getDescendantsOfKind(SyntaxKind.FunctionExpression),
             ...sourceFile.getDescendantsOfKind(SyntaxKind.ArrowFunction),
         ];
-        // --- Gather all class AST nodes ---
         const allClassNodes = [
             ...sourceFile.getDescendantsOfKind(SyntaxKind.ClassDeclaration),
             ...sourceFile.getDescendantsOfKind(SyntaxKind.ClassExpression),
         ];
         if (allFuncNodes.length === 0 && allClassNodes.length === 0) {
             log(`⚠️ No functions/classes found in TS file: ${filePath}`);
-            db.prepare(markFileAsSkippedTemplate).run({ id: fileId });
+            try {
+                db.prepare(markFileAsSkippedTemplate).run({ id: fileId });
+            }
+            catch { }
             return false;
         }
         log(`🔍 Found ${allFuncNodes.length} functions and ${allClassNodes.length} classes in ${filePath}`);
-        // --- LLM tagging ---
+        // --- Knowledge Graph tagging ---
         try {
             const kgInput = { fileId, filepath: filePath, summary: undefined };
             const kgResult = await kgModule.run(kgInput, content);
-            if (kgResult.entities?.length > 0) {
-                const insertTag = db.prepare(insertGraphTagTemplate);
-                const getTagId = db.prepare(selectGraphTagIdTemplate);
-                const insertEntityTag = db.prepare(insertGraphEntityTagTemplate);
-                for (const entity of kgResult.entities) {
-                    if (!entity.type || !Array.isArray(entity.tags) || entity.tags.length === 0)
+            if (!kgResult.entities?.length) {
+                log(`⚠️ KG returned no entities for ${filePath}`);
+                try {
+                    db.prepare(markFileAsFailedTemplate).run({ id: fileId });
+                }
+                catch { }
+                return false;
+            }
+            const insertTagStmt = db.prepare(insertGraphTagTemplate);
+            const getTagIdStmt = db.prepare(selectGraphTagIdTemplate);
+            const insertEntityTagStmt = db.prepare(insertGraphEntityTagTemplate);
+            const persistTag = (tag) => {
+                try {
+                    insertTagStmt.run({ name: tag });
+                    const tagRow = getTagIdStmt.get({ name: tag });
+                    return tagRow?.id;
+                }
+                catch {
+                    return undefined;
+                }
+            };
+            const persistEntityTags = (entity) => {
+                if (!entity.type || !Array.isArray(entity.tags) || entity.tags.length === 0)
+                    return;
+                for (const tag of entity.tags) {
+                    if (!tag || typeof tag !== 'string')
                         continue;
-                    for (const tag of entity.tags) {
-                        if (!tag || typeof tag !== 'string')
-                            continue;
-                        try {
-                            insertTag.run({ name: tag });
-                            const tagRow = getTagId.get({ name: tag });
-                            if (!tagRow)
-                                continue;
-                            const matchedUniqueId = functions.find(f => f.name === entity.name)?.unique_id ||
-                                classes.find(c => c.name === entity.name)?.unique_id ||
-                                `${entity.name}@${filePath}`;
-                            insertEntityTag.run({
-                                entity_type: entity.type,
-                                entity_unique_id: matchedUniqueId,
-                                tag_id: tagRow.id,
-                            });
-                        }
-                        catch (err) {
-                            console.error('❌ Failed to persist entity/tag', { entity, tag, error: err });
-                        }
+                    const tagId = persistTag(tag);
+                    if (!tagId)
+                        continue;
+                    const matchedUniqueId = functions.find(f => f.name === entity.name)?.unique_id ||
+                        classes.find(c => c.name === entity.name)?.unique_id ||
+                        `${entity.name}@${filePath}`;
+                    try {
+                        insertEntityTagStmt.run({
+                            entity_type: entity.type,
+                            entity_unique_id: matchedUniqueId,
+                            tag_id: tagId,
+                        });
                     }
+                    catch { }
                 }
-                log(`🏷 Persisted LLM-generated tags for ${filePath}`);
-            }
+            };
+            kgResult.entities.forEach(persistEntityTags);
+            log(`🏷 Persisted LLM-generated tags for ${filePath}`);
         }
         catch (kgErr) {
-            console.warn(`⚠️ KG tagging failed for ${filePath}:`, kgErr instanceof Error ? kgErr.message : kgErr);
+            log(`⚠️ KG tagging failed for ${filePath}: ${kgErr.message}`);
+            try {
+                db.prepare(markFileAsFailedTemplate).run({ id: fileId });
+            }
+            catch { }
+            return false;
         }
-        // --- Process functions ---
+        // --- Collect functions ---
+        const tsBuiltins = BUILTINS.ts;
         for (const funcNode of allFuncNodes) {
-            const symbolName = funcNode.getSymbol()?.getName();
-            const name = symbolName ?? '<anon>';
+            const name = funcNode.getSymbol()?.getName() ?? '<anon>';
             const start = funcNode.getStartLineNumber();
             const end = funcNode.getEndLineNumber();
             const code = funcNode.getText();
             const unique_id = getUniqueId(name, filePath, start, funcNode.getStart(), code);
             functions.push({ name, start_line: start, end_line: end, content: code, unique_id });
-            try {
-                const embedding = await generateEmbedding(code);
-                db.prepare(insertFunctionTemplate).run({
-                    file_id: fileId,
-                    name,
-                    start_line: start,
-                    end_line: end,
-                    content: code,
-                    embedding: JSON.stringify(embedding),
-                    lang: 'ts',
-                    unique_id,
-                });
-            }
-            catch (err) {
-                console.error('❌ Failed to insert function row', { filePath, name, error: err });
-            }
-            // File -> Function 'contains' edge
-            try {
-                db.prepare(insertEdgeTemplate).run({
-                    source_type: 'file',
-                    source_unique_id: normalizedPath,
-                    target_type: 'function',
-                    target_unique_id: unique_id,
-                    relation: 'contains',
-                });
-            }
-            catch (err) {
-                console.error('❌ Failed to persist file->function edge', { filePath, name, error: err });
-            }
-            // AST-based calls
             try {
                 const callExprs = funcNode.getDescendantsOfKind(SyntaxKind.CallExpression);
                 const edgeSet = new Set();
@@ -120,10 +109,8 @@ export async function extractFromTS(filePath, content, fileId) {
                         const expr = callExpr.getExpression();
                         calleeName = expr.getSymbol()?.getName() ?? expr.getText();
                     }
-                    catch {
-                        calleeName = undefined;
-                    }
-                    if (!calleeName || BUILTINS.has(calleeName))
+                    catch { /* ignore symbol resolution errors */ }
+                    if (!calleeName || tsBuiltins.has(calleeName))
                         continue;
                     const targetUniqueId = `${calleeName}@${normalizedPath}`;
                     const edgeKey = `${unique_id}->${targetUniqueId}`;
@@ -138,54 +125,27 @@ export async function extractFromTS(filePath, content, fileId) {
                                 relation: 'calls',
                             });
                         }
-                        catch (err) {
-                            console.error('❌ Failed to persist call edge', { filePath, name, calleeName, error: err });
-                        }
+                        catch { /* ignore DB insert errors */ }
                     }
                 }
             }
-            catch (err) {
-                console.warn(`⚠️ Failed to inspect call expressions for function ${name} in ${filePath}:`, err);
+            catch (funcErr) {
+                log(`⚠️ Failed to process call expressions in function ${name} of ${filePath}: ${funcErr.message}`);
             }
-            log(`📌 Indexed TS function: ${name} (unique_id=${unique_id})`);
         }
-        // --- Process classes ---
+        // --- Collect classes ---
         for (const clsNode of allClassNodes) {
             const name = clsNode.getName() ?? `${path.basename(filePath)}:<anon-class>`;
             const start = clsNode.getStartLineNumber();
             const end = clsNode.getEndLineNumber();
             const code = clsNode.getText();
-            const superClass = clsNode.getExtends()?.getText() ?? null;
-            const unique_id = getUniqueId(name, filePath, start, clsNode.getStart(), code);
-            classes.push({ name, start_line: start, end_line: end, content: code, superClass, unique_id });
+            let superClass = null;
             try {
-                const embedding = await generateEmbedding(code);
-                db.prepare(insertGraphClassTemplate).run({
-                    file_id: fileId,
-                    name,
-                    start_line: start,
-                    end_line: end,
-                    content: code,
-                    embedding: JSON.stringify(embedding),
-                    lang: 'ts',
-                    unique_id,
-                });
-            }
-            catch (err) {
-                console.error('❌ Failed to insert graph class row', { filePath, name, error: err });
-            }
-            try {
-                db.prepare(insertEdgeTemplate).run({
-                    source_type: 'file',
-                    source_unique_id: normalizedPath,
-                    target_type: 'class',
-                    target_unique_id: unique_id,
-                    relation: 'contains',
-                });
-            }
-            catch (err) {
-                console.error('❌ Failed to persist file->class edge', { filePath, name, error: err });
+                superClass = clsNode.getExtends()?.getText() ?? null;
             }
+            catch { /* ignore */ }
+            const unique_id = getUniqueId(name, filePath, start, clsNode.getStart(), code);
+            classes.push({ name, start_line: start, end_line: end, content: code, superClass, unique_id });
             if (superClass) {
                 try {
                     db.prepare(insertEdgeTemplate).run({
@@ -196,72 +156,111 @@ export async function extractFromTS(filePath, content, fileId) {
                         relation: 'inherits',
                     });
                 }
-                catch (err) {
-                    console.error('❌ Failed to persist inherits edge', { filePath, name, superClass, error: err });
-                }
-                log(`🔗 Class ${name} extends ${superClass} (edge stored for later resolution)`);
+                catch { /* ignore DB insert errors */ }
             }
-            log(`🏷 Indexed TS class: ${name} (unique_id=${unique_id})`);
         }
-        // --- Imports ---
-        try {
-            const importDecls = sourceFile.getDescendantsOfKind(SyntaxKind.ImportDeclaration);
-            for (const imp of importDecls) {
-                const moduleSpecifier = imp.getModuleSpecifierValue();
-                if (!moduleSpecifier)
-                    continue;
-                const targetUniqueId = `file@${moduleSpecifier}`;
-                try {
-                    db.prepare(insertEdgeTemplate).run({
-                        source_type: 'file',
-                        source_unique_id: normalizedPath,
-                        target_type: 'file',
-                        target_unique_id: targetUniqueId,
-                        relation: 'imports',
-                    });
-                }
-                catch (err) {
-                    console.error('❌ Failed to persist import edge', { filePath, moduleSpecifier, error: err });
-                }
+        const seenEdges = new Set();
+        // --- Insert functions ---
+        for (const fn of functions) {
+            db.prepare(insertFunctionTemplate).run({
+                file_id: fileId,
+                name: fn.name ?? '<anon>',
+                start_line: fn.start_line ?? -1,
+                end_line: fn.end_line ?? -1,
+                content: fn.content ?? '',
+                embedding: null,
+                lang: 'ts',
+                unique_id: fn.unique_id,
+            });
+            const containsEdgeKey = `file->${fn.unique_id}`;
+            if (!seenEdges.has(containsEdgeKey)) {
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'file',
+                    source_unique_id: normalizedPath,
+                    target_type: 'function',
+                    target_unique_id: fn.unique_id,
+                    relation: 'contains',
+                });
+                seenEdges.add(containsEdgeKey);
             }
         }
-        catch (err) {
-            console.warn(`⚠️ Import extraction failed for ${filePath}:`, err);
+        // --- Insert classes ---
+        for (const cls of classes) {
+            db.prepare(insertGraphClassTemplate).run({
+                file_id: fileId,
+                name: cls.name,
+                start_line: cls.start_line ?? -1,
+                end_line: cls.end_line ?? -1,
+                content: cls.content ?? '',
+                embedding: null,
+                lang: 'ts',
+                unique_id: cls.unique_id,
+            });
+            const containsEdgeKey = `file->${cls.unique_id}`;
+            if (!seenEdges.has(containsEdgeKey)) {
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'file',
+                    source_unique_id: normalizedPath,
+                    target_type: 'class',
+                    target_unique_id: cls.unique_id,
+                    relation: 'contains',
+                });
+                seenEdges.add(containsEdgeKey);
+            }
         }
-        // --- Exports ---
-        try {
-            const exportDecls = sourceFile.getDescendantsOfKind(SyntaxKind.ExportDeclaration);
-            for (const exp of exportDecls) {
-                const moduleSpecifier = exp.getModuleSpecifierValue();
-                if (!moduleSpecifier)
-                    continue;
-                const targetUniqueId = `file@${moduleSpecifier}`;
-                try {
-                    db.prepare(insertEdgeTemplate).run({
-                        source_type: 'file',
-                        source_unique_id: normalizedPath,
-                        target_type: 'file',
-                        target_unique_id: targetUniqueId,
-                        relation: 'exports',
-                    });
-                }
-                catch (err) {
-                    console.error('❌ Failed to persist export edge', { filePath, moduleSpecifier, error: err });
-                }
+        // --- Imports ---
+        const importDecls = sourceFile.getDescendantsOfKind(SyntaxKind.ImportDeclaration);
+        for (const imp of importDecls) {
+            const moduleSpecifier = imp.getModuleSpecifierValue();
+            if (!moduleSpecifier)
+                continue;
+            const resolvedPath = moduleSpecifier.startsWith('.')
+                ? path.resolve(path.dirname(filePath), moduleSpecifier)
+                : moduleSpecifier;
+            const normalizedTarget = path.normalize(resolvedPath).replace(/\\/g, '/');
+            try {
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'file',
+                    source_unique_id: normalizedPath,
+                    target_type: 'file',
+                    target_unique_id: normalizedTarget,
+                    relation: 'imports',
+                });
             }
+            catch { /* ignore */ }
         }
-        catch (err) {
-            console.warn(`⚠️ Export extraction failed for ${filePath}:`, err);
+        // --- Exports ---
+        const exportDecls = sourceFile.getDescendantsOfKind(SyntaxKind.ExportDeclaration);
+        for (const exp of exportDecls) {
+            const moduleSpecifier = exp.getModuleSpecifierValue();
+            if (!moduleSpecifier)
+                continue;
+            const resolvedPath = moduleSpecifier.startsWith('.')
+                ? path.resolve(path.dirname(filePath), moduleSpecifier)
+                : moduleSpecifier;
+            const normalizedTarget = path.normalize(resolvedPath).replace(/\\/g, '/');
+            try {
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'file',
+                    source_unique_id: normalizedPath,
+                    target_type: 'file',
+                    target_unique_id: normalizedTarget,
+                    relation: 'exports',
+                });
+            }
+            catch { /* ignore */ }
         }
         db.prepare(markFileAsExtractedTemplate).run({ id: fileId });
         log(`📊 Extraction summary for ${filePath}: ${functions.length} functions, ${classes.length} classes`);
-        log(`✅ Marked TS functions/classes as extracted for ${filePath}`);
         return true;
     }
     catch (err) {
         log(`❌ Failed to extract from TS file: ${filePath}`);
-        log(`   ↳ ${String(err.message)}`);
-        db.prepare(markFileAsFailedTemplate).run({ id: fileId });
+        log(`   ↳ ${err.message}`);
+        try {
+            db.prepare(markFileAsFailedTemplate).run({ id: fileId });
+        }
+        catch { }
         return false;
     }
 }

package/dist/db/functionExtractors/index.js

@@ -6,9 +6,15 @@ import { extractFromTS } from "./extractFromTs.js";
 import { extractFromXML } from "./extractFromXML.js";
 import { detectFileType } from "../../fileRules/detectFileType.js";
 import { log } from "../../utils/log.js";
+import { CODE_ALLOWED_EXTENSIONS } from "../../fileRules/codeAllowedExtensions.js";
 export async function extractFunctionsFromFile(filePath, content, fileId) {
     const type = detectFileType(filePath).trim().toLowerCase();
     const db = getDbForRepo();
+    if (!CODE_ALLOWED_EXTENSIONS.includes(type)) {
+        log(`⚠️ File type not allowed for code extraction: ${type}. Skipping ${filePath}`);
+        db.prepare(markFileAsSkippedByPath).run({ path: filePath });
+        return false;
+    }
     try {
         let success = false;
         switch (type) {
@@ -23,17 +29,13 @@ export async function extractFunctionsFromFile(filePath, content, fileId) {
                 success = await extractFromTS(filePath, content, fileId);
                 break;
             case 'java':
-                log(`⚠️ Java extraction not implemented for ${filePath}`);
+                log(`📘 Extracting Java code from ${filePath}`);
                 await extractFromJava(filePath, content, fileId);
                 return false;
             case 'xml':
                 log(`⚠️ XML extraction not implemented for ${filePath}`);
                 await extractFromXML(filePath, content, fileId);
                 return false;
-            default:
-                log(`⚠️ Unsupported file type: ${type}. Skipping ${filePath}`);
-                db.prepare(markFileAsSkippedByPath).run({ path: filePath });
-                return false;
         }
         return success;
     }

package/dist/db/schema.js

@@ -5,20 +5,46 @@ export function initSchema() {
     db.exec(`
     CREATE TABLE IF NOT EXISTS files (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
-      path TEXT UNIQUE,  -- full path
+      path TEXT UNIQUE,             -- full path
       filename TEXT,
-      summary TEXT,
-      type TEXT,
-      indexed_at TEXT,
-      last_modified TEXT,
-      embedding TEXT,
-      processing_status TEXT NOT NULL DEFAULT 'unprocessed',
-      functions_extracted_at TEXT
+      summary TEXT,                 -- optional, can be generated later
+      content_text TEXT,            -- actual file content
+      type TEXT,                    -- file type
+      last_modified TEXT,           -- file system last modified
+      indexed_at TEXT,              -- timestamp when indexed
+      functions_extracted_at TEXT,  -- timestamp when functions/classes extracted
+      processing_status TEXT NOT NULL DEFAULT 'unprocessed'  -- tracks pipeline stage
     );
 
     CREATE VIRTUAL TABLE IF NOT EXISTS files_fts
-    USING fts5(filename, summary, path, content='files', content_rowid='id');
+    USING fts5(
+      filename, 
+      summary, 
+      path, 
+      content_text, 
+      content='files', 
+      content_rowid='id'
+    );
   `);
+    // --- Folder capsules ---
+    db.exec(`
+  CREATE TABLE IF NOT EXISTS folder_capsules (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    path TEXT UNIQUE NOT NULL,
+    depth INTEGER NOT NULL,
+    capsule_json TEXT NOT NULL,
+    confidence REAL,
+    last_generated TEXT,
+    source_file_count INTEGER
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_folder_capsules_path
+    ON folder_capsules(path);
+
+  CREATE INDEX IF NOT EXISTS idx_folder_capsules_depth
+    ON folder_capsules(depth);
+`);
+    // --- Functions table ---
     db.exec(`
     CREATE TABLE IF NOT EXISTS functions (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -28,7 +54,6 @@ export function initSchema() {
       start_line INTEGER,
       end_line INTEGER,
       content TEXT,
-      embedding TEXT,
       lang TEXT
     );
 
@@ -41,33 +66,30 @@ export function initSchema() {
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       file_id INTEGER REFERENCES files(id),
       name TEXT,
-      unique_id TEXT UNIQUE, -- e.g. "UserService@src/services/UserService.ts"
+      unique_id TEXT UNIQUE,
       start_line INTEGER,
       end_line INTEGER,
       content TEXT,
-      embedding TEXT,
       lang TEXT
     );
 
     CREATE INDEX IF NOT EXISTS idx_graph_classes_file_id ON graph_classes(file_id);
     CREATE INDEX IF NOT EXISTS idx_graph_classes_unique_id ON graph_classes(unique_id);
   `);
-    // Edges table (all relationships live here: calls, inherits, contains, imports, exports, tests, etc.)
     db.exec(`
     CREATE TABLE IF NOT EXISTS graph_edges (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
-      source_type TEXT NOT NULL, -- 'function' | 'class' | 'file' | 'test'
-      source_unique_id TEXT NOT NULL, -- full unique_id (e.g., function@filePath, filePath)
+      source_type TEXT NOT NULL,
+      source_unique_id TEXT NOT NULL,
       target_type TEXT NOT NULL,
       target_unique_id TEXT NOT NULL,
-      relation TEXT NOT NULL -- e.g., 'calls', 'imports', 'exports', 'tests'
+      relation TEXT NOT NULL
     );
 
     CREATE INDEX IF NOT EXISTS idx_graph_edges_source ON graph_edges(source_type, source_unique_id);
     CREATE INDEX IF NOT EXISTS idx_graph_edges_target ON graph_edges(target_type, target_unique_id);
     CREATE INDEX IF NOT EXISTS idx_graph_edges_relation ON graph_edges(relation);
   `);
-    // Tags setup
     db.exec(`
     CREATE TABLE IF NOT EXISTS graph_tags_master (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -76,8 +98,8 @@ export function initSchema() {
 
     CREATE TABLE IF NOT EXISTS graph_entity_tags (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
-      entity_type TEXT NOT NULL,      -- 'function' | 'class' | 'file'
-      entity_unique_id TEXT NOT NULL, -- match against functions.unique_id, classes.unique_id, or files.path
+      entity_type TEXT NOT NULL,
+      entity_unique_id TEXT NOT NULL,
       tag_id INTEGER NOT NULL REFERENCES graph_tags_master(id),
       UNIQUE(entity_type, entity_unique_id, tag_id)
     );
@@ -85,5 +107,18 @@ export function initSchema() {
     CREATE INDEX IF NOT EXISTS idx_graph_entity_tags_entity ON graph_entity_tags(entity_type, entity_unique_id);
     CREATE INDEX IF NOT EXISTS idx_graph_entity_tags_tag ON graph_entity_tags(tag_id);
   `);
-    console.log("✅ Graph schema initialized (files, functions, classes, edges, tags with unique_ids)");
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS summaries (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      path TEXT UNIQUE,
+      type TEXT NOT NULL CHECK (type IN ('folder','project')),
+      summary TEXT,
+      last_generated TEXT,
+      child_latest_modified TEXT
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_summaries_type ON summaries(type);
+    CREATE INDEX IF NOT EXISTS idx_summaries_path ON summaries(path);
+  `);
+    console.log("✅ Graph schema initialized (files, functions, classes, edges, tags, summaries, FTS content_text)");
 }