scai 0.1.116 → 0.1.118

package/dist/db/functionExtractors/extractFromJs.js

@@ -1,267 +1,255 @@
 import { parse } from 'acorn';
 import { ancestor as walkAncestor } from 'acorn-walk';
-import { generateEmbedding } from '../../lib/generateEmbedding.js';
 import path from 'path';
+import chalk from 'chalk';
 import { log } from '../../utils/log.js';
 import { markFileAsSkippedTemplate, markFileAsExtractedTemplate, markFileAsFailedTemplate, insertFunctionTemplate, insertGraphClassTemplate, insertEdgeTemplate, insertGraphEntityTagTemplate, insertGraphTagTemplate, selectGraphTagIdTemplate, } from '../sqlTemplates.js';
 import { getDbForRepo } from '../client.js';
 import { kgModule } from '../../pipeline/modules/kgModule.js';
-function getFunctionName(node, parent, fileName) {
+import { getUniqueId } from '../../utils/sharedUtils.js';
+import { BUILTINS } from '../../fileRules/builtins.js';
+/** Determine function name from AST node and its parent */
+function getFunctionName(node, parent) {
     if (node.id?.name)
         return node.id.name;
     if (parent?.type === 'VariableDeclarator' && parent.id?.name)
-        return parent.id.name;
+        return parent.id?.name;
     if (parent?.type === 'Property' && parent.key?.name)
-        return parent.key.name;
+        return parent.key?.name;
     if (parent?.type === 'AssignmentExpression' && parent.left?.name)
-        return parent.left.name;
+        return parent.left?.name;
     if (parent?.type === 'MethodDefinition' && parent.key?.name)
-        return parent.key.name;
+        return parent.key?.name;
     return '<anon>';
 }
 export async function extractFromJS(filePath, content, fileId) {
     const db = getDbForRepo();
     const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
+    if (!fileId || typeof fileId !== 'number') {
+        console.error(`❌ extractFromJS: invalid or missing fileId for ${filePath}`);
+        return false;
+    }
+    if (!content || typeof content !== 'string' || !content.trim()) {
+        console.error(`❌ extractFromJS: empty or invalid file content for ${filePath}`);
+        db.prepare(markFileAsFailedTemplate).run({ id: fileId });
+        return false;
+    }
     try {
         const ast = parse(content, { ecmaVersion: 'latest', sourceType: 'module', locations: true });
         const functions = [];
         const classes = [];
         const imports = [];
         const exports = [];
-        // --- Traverse AST ---
+        // --- Function info ---
+        const handleFunctionNode = (node, ancestors) => {
+            const parent = ancestors[ancestors.length - 2];
+            const name = getFunctionName(node, parent);
+            const funcContent = content.slice(node.start, node.end);
+            const unique_id = getUniqueId(name, filePath, node.loc?.start.line ?? -1, node.start, funcContent);
+            functions.push({
+                name,
+                start_line: node.loc?.start.line ?? -1,
+                end_line: node.loc?.end.line ?? -1,
+                content: funcContent,
+                unique_id,
+            });
+        };
+        // --- Class info ---
+        const handleClassNode = (node) => {
+            const className = node.id?.name || `${path.basename(filePath)}:<anon-class>`;
+            const classContent = content.slice(node.start, node.end);
+            const unique_id = node.id?.name
+                ? `${className}@${normalizedPath}`
+                : getUniqueId(className, filePath, node.loc?.start.line ?? -1, node.start, classContent);
+            classes.push({
+                name: className,
+                start_line: node.loc?.start.line ?? -1,
+                end_line: node.loc?.end.line ?? -1,
+                content: classContent,
+                superClass: node.superClass?.name ?? null,
+                unique_id,
+            });
+        };
+        // --- AST traversal ---
         walkAncestor(ast, {
-            ImportDeclaration(node) {
-                if (node.source?.value)
-                    imports.push(node.source.value);
-            },
-            ExportNamedDeclaration(node) {
-                if (node.source?.value)
-                    exports.push(node.source.value);
-            },
-            ExportAllDeclaration(node) {
-                if (node.source?.value)
-                    exports.push(node.source.value);
-            },
-            FunctionDeclaration(node, ancestors) {
-                const parent = ancestors[ancestors.length - 2];
-                const name = getFunctionName(node, parent, path.basename(filePath));
-                const uniqueId = name !== '<anon>'
-                    ? `${name}@${normalizedPath}`
-                    : `${path.basename(filePath)}:<anon>@${normalizedPath}:${node.loc?.start.line}`;
-                functions.push({
-                    name,
-                    start_line: node.loc?.start.line ?? -1,
-                    end_line: node.loc?.end.line ?? -1,
-                    content: content.slice(node.start, node.end),
-                    uniqueId,
-                });
-            },
-            FunctionExpression(node, ancestors) {
-                const parent = ancestors[ancestors.length - 2];
-                const name = getFunctionName(node, parent, path.basename(filePath));
-                const uniqueId = name !== '<anon>'
-                    ? `${name}@${normalizedPath}`
-                    : `${path.basename(filePath)}:<anon>@${normalizedPath}:${node.loc?.start.line}`;
-                functions.push({
-                    name,
-                    start_line: node.loc?.start.line ?? -1,
-                    end_line: node.loc?.end.line ?? -1,
-                    content: content.slice(node.start, node.end),
-                    uniqueId,
-                });
-            },
-            ArrowFunctionExpression(node, ancestors) {
-                const parent = ancestors[ancestors.length - 2];
-                const name = getFunctionName(node, parent, path.basename(filePath));
-                const uniqueId = name !== '<anon>'
-                    ? `${name}@${normalizedPath}`
-                    : `${path.basename(filePath)}:<anon>@${normalizedPath}:${node.loc?.start.line}`;
-                functions.push({
-                    name,
-                    start_line: node.loc?.start.line ?? -1,
-                    end_line: node.loc?.end.line ?? -1,
-                    content: content.slice(node.start, node.end),
-                    uniqueId,
-                });
-            },
-            ClassDeclaration(node) {
-                const className = node.id?.name || '<anon-class>';
-                const uniqueId = className !== '<anon-class>'
-                    ? `${className}@${normalizedPath}`
-                    : `${path.basename(filePath)}:<anon-class>@${normalizedPath}:${node.loc?.start.line}`;
-                classes.push({
-                    name: className,
-                    start_line: node.loc?.start.line ?? -1,
-                    end_line: node.loc?.end.line ?? -1,
-                    content: content.slice(node.start, node.end),
-                    superClass: node.superClass?.name ?? null,
-                    uniqueId,
-                });
-            },
-            ClassExpression(node) {
-                const className = node.id?.name || '<anon-class>';
-                const uniqueId = className !== '<anon-class>'
-                    ? `${className}@${normalizedPath}`
-                    : `${path.basename(filePath)}:<anon-class>@${normalizedPath}:${node.loc?.start.line}`;
-                classes.push({
-                    name: className,
-                    start_line: node.loc?.start.line ?? -1,
-                    end_line: node.loc?.end.line ?? -1,
-                    content: content.slice(node.start, node.end),
-                    superClass: node.superClass?.name ?? null,
-                    uniqueId,
-                });
-            },
+            ImportDeclaration(node) { if (node.source?.value)
+                imports.push(node.source.value); },
+            ExportNamedDeclaration(node) { if (node.source?.value)
+                exports.push(node.source.value); },
+            ExportAllDeclaration(node) { if (node.source?.value)
+                exports.push(node.source.value); },
+            FunctionDeclaration: handleFunctionNode,
+            FunctionExpression: handleFunctionNode,
+            ArrowFunctionExpression: handleFunctionNode,
+            ClassDeclaration: handleClassNode,
+            ClassExpression: handleClassNode,
         });
         if (functions.length === 0 && classes.length === 0) {
-            log(`⚠️ No functions/classes found in: ${filePath}`);
-            db.prepare(markFileAsSkippedTemplate).run({ id: fileId });
+            log(`⚠️ No functions/classes found in JS file: ${filePath}`);
+            try {
+                db.prepare(markFileAsSkippedTemplate).run({ id: fileId });
+            }
+            catch { }
             return false;
         }
-        log(`🔍 Found ${functions.length} functions and ${classes.length} classes in ${filePath}`);
         // --- KG tagging ---
         try {
             const kgInput = { fileId, filepath: filePath, summary: undefined };
             const kgResult = await kgModule.run(kgInput, content);
-            if (kgResult.entities?.length > 0) {
-                const insertTag = db.prepare(insertGraphTagTemplate);
-                const getTagId = db.prepare(selectGraphTagIdTemplate);
-                const insertEntityTag = db.prepare(insertGraphEntityTagTemplate);
-                for (const entity of kgResult.entities) {
-                    if (!entity.type || !Array.isArray(entity.tags) || entity.tags.length === 0)
+            if (!kgResult.entities?.length) {
+                console.log(chalk.yellow(`⚠️ [KG] Garbage or empty result for ${filePath}`));
+                db.prepare(markFileAsFailedTemplate).run({ id: fileId });
+                return false;
+            }
+            const insertTagStmt = db.prepare(insertGraphTagTemplate);
+            const getTagIdStmt = db.prepare(selectGraphTagIdTemplate);
+            const insertEntityTagStmt = db.prepare(insertGraphEntityTagTemplate);
+            const persistTag = (tag) => {
+                try {
+                    insertTagStmt.run({ name: tag });
+                    return getTagIdStmt.get({ name: tag })?.id;
+                }
+                catch {
+                    return undefined;
+                }
+            };
+            const persistEntityTags = (entity) => {
+                if (!entity.type || !Array.isArray(entity.tags) || !entity.tags.length)
+                    return;
+                for (const tag of entity.tags) {
+                    if (!tag || typeof tag !== 'string')
                         continue;
-                    for (const tag of entity.tags) {
-                        if (!tag || typeof tag !== 'string')
-                            continue;
-                        try {
-                            insertTag.run({ name: tag });
-                            const tagRow = getTagId.get({ name: tag });
-                            if (!tagRow)
-                                continue;
-                            const matchedUniqueId = functions.find(f => f.name === entity.name)?.uniqueId ||
-                                classes.find(c => c.name === entity.name)?.uniqueId ||
-                                `${entity.name}@${filePath}`;
-                            insertEntityTag.run({
-                                entity_type: entity.type,
-                                entity_unique_id: matchedUniqueId,
-                                tag_id: tagRow.id,
-                            });
-                        }
-                        catch (err) {
-                            console.error('❌ Failed to persist entity/tag', { entity, tag, error: err });
-                        }
+                    const tagId = persistTag(tag);
+                    if (!tagId)
+                        continue;
+                    const matchedUniqueId = functions.find(f => f.name === entity.name)?.unique_id ||
+                        classes.find(c => c.name === entity.name)?.unique_id ||
+                        `${entity.name}@${filePath}`;
+                    try {
+                        insertEntityTagStmt.run({
+                            entity_type: entity.type,
+                            entity_unique_id: matchedUniqueId,
+                            tag_id: tagId,
+                        });
                     }
+                    catch { /* ignore */ }
                 }
-                log(`🏷 Persisted LLM-generated tags for ${filePath}`);
-            }
+            };
+            kgResult.entities.forEach(persistEntityTags);
         }
         catch (kgErr) {
-            console.warn(`⚠️ KG tagging failed for ${filePath}:`, kgErr instanceof Error ? kgErr.message : kgErr);
+            console.log(chalk.yellow(`⚠️ [KG] Garbage or failed parse for ${filePath}: ${kgErr instanceof Error ? kgErr.message : kgErr}`));
+            db.prepare(markFileAsFailedTemplate).run({ id: fileId });
+            return false;
         }
-        // --- Insert functions + edges ---
+        const seenEdges = new Set();
+        const jsBuiltins = BUILTINS.ts;
+        // --- Insert functions ---
         for (const fn of functions) {
-            try {
-                const embedding = await generateEmbedding(fn.content);
-                db.prepare(insertFunctionTemplate).run({
-                    file_id: fileId,
-                    name: fn.name,
-                    start_line: fn.start_line,
-                    end_line: fn.end_line,
-                    content: fn.content,
-                    embedding: JSON.stringify(embedding),
-                    lang: 'js',
-                    unique_id: fn.uniqueId,
-                });
+            db.prepare(insertFunctionTemplate).run({
+                file_id: fileId,
+                name: fn.name,
+                start_line: fn.start_line,
+                end_line: fn.end_line,
+                content: fn.content,
+                embedding: null,
+                lang: 'js',
+                unique_id: fn.unique_id,
+            });
+            const edgeKey = `file->${fn.unique_id}`;
+            if (!seenEdges.has(edgeKey)) {
                 db.prepare(insertEdgeTemplate).run({
                     source_type: 'file',
                     source_unique_id: normalizedPath,
                     target_type: 'function',
-                    target_unique_id: fn.uniqueId,
+                    target_unique_id: fn.unique_id,
                     relation: 'contains',
                 });
+                seenEdges.add(edgeKey);
+            }
+            // --- JS call edges ---
+            try {
                 const fnAst = parse(fn.content, { ecmaVersion: 'latest', sourceType: 'module', locations: true });
                 walkAncestor(fnAst, {
                     CallExpression(node) {
-                        const calleeName = node.callee?.name || 'unresolved';
+                        let calleeName;
+                        if (node.callee?.name)
+                            calleeName = node.callee.name;
+                        else if (node.callee?.property?.name)
+                            calleeName = node.callee.property.name;
+                        if (!calleeName || jsBuiltins.has(calleeName))
+                            return;
                         const targetUniqueId = `${calleeName}@${normalizedPath}`;
-                        db.prepare(insertEdgeTemplate).run({
-                            source_type: 'function',
-                            source_unique_id: fn.uniqueId,
-                            target_type: 'function',
-                            target_unique_id: targetUniqueId,
-                            relation: 'calls',
-                        });
+                        const callEdgeKey = `${fn.unique_id}->${targetUniqueId}`;
+                        if (!seenEdges.has(callEdgeKey)) {
+                            try {
+                                db.prepare(insertEdgeTemplate).run({
+                                    source_type: 'function',
+                                    source_unique_id: fn.unique_id,
+                                    target_type: 'function',
+                                    target_unique_id: targetUniqueId,
+                                    relation: 'calls',
+                                });
+                                seenEdges.add(callEdgeKey);
+                            }
+                            catch { /* ignore */ }
+                        }
                     },
                 });
-                log(`📌 Indexed JS function: ${fn.name}`);
             }
-            catch (err) {
-                console.error('❌ Failed to insert function or call edges', { fn, error: err });
+            catch (parseErr) {
+                console.log(chalk.yellow(`⚠️ [Extract] Failed to parse function ${fn.name} in ${filePath}: ${parseErr.message}`));
             }
         }
-        // --- Insert classes + edges ---
+        // --- Insert classes ---
         for (const cls of classes) {
-            try {
-                const embedding = await generateEmbedding(cls.content);
-                db.prepare(insertGraphClassTemplate).run({
-                    file_id: fileId,
-                    name: cls.name,
-                    start_line: cls.start_line,
-                    end_line: cls.end_line,
-                    content: cls.content,
-                    embedding: JSON.stringify(embedding),
-                    lang: 'js',
-                    unique_id: cls.uniqueId,
-                });
-                db.prepare(insertEdgeTemplate).run({
-                    source_type: 'file',
-                    source_unique_id: normalizedPath,
-                    target_type: 'class',
-                    target_unique_id: cls.uniqueId,
-                    relation: 'contains',
-                });
-                if (cls.superClass) {
-                    db.prepare(insertEdgeTemplate).run({
-                        source_type: 'class',
-                        source_unique_id: cls.uniqueId,
-                        target_type: `unresolved:${cls.superClass}`,
-                        relation: 'inherits',
-                    });
-                    log(`🔗 Class ${cls.name} extends ${cls.superClass}`);
-                }
-                log(`🏷 Indexed JS class: ${cls.name}`);
-            }
-            catch (err) {
-                console.error('❌ Failed to insert class or edges', { cls, error: err });
-            }
-        }
-        // --- Imports / Exports edges ---
-        for (const imp of imports) {
-            db.prepare(insertEdgeTemplate).run({
-                source_type: 'file',
-                source_unique_id: normalizedPath,
-                target_type: 'file',
-                target_unique_id: `file@${imp}`,
-                relation: 'imports',
+            db.prepare(insertGraphClassTemplate).run({
+                file_id: fileId,
+                name: cls.name,
+                start_line: cls.start_line,
+                end_line: cls.end_line,
+                content: cls.content,
+                embedding: null,
+                lang: 'js',
+                unique_id: cls.unique_id,
             });
-        }
-        for (const exp of exports) {
             db.prepare(insertEdgeTemplate).run({
                 source_type: 'file',
                 source_unique_id: normalizedPath,
-                target_type: 'file',
-                target_unique_id: `file@${exp}`,
-                relation: 'exports',
+                target_type: 'class',
+                target_unique_id: cls.unique_id,
+                relation: 'contains',
             });
+            if (cls.superClass) {
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'class',
+                    source_unique_id: cls.unique_id,
+                    target_type: `unresolved:${cls.superClass}`,
+                    relation: 'inherits',
+                });
+            }
         }
-        log(`📊 Extraction summary for ${filePath}: ${functions.length} functions, ${classes.length} classes, ${imports.length} imports, ${exports.length} exports`);
+        // --- Imports/Exports edges ---
+        const handleEdges = (items, relation) => {
+            for (const item of items) {
+                const resolved = item.startsWith('.') ? path.resolve(path.dirname(filePath), item) : item;
+                const normalizedTarget = path.normalize(resolved).replace(/\\/g, '/');
+                db.prepare(insertEdgeTemplate).run({
+                    source_type: 'file',
+                    source_unique_id: normalizedPath,
+                    target_type: 'file',
+                    target_unique_id: normalizedTarget,
+                    relation,
+                });
+            }
+        };
+        handleEdges(imports, 'imports');
+        handleEdges(exports, 'exports');
         db.prepare(markFileAsExtractedTemplate).run({ id: fileId });
-        log(`✅ Marked JS functions/classes as extracted for ${filePath}`);
         return true;
     }
     catch (err) {
-        log(`❌ Failed to extract from: ${filePath}`);
-        log(`   ↳ ${err.message}`);
+        console.log(chalk.yellow(`⚠️ [Extract] Failed for ${filePath}: ${err.message}`));
         db.prepare(markFileAsFailedTemplate).run({ id: fileId });
         return false;
     }