scai 0.1.35 → 0.1.36

package/dist/daemon/daemonBatch.js

@@ -1,4 +1,4 @@
-import { summaryModule } from '../pipeline/modules/summaryModule.js';
+import { indexFunctionsForFile } from '../db/functionIndex.js';
 import { db } from '../db/client.js';
 import fs from 'fs/promises';
 import fsSync from 'fs';
@@ -7,6 +7,7 @@ import { DB_PATH } from '../constants.js';
 import { log } from '../utils/log.js';
 import lockfile from 'proper-lockfile';
 import { shouldIgnoreFile } from '../utils/shouldIgnoreFiles.js';
+import { summaryModule } from '../pipeline/modules/summaryModule.js';
 const MAX_FILES_PER_BATCH = 5;
 async function lockDb() {
     try {
@@ -18,9 +19,11 @@ async function lockDb() {
     }
 }
 export async function runDaemonBatch() {
+    log('🟡 Starting daemon batch...');
     const rows = db.prepare(`
-    SELECT path, type FROM files
-    WHERE summary IS NULL OR summary = ''
+    SELECT path, type, functions_extracted FROM files
+    WHERE (summary IS NULL OR summary = '') 
+      OR (functions_extracted IS NULL OR functions_extracted = 0)
     ORDER BY last_modified DESC
     LIMIT ?
   `).all(MAX_FILES_PER_BATCH);
@@ -30,6 +33,7 @@ export async function runDaemonBatch() {
     }
     const release = await lockDb();
     for (const row of rows) {
+        log(`📂 Processing file: ${row.path}`);
         if (!fsSync.existsSync(row.path)) {
             log(`⚠️ Skipped missing file: ${row.path}`);
             continue;
@@ -40,26 +44,41 @@ export async function runDaemonBatch() {
         }
         try {
             const content = await fs.readFile(row.path, 'utf-8');
-            const result = await summaryModule.run({ content, filepath: row.path });
-            const summary = result?.summary?.trim() || null;
+            // 1. Extract functions and function calls
+            const fileResult = db
+                .prepare(`SELECT id FROM files WHERE path = ?`)
+                .get(row.path);
+            const fileId = fileResult?.id;
+            if (!fileId) {
+                log(`⚠️ Could not find fileId for ${row.path}`);
+            }
+            else {
+                await indexFunctionsForFile(row.path, fileId);
+            }
+            // 2. Summarize the file 
+            log(`📝 Generating summary for ${row.path}...`);
+            const summaryResult = await summaryModule.run({ content, filepath: row.path });
+            const summary = summaryResult?.summary?.trim() || null;
             let embedding = null;
             if (summary) {
                 const vector = await generateEmbedding(summary);
-                if (vector)
+                if (vector) {
                     embedding = JSON.stringify(vector);
+                }
             }
             db.prepare(`
         UPDATE files
         SET summary = @summary, embedding = @embedding, indexed_at = datetime('now')
         WHERE path = @path
       `).run({ summary, embedding, path: row.path });
-            log(`📝 Summarized: ${row.path}`);
+            log(`✅ Updated summary & embedding for ${row.path}\n`);
         }
         catch (err) {
-            log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}`);
+            log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}\n`);
         }
         await new Promise(resolve => setTimeout(resolve, 200));
     }
     await release();
+    log('✅ Finished daemon batch.\n\n');
     return true;
 }

package/dist/db/fileIndex.js

@@ -2,8 +2,8 @@ import { db } from './client.js';
 import fs from 'fs';
 import path from 'path';
 import { generateEmbedding } from '../lib/generateEmbedding.js';
-import * as sqlTemplates from './sqlTemplates.js';
 import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
+import * as sqlTemplates from './sqlTemplates.js';
 /**
  * Index a file into the local SQLite database.
  *
@@ -16,6 +16,7 @@ export function indexFile(filePath, summary, type) {
     const lastModified = stats.mtime.toISOString();
     const indexedAt = new Date().toISOString();
     const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
+    // Index the file metadata
     db.prepare(sqlTemplates.upsertFileTemplate).run({
         path: normalizedPath,
         summary,
@@ -64,9 +65,6 @@ export async function searchFiles(query, topK = 5) {
     if (ftsResults.length === 0) {
         return [];
     }
-    ftsResults.forEach(result => {
-        console.log(`📄 FTS found: ${result.path}`);
-    });
     const bm25Min = Math.min(...ftsResults.map(r => r.bm25Score));
     const bm25Max = Math.max(...ftsResults.map(r => r.bm25Score));
     const scored = ftsResults.map(result => {
@@ -85,7 +83,6 @@ export async function searchFiles(query, topK = 5) {
             }
         }
         else {
-            // No embedding: fallback to inverse bm25-only
             finalScore = 1 - ((result.bm25Score - bm25Min) / (bm25Max - bm25Min + 1e-5));
         }
         return {

package/dist/db/functionExtractors/extractFromJava.js

@@ -0,0 +1,4 @@
+// extractFromJava.ts
+export async function extractFromJava(filePath, content, fileId) {
+    console.warn(`⛔️ Java extraction not implemented: ${filePath}`);
+}

package/dist/db/functionExtractors/extractFromJs.js

@@ -0,0 +1,89 @@
+import { parse } from 'acorn';
+import { simple as walkSimple } from 'acorn-walk';
+import { generateEmbedding } from '../../lib/generateEmbedding.js';
+import { db } from '../client.js';
+import path from 'path';
+import { log } from '../../utils/log.js';
+/**
+ * Parses a JavaScript/TypeScript file, extracts all top-level functions,
+ * generates embeddings, and indexes both the functions and any calls made
+ * within each function into the database.
+ */
+export async function extractFromJS(filePath, content, fileId) {
+    const ast = parse(content, { ecmaVersion: 'latest', locations: true });
+    const functions = [];
+    walkSimple(ast, {
+        FunctionDeclaration(node) {
+            const name = node.id?.name || `${path.basename(filePath)}:<anon>`;
+            const startLine = node?.loc?.start.line ?? -1;
+            const endLine = node?.loc?.end.line ?? -1;
+            const body = content.slice(node.start, node.end);
+            functions.push({ name, startLine, endLine, body });
+        },
+        FunctionExpression(node) {
+            const name = `${path.basename(filePath)}:<anon>`;
+            const startLine = node?.loc?.start.line ?? -1;
+            const endLine = node?.loc?.end.line ?? -1;
+            const body = content.slice(node.start, node.end);
+            functions.push({ name, startLine, endLine, body });
+        },
+        ArrowFunctionExpression(node) {
+            const name = `${path.basename(filePath)}:<anon>`;
+            const startLine = node?.loc?.start.line ?? -1;
+            const endLine = node?.loc?.end.line ?? -1;
+            const body = content.slice(node.start, node.end);
+            functions.push({ name, startLine, endLine, body });
+        },
+    });
+    if (functions.length === 0) {
+        log(`⚠️ No functions found in: ${filePath}`);
+        return;
+    }
+    log(`🔍 Found ${functions.length} functions in ${filePath}`);
+    for (const fn of functions) {
+        const embedding = await generateEmbedding(fn.body);
+        const result = db.prepare(`
+      INSERT INTO functions (
+        file_id, name, start_line, end_line, content, embedding, lang
+      ) VALUES (
+        @file_id, @name, @start_line, @end_line, @content, @embedding, @lang
+      )
+    `).run({
+            file_id: fileId,
+            name: fn.name,
+            start_line: fn.startLine,
+            end_line: fn.endLine,
+            content: fn.body,
+            embedding: JSON.stringify(embedding),
+            lang: 'js'
+        });
+        const callerId = result.lastInsertRowid;
+        const fnAst = parse(fn.body, { ecmaVersion: 'latest' });
+        const calls = [];
+        walkSimple(fnAst, {
+            CallExpression(node) {
+                if (node.callee.type === 'Identifier' && node.callee.name) {
+                    calls.push({ calleeName: node.callee.name });
+                }
+            }
+        });
+        for (const call of calls) {
+            db.prepare(`
+        INSERT INTO function_calls (caller_id, callee_name)
+        VALUES (@caller_id, @callee_name)
+      `).run({
+                caller_id: callerId,
+                callee_name: call.calleeName
+            });
+        }
+        log(`📌 Indexed function: ${fn.name} with ${calls.length} calls`);
+    }
+    // Mark the file as successfully extracted
+    db.prepare(`
+    UPDATE files 
+    SET functions_extracted = 1,
+        functions_extracted_at = datetime('now')
+    WHERE id = @fileId
+`).run({ fileId });
+    log(`✅ Marked functions as extracted for ${filePath}`);
+}

package/dist/db/functionExtractors/extractFromXML.js

@@ -0,0 +1,4 @@
+// extractFromJava.ts
+export async function extractFromXML(filePath, content, fileId) {
+    console.warn(`⛔️ XML extraction not implemented: ${filePath}`);
+}

package/dist/db/functionExtractors/index.js

@@ -0,0 +1,29 @@
+import { log } from '../../utils/log.js';
+import { detectFileType } from '../../utils/detectFileType.js';
+import { extractFromJava } from './extractFromJava.js';
+import { extractFromJS } from './extractFromJs.js';
+import { extractFromXML } from './extractFromXML.js';
+/**
+ * Detects file type and delegates to the appropriate extractor.
+ */
+export async function extractFunctionsFromFile(filePath, content, fileId) {
+    const type = detectFileType(filePath).trim().toLowerCase();
+    ;
+    if (type === 'js' || type === 'ts') {
+        log(`✅ Attempting to extract JS functions from ${filePath}\n`);
+        await extractFromJS(filePath, content, fileId);
+        return true;
+    }
+    if (type === 'java') {
+        log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
+        await extractFromJava(filePath, content, fileId);
+        return false;
+    }
+    if (type === 'xml') {
+        log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
+        await extractFromXML(filePath, content, fileId);
+        return false;
+    }
+    log(`⚠️ Unsupported file type: ${type} for function extraction. Skipping ${filePath}`);
+    return false;
+}

package/dist/db/functionIndex.js

@@ -0,0 +1,11 @@
+import fs from 'fs';
+import path from 'path';
+import { extractFunctionsFromFile } from './functionExtractors/index.js';
+/**
+ * Extracts functions from file if language is supported.
+ */
+export async function indexFunctionsForFile(filePath, fileId) {
+    const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
+    const content = fs.readFileSync(filePath, 'utf-8');
+    await extractFunctionsFromFile(normalizedPath, content, fileId);
+}

package/dist/db/schema.js

@@ -32,4 +32,22 @@ export function initSchema() {
     END;
   `);
     console.log('✅ SQLite schema initialized with FTS5 triggers');
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS functions (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      file_id INTEGER REFERENCES files(id),
+      name TEXT,
+      start_line INTEGER,
+      end_line INTEGER,
+      content TEXT,
+      embedding TEXT,
+      lang TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS function_calls (
+      caller_id INTEGER REFERENCES functions(id),
+      callee_name TEXT
+    );
+  `);
+    console.log('✅ Schema for functions and function_calls initialized');
 }

package/dist/db/sqlTemplates.js

@@ -39,3 +39,13 @@ export const rawQueryTemplate = `
   ORDER BY rank
   LIMIT :limit
 `;
+// Insert function metadata
+export const insertFunctionTemplate = `
+  INSERT INTO functions (file_id, name, start_line, end_line, content, embedding, lang)
+  VALUES (:file_id, :name, :start_line, :end_line, :content, :embedding, :lang)
+`;
+// Insert function call edge
+export const insertFunctionCallTemplate = `
+  INSERT INTO function_calls (caller_id, callee_name)
+  VALUES (:caller_id, :callee_name)
+`;

package/dist/scripts/addFunctionsExtractedColumn.js

@@ -0,0 +1,29 @@
+import { db } from '../db/client.js';
+try {
+    db.prepare(`
+    ALTER TABLE files ADD COLUMN functions_extracted BOOLEAN DEFAULT 0
+  `).run();
+    console.log('✅ Added functions_extracted column.');
+}
+catch (e) {
+    if (e instanceof Error ? e.message.includes('duplicate column name') : e) {
+        console.log('ℹ️ Column functions_extracted already exists. Skipping.');
+    }
+    else {
+        console.error('❌ Migration failed:', e instanceof Error ? e.message : e);
+    }
+}
+try {
+    db.prepare(`
+    ALTER TABLE files ADD COLUMN functions_extracted_at DATETIME
+  `).run();
+    console.log('✅ Added functions_extracted_at column.');
+}
+catch (e) {
+    if (e instanceof Error ? e.message.includes('duplicate column name') : e) {
+        console.log('ℹ️ Column functions_extracted_at already exists. Skipping.');
+    }
+    else {
+        console.error('❌ Migration failed:', e instanceof Error ? e.message : e);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.35",
+  "version": "0.1.36",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"
@@ -25,6 +25,8 @@
     "start": "node dist/index.js"
   },
   "dependencies": {
+    "acorn": "^8.11.3",
+    "acorn-walk": "^8.3.2",
     "better-sqlite3": "^12.1.1",
     "commander": "^11.0.0",
     "fast-glob": "^3.3.3",
@@ -33,7 +35,7 @@
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",
     "@types/jest": "^30.0.0",
-    "@types/node": "^24.0.1",
+    "@types/node": "^24.0.13",
     "@types/proper-lockfile": "^4.1.4",
     "jest": "^30.0.2",
     "ts-jest": "^29.4.0",