scai 0.1.34 → 0.1.36

package/dist/daemon/daemonBatch.js

@@ -1,4 +1,4 @@
-import { summaryModule } from '../pipeline/modules/summaryModule.js';
+import { indexFunctionsForFile } from '../db/functionIndex.js';
 import { db } from '../db/client.js';
 import fs from 'fs/promises';
 import fsSync from 'fs';
@@ -7,6 +7,7 @@ import { DB_PATH } from '../constants.js';
 import { log } from '../utils/log.js';
 import lockfile from 'proper-lockfile';
 import { shouldIgnoreFile } from '../utils/shouldIgnoreFiles.js';
+import { summaryModule } from '../pipeline/modules/summaryModule.js';
 const MAX_FILES_PER_BATCH = 5;
 async function lockDb() {
     try {
@@ -18,9 +19,11 @@ async function lockDb() {
     }
 }
 export async function runDaemonBatch() {
+    log('🟡 Starting daemon batch...');
     const rows = db.prepare(`
-    SELECT path, type FROM files
-    WHERE summary IS NULL OR summary = ''
+    SELECT path, type, functions_extracted FROM files
+    WHERE (summary IS NULL OR summary = '') 
+      OR (functions_extracted IS NULL OR functions_extracted = 0)
     ORDER BY last_modified DESC
     LIMIT ?
   `).all(MAX_FILES_PER_BATCH);
@@ -30,6 +33,7 @@ export async function runDaemonBatch() {
     }
     const release = await lockDb();
     for (const row of rows) {
+        log(`📂 Processing file: ${row.path}`);
         if (!fsSync.existsSync(row.path)) {
             log(`⚠️ Skipped missing file: ${row.path}`);
             continue;
@@ -40,26 +44,41 @@ export async function runDaemonBatch() {
         }
         try {
             const content = await fs.readFile(row.path, 'utf-8');
-            const result = await summaryModule.run({ content, filepath: row.path });
-            const summary = result?.summary?.trim() || null;
+            // 1. Extract functions and function calls
+            const fileResult = db
+                .prepare(`SELECT id FROM files WHERE path = ?`)
+                .get(row.path);
+            const fileId = fileResult?.id;
+            if (!fileId) {
+                log(`⚠️ Could not find fileId for ${row.path}`);
+            }
+            else {
+                await indexFunctionsForFile(row.path, fileId);
+            }
+            // 2. Summarize the file 
+            log(`📝 Generating summary for ${row.path}...`);
+            const summaryResult = await summaryModule.run({ content, filepath: row.path });
+            const summary = summaryResult?.summary?.trim() || null;
             let embedding = null;
             if (summary) {
                 const vector = await generateEmbedding(summary);
-                if (vector)
+                if (vector) {
                     embedding = JSON.stringify(vector);
+                }
             }
             db.prepare(`
         UPDATE files
         SET summary = @summary, embedding = @embedding, indexed_at = datetime('now')
         WHERE path = @path
       `).run({ summary, embedding, path: row.path });
-            log(`📝 Summarized: ${row.path}`);
+            log(`✅ Updated summary & embedding for ${row.path}\n`);
         }
         catch (err) {
-            log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}`);
+            log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}\n`);
         }
         await new Promise(resolve => setTimeout(resolve, 200));
     }
     await release();
+    log('✅ Finished daemon batch.\n\n');
     return true;
 }

package/dist/db/fileIndex.js

@@ -2,8 +2,8 @@ import { db } from './client.js';
 import fs from 'fs';
 import path from 'path';
 import { generateEmbedding } from '../lib/generateEmbedding.js';
-import * as sqlTemplates from './sqlTemplates.js';
 import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
+import * as sqlTemplates from './sqlTemplates.js';
 /**
  * Index a file into the local SQLite database.
  *
@@ -16,6 +16,7 @@ export function indexFile(filePath, summary, type) {
     const lastModified = stats.mtime.toISOString();
     const indexedAt = new Date().toISOString();
     const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
+    // Index the file metadata
     db.prepare(sqlTemplates.upsertFileTemplate).run({
         path: normalizedPath,
         summary,
@@ -64,9 +65,6 @@ export async function searchFiles(query, topK = 5) {
     if (ftsResults.length === 0) {
         return [];
     }
-    ftsResults.forEach(result => {
-        console.log(`📄 FTS found: ${result.path}`);
-    });
     const bm25Min = Math.min(...ftsResults.map(r => r.bm25Score));
     const bm25Max = Math.max(...ftsResults.map(r => r.bm25Score));
     const scored = ftsResults.map(result => {
@@ -85,7 +83,6 @@ export async function searchFiles(query, topK = 5) {
             }
         }
         else {
-            // No embedding: fallback to inverse bm25-only
             finalScore = 1 - ((result.bm25Score - bm25Min) / (bm25Max - bm25Min + 1e-5));
         }
         return {

package/dist/db/functionExtractors/extractFromJava.js

@@ -0,0 +1,4 @@
+// extractFromJava.ts
+export async function extractFromJava(filePath, content, fileId) {
+    console.warn(`⛔️ Java extraction not implemented: ${filePath}`);
+}

package/dist/db/functionExtractors/extractFromJs.js

@@ -0,0 +1,89 @@
+import { parse } from 'acorn';
+import { simple as walkSimple } from 'acorn-walk';
+import { generateEmbedding } from '../../lib/generateEmbedding.js';
+import { db } from '../client.js';
+import path from 'path';
+import { log } from '../../utils/log.js';
+/**
+ * Parses a JavaScript/TypeScript file, extracts all top-level functions,
+ * generates embeddings, and indexes both the functions and any calls made
+ * within each function into the database.
+ */
+export async function extractFromJS(filePath, content, fileId) {
+    const ast = parse(content, { ecmaVersion: 'latest', locations: true });
+    const functions = [];
+    walkSimple(ast, {
+        FunctionDeclaration(node) {
+            const name = node.id?.name || `${path.basename(filePath)}:<anon>`;
+            const startLine = node?.loc?.start.line ?? -1;
+            const endLine = node?.loc?.end.line ?? -1;
+            const body = content.slice(node.start, node.end);
+            functions.push({ name, startLine, endLine, body });
+        },
+        FunctionExpression(node) {
+            const name = `${path.basename(filePath)}:<anon>`;
+            const startLine = node?.loc?.start.line ?? -1;
+            const endLine = node?.loc?.end.line ?? -1;
+            const body = content.slice(node.start, node.end);
+            functions.push({ name, startLine, endLine, body });
+        },
+        ArrowFunctionExpression(node) {
+            const name = `${path.basename(filePath)}:<anon>`;
+            const startLine = node?.loc?.start.line ?? -1;
+            const endLine = node?.loc?.end.line ?? -1;
+            const body = content.slice(node.start, node.end);
+            functions.push({ name, startLine, endLine, body });
+        },
+    });
+    if (functions.length === 0) {
+        log(`⚠️ No functions found in: ${filePath}`);
+        return;
+    }
+    log(`🔍 Found ${functions.length} functions in ${filePath}`);
+    for (const fn of functions) {
+        const embedding = await generateEmbedding(fn.body);
+        const result = db.prepare(`
+      INSERT INTO functions (
+        file_id, name, start_line, end_line, content, embedding, lang
+      ) VALUES (
+        @file_id, @name, @start_line, @end_line, @content, @embedding, @lang
+      )
+    `).run({
+            file_id: fileId,
+            name: fn.name,
+            start_line: fn.startLine,
+            end_line: fn.endLine,
+            content: fn.body,
+            embedding: JSON.stringify(embedding),
+            lang: 'js'
+        });
+        const callerId = result.lastInsertRowid;
+        const fnAst = parse(fn.body, { ecmaVersion: 'latest' });
+        const calls = [];
+        walkSimple(fnAst, {
+            CallExpression(node) {
+                if (node.callee.type === 'Identifier' && node.callee.name) {
+                    calls.push({ calleeName: node.callee.name });
+                }
+            }
+        });
+        for (const call of calls) {
+            db.prepare(`
+        INSERT INTO function_calls (caller_id, callee_name)
+        VALUES (@caller_id, @callee_name)
+      `).run({
+                caller_id: callerId,
+                callee_name: call.calleeName
+            });
+        }
+        log(`📌 Indexed function: ${fn.name} with ${calls.length} calls`);
+    }
+    // Mark the file as successfully extracted
+    db.prepare(`
+    UPDATE files 
+    SET functions_extracted = 1,
+        functions_extracted_at = datetime('now')
+    WHERE id = @fileId
+`).run({ fileId });
+    log(`✅ Marked functions as extracted for ${filePath}`);
+}

package/dist/db/functionExtractors/extractFromXML.js

@@ -0,0 +1,4 @@
+// extractFromJava.ts
+export async function extractFromXML(filePath, content, fileId) {
+    console.warn(`⛔️ XML extraction not implemented: ${filePath}`);
+}

package/dist/db/functionExtractors/index.js

@@ -0,0 +1,29 @@
+import { log } from '../../utils/log.js';
+import { detectFileType } from '../../utils/detectFileType.js';
+import { extractFromJava } from './extractFromJava.js';
+import { extractFromJS } from './extractFromJs.js';
+import { extractFromXML } from './extractFromXML.js';
+/**
+ * Detects file type and delegates to the appropriate extractor.
+ */
+export async function extractFunctionsFromFile(filePath, content, fileId) {
+    const type = detectFileType(filePath).trim().toLowerCase();
+    ;
+    if (type === 'js' || type === 'ts') {
+        log(`✅ Attempting to extract JS functions from ${filePath}\n`);
+        await extractFromJS(filePath, content, fileId);
+        return true;
+    }
+    if (type === 'java') {
+        log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
+        await extractFromJava(filePath, content, fileId);
+        return false;
+    }
+    if (type === 'xml') {
+        log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
+        await extractFromXML(filePath, content, fileId);
+        return false;
+    }
+    log(`⚠️ Unsupported file type: ${type} for function extraction. Skipping ${filePath}`);
+    return false;
+}

package/dist/db/functionIndex.js

@@ -0,0 +1,11 @@
+import fs from 'fs';
+import path from 'path';
+import { extractFunctionsFromFile } from './functionExtractors/index.js';
+/**
+ * Extracts functions from file if language is supported.
+ */
+export async function indexFunctionsForFile(filePath, fileId) {
+    const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
+    const content = fs.readFileSync(filePath, 'utf-8');
+    await extractFunctionsFromFile(normalizedPath, content, fileId);
+}

package/dist/db/schema.js

@@ -32,4 +32,22 @@ export function initSchema() {
     END;
   `);
     console.log('✅ SQLite schema initialized with FTS5 triggers');
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS functions (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      file_id INTEGER REFERENCES files(id),
+      name TEXT,
+      start_line INTEGER,
+      end_line INTEGER,
+      content TEXT,
+      embedding TEXT,
+      lang TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS function_calls (
+      caller_id INTEGER REFERENCES functions(id),
+      callee_name TEXT
+    );
+  `);
+    console.log('✅ Schema for functions and function_calls initialized');
 }

package/dist/db/sqlTemplates.js

@@ -39,3 +39,13 @@ export const rawQueryTemplate = `
   ORDER BY rank
   LIMIT :limit
 `;
+// Insert function metadata
+export const insertFunctionTemplate = `
+  INSERT INTO functions (file_id, name, start_line, end_line, content, embedding, lang)
+  VALUES (:file_id, :name, :start_line, :end_line, :content, :embedding, :lang)
+`;
+// Insert function call edge
+export const insertFunctionCallTemplate = `
+  INSERT INTO function_calls (caller_id, callee_name)
+  VALUES (:caller_id, :callee_name)
+`;

package/dist/lib/generate.js

@@ -1,7 +1,8 @@
 // File: lib/generate.ts
-import ora from 'ora';
+import { Spinner } from './spinner.js';
 export async function generate(input, model) {
-    const spinner = ora(`🧠 Thinking with ${model}...`).start();
+    const spinner = new Spinner(`🧠 Thinking with ${model}...`);
+    spinner.start();
     try {
         const res = await fetch('http://localhost:11434/api/generate', {
             method: 'POST',
@@ -13,16 +14,16 @@ export async function generate(input, model) {
             }),
         });
         const data = await res.json();
-        spinner.succeed('✅ Model response received.');
-        process.stdout.write('\n'); // ✅ Prevents terminal suppression bug
+        spinner.succeed('Model response received.');
+        process.stdout.write('\n');
         return {
             content: data.response?.trim() ?? '',
             filepath: input.filepath,
         };
     }
     catch (err) {
-        spinner.fail('❌ Model request failed.');
-        process.stdout.write('\n'); // In case of error flush output too
+        spinner.fail('Model request failed.');
+        process.stdout.write('\n');
         throw err;
     }
 }

package/dist/lib/spinner.js

@@ -0,0 +1,31 @@
+// lib/spinner.ts
+export class Spinner {
+    constructor(message) {
+        this.frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+        this.interval = null;
+        this.i = 0;
+        this.text = message;
+    }
+    start() {
+        process.stdout.write('\x1b[?25l'); // hide cursor
+        this.interval = setInterval(() => {
+            const frame = this.frames[this.i = ++this.i % this.frames.length];
+            process.stdout.write(`\r${frame} ${this.text} `);
+        }, 80);
+    }
+    succeed(msg) {
+        this.stop();
+        process.stdout.write(`\r✅ ${msg}\n`);
+    }
+    fail(msg) {
+        this.stop();
+        process.stdout.write(`\r❌ ${msg}\n`);
+    }
+    stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+        process.stdout.write('\x1b[?25h'); // show cursor
+    }
+}

package/dist/scripts/addFunctionsExtractedColumn.js

@@ -0,0 +1,29 @@
+import { db } from '../db/client.js';
+try {
+    db.prepare(`
+    ALTER TABLE files ADD COLUMN functions_extracted BOOLEAN DEFAULT 0
+  `).run();
+    console.log('✅ Added functions_extracted column.');
+}
+catch (e) {
+    if (e instanceof Error ? e.message.includes('duplicate column name') : e) {
+        console.log('ℹ️ Column functions_extracted already exists. Skipping.');
+    }
+    else {
+        console.error('❌ Migration failed:', e instanceof Error ? e.message : e);
+    }
+}
+try {
+    db.prepare(`
+    ALTER TABLE files ADD COLUMN functions_extracted_at DATETIME
+  `).run();
+    console.log('✅ Added functions_extracted_at column.');
+}
+catch (e) {
+    if (e instanceof Error ? e.message.includes('duplicate column name') : e) {
+        console.log('ℹ️ Column functions_extracted_at already exists. Skipping.');
+    }
+    else {
+        console.error('❌ Migration failed:', e instanceof Error ? e.message : e);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.34",
+  "version": "0.1.36",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"
@@ -25,16 +25,17 @@
     "start": "node dist/index.js"
   },
   "dependencies": {
+    "acorn": "^8.11.3",
+    "acorn-walk": "^8.3.2",
     "better-sqlite3": "^12.1.1",
     "commander": "^11.0.0",
     "fast-glob": "^3.3.3",
-    "ora": "^8.2.0",
     "proper-lockfile": "^4.1.2"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",
     "@types/jest": "^30.0.0",
-    "@types/node": "^24.0.1",
+    "@types/node": "^24.0.13",
     "@types/proper-lockfile": "^4.1.4",
     "jest": "^30.0.2",
     "ts-jest": "^29.4.0",