scai 0.1.43 → 0.1.45

package/dist/commands/AskCmd.js

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import readline from 'readline';
-import { searchFiles, queryFiles } from '../db/fileIndex.js';
+import { searchFiles, queryFiles, getFunctionsForFiles } from '../db/fileIndex.js';
 import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
 import { generate } from '../lib/generate.js';
 import { buildContextualPrompt } from '../utils/buildContextualPrompt.js';
@@ -10,7 +10,6 @@ import { log } from '../utils/log.js';
 import { PROMPT_LOG_PATH, SCAI_HOME, INDEX_DIR } from '../constants.js';
 const MAX_RELATED_FILES = 5;
 export async function runAskCommand(query) {
-    // 🧠 Prompt the user if no query is passed
     if (!query) {
         query = await promptOnce('🧠 Ask your question:\n> ');
     }
@@ -21,26 +20,24 @@ export async function runAskCommand(query) {
     }
     console.log(`📁 Using index root: ${INDEX_DIR}`);
     console.log(`🔍 Searching for: "${query}"\n`);
-    // 🧠 Step 1: Semantic + fallback search
+    // 🟩 STEP 1: Semantic Search
     const start = Date.now();
-    const semanticResults = await searchFiles(query, MAX_RELATED_FILES);
+    const semanticResults = await searchFiles(query, MAX_RELATED_FILES); // RankedFile[]
     const duration = Date.now() - start;
     console.log(`⏱️ searchFiles took ${duration}ms and returned ${semanticResults.length} result(s)`);
-    // 🔍 Log raw semantic results
-    console.log('🔍 Raw semantic search results:');
     semanticResults.forEach((file, i) => {
         console.log(`  ${i + 1}. 📄 Path: ${file.path} | Score: ${file.score?.toFixed(3) ?? 'n/a'}`);
     });
+    // 🟩 STEP 1.5: Fallback FTS search
     const safeQuery = sanitizeQueryForFts(query);
-    const fallbackResults = queryFiles(safeQuery, 10);
-    // 🔍 Log raw keyword fallback results
-    console.log('\n🔍 Raw fallback keyword (FTS) search results:');
+    const fallbackResults = queryFiles(safeQuery, 10); // FileRow[]
     fallbackResults.forEach((file, i) => {
-        console.log(`  ${i + 1}. 📄 Path: ${file.path}`);
+        console.log(`  ${i + 1}. 🔎 Fallback Match: ${file.path}`);
     });
-    // 🧠 Step 2: Merge results
+    // 🟩 STEP 2: Merge results (de-duplicate by full resolved path)
     const seen = new Set();
     const combinedResults = [];
+    // Merging results now ensures all have `id`
     for (const file of semanticResults) {
         const resolved = path.resolve(file.path);
         seen.add(resolved);
@@ -51,43 +48,71 @@ export async function runAskCommand(query) {
         if (!seen.has(resolved)) {
             seen.add(resolved);
             combinedResults.push({
+                id: file.id, // Ensure the id is included here
                 path: file.path,
-                summary: file.summary,
+                summary: file.summary || '',
                 score: 0.0,
+                sim: 0,
+                bm25: 0
             });
         }
     }
-    if (combinedResults.length > 0) {
-        console.log('📊 Closest files based on semantic + keyword match:');
-        combinedResults.forEach((file, i) => {
-            console.log(`  ${i + 1}. 📄 Path: ${file.path} | Score: ${file.score?.toFixed(3) ?? 'fallback'}`);
+    // 🟩 STEP 3: Log results
+    if (combinedResults.length) {
+        console.log('\n📊 Final Related Files:');
+        combinedResults.forEach((f, i) => {
+            console.log(`  ${i + 1}. ${f.path} (${f.score?.toFixed(3) ?? 'fallback'})`);
         });
     }
     else {
-        console.log('⚠️ No similar files found. Asking the model using question only...');
+        console.log('⚠️ No similar files found. Using query only.');
     }
-    // 🧠 Step 3: Build metadata for prompt
-    const relatedFiles = combinedResults.slice(0, MAX_RELATED_FILES).map((file, index) => ({
-        path: file.path,
-        summary: file.summary || '(No summary available)', // Ensure summary is included
-    }));
-    // Get the top-ranked file (the first one in the sorted results)
-    const topRankedFile = combinedResults[0]; // The most relevant file
+    // 🟩 STEP 4: Load top file code + metadata
+    const topFile = combinedResults[0];
+    const filepath = topFile?.path || '';
+    let code = '';
+    let topSummary = topFile.summary || '(No summary available)';
+    let topFunctions = [];
+    // Gather all file IDs from the combined results
+    const allFileIds = combinedResults
+        .map(file => file.id) // Now file.id exists
+        .filter((id) => typeof id === 'number');
+    const allFunctionsMap = getFunctionsForFiles(allFileIds);
+    try {
+        code = fs.readFileSync(filepath, 'utf-8');
+        const topFileId = topFile.id;
+        topFunctions = allFunctionsMap[topFileId]?.map(fn => fn.name) || [];
+    }
+    catch (err) {
+        console.warn(`⚠️ Failed to read or analyze top file (${filepath}):`, err);
+    }
+    // 🟩 STEP 5: Build relatedFiles with functions
+    const relatedFiles = combinedResults.slice(0, MAX_RELATED_FILES).map(file => {
+        const fileId = file.id;
+        return {
+            path: file.path,
+            summary: file.summary || '(No summary available)',
+            functions: allFunctionsMap[fileId]?.map(fn => fn.name) || [],
+        };
+    });
+    // 🟩 STEP 6: Generate file tree
     let fileTree = '';
     try {
-        fileTree = generateFileTree(INDEX_DIR, 2); // Limit depth
+        fileTree = generateFileTree(INDEX_DIR, 2);
     }
     catch (e) {
-        console.warn('⚠️ Failed to generate file tree:', e);
+        console.warn('⚠️ Could not generate file tree:', e);
     }
-    // Now we can build the prompt with summaries included for each file
+    // 🟩 STEP 7: Build prompt
     const promptContent = buildContextualPrompt({
         baseInstruction: query,
-        code: '', // No specific code selected
-        relatedFiles, // This now includes both path and summary for each file
+        code,
+        summary: topSummary,
+        functions: topFunctions,
+        relatedFiles,
         projectFileTree: fileTree || undefined,
     });
-    // 🧠 Step 4: Log prompt to file
+    // 🟩 STEP 8: Save prompt for trace/debug
     try {
         if (!fs.existsSync(SCAI_HOME))
             fs.mkdirSync(SCAI_HOME, { recursive: true });
@@ -97,26 +122,21 @@ export async function runAskCommand(query) {
     catch (err) {
         log('❌ Failed to write prompt log:', err);
     }
-    // 🧠 Step 5: Call the model
+    // 🟩 STEP 9: Ask model
     try {
-        console.log('🤖 Asking the model...');
-        // Create a more structured PromptInput object
+        console.log('\n🤖 Asking the model...');
         const input = {
-            content: query, // Main instruction (the query)
-            filepath: topRankedFile?.path || '', // Include the path of the top-ranked file
-            metadata: {
-                summary: topRankedFile?.summary || '', // Add summary of the top-ranked file
-                relatedFiles: relatedFiles, // Pass related files as part of metadata
-            },
-            projectFileTree: fileTree || '' // Include file structure in metadata
+            content: promptContent,
+            filepath,
         };
         const modelResponse = await generate(input, 'llama3');
-        console.log(`\n📝 Model response:\n${modelResponse.content}`);
+        console.log(`\n🧠 Model Response:\n${modelResponse.content}`);
     }
     catch (err) {
         console.error('❌ Model request failed:', err);
     }
 }
+// 🟩 Helper: Prompt once
 function promptOnce(promptText) {
     return new Promise(resolve => {
         const rl = readline.createInterface({

package/dist/daemon/daemonWorker.js

@@ -1,27 +1,45 @@
-import fsSync from 'fs';
-import { LOG_PATH, PID_PATH, SCAI_HOME } from '../constants.js';
+import { db } from '../db/client.js';
+import { runDaemonBatch } from './daemonBatch.js'; // assuming this function is already defined
 import { log } from '../utils/log.js';
-import { runDaemonBatch } from '../daemon/daemonBatch.js'; // ✅ now from utils
-const SLEEP_MS = 30 * 1000;
-const IDLE_SLEEP_MS = 4 * SLEEP_MS;
-function sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
+// Time between each batch in milliseconds
+const SLEEP_MS = 2000; // Adjust as needed
+const IDLE_SLEEP_MS = 5000; // Adjust as needed
+// Check if there are any files left to process
+async function isQueueEmpty() {
+    // Query the database for the count of files with certain processing statuses
+    const row = db.prepare(`
+    SELECT COUNT(*) AS count
+    FROM files
+    WHERE processing_status IN ('unprocessed')
+  `).get();
+    // Cast the row to an object that has a `count` property of type number
+    const castRow = row;
+    // Check if the casted `row` has a valid `count` property (number)
+    if (typeof castRow.count !== 'number') {
+        console.error('Error: Invalid count value in the database query result.');
+        return true; // Assume queue is empty if the count is invalid
+    }
+    // Return true if count is 0, otherwise false
+    return castRow.count === 0;
 }
-async function runDaemonScheduler() {
-    fsSync.mkdirSync(SCAI_HOME, { recursive: true });
-    fsSync.writeFileSync(PID_PATH, process.pid.toString(), 'utf-8');
-    fsSync.appendFileSync(LOG_PATH, `\n\n🧠 Daemon started at ${new Date().toISOString()} — PID ${process.pid}\n`);
-    let cycles = 0;
+export async function daemonWorker() {
     while (true) {
+        // Execute a batch job
         const didWork = await runDaemonBatch();
-        cycles++;
-        if (cycles % 20 === 0) {
-            log(`🌀 Still running. Cycles: ${cycles}`);
+        if (!didWork) {
+            // Check if the queue is empty after a batch job
+            const queueEmpty = await isQueueEmpty();
+            if (queueEmpty) {
+                // If no files are left to process, stop the daemon
+                log("✅ No more work left. Stopping daemon.");
+                break;
+            }
         }
+        // Sleep for a set amount of time before checking again
         await sleep(didWork ? SLEEP_MS : IDLE_SLEEP_MS);
     }
 }
-runDaemonScheduler().catch(err => {
-    log(`❌ Daemon crashed: ${err instanceof Error ? err.message : String(err)}`);
-    process.exit(1);
-});
+// Sleep function to control how often the worker checks
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}

package/dist/db/fileIndex.js

@@ -5,18 +5,17 @@ import { generateEmbedding } from '../lib/generateEmbedding.js';
 import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
 import * as sqlTemplates from './sqlTemplates.js';
 /**
- * Index a file into the local SQLite database.
+ * 📄 Index a single file into the database.
  *
- * - Normalizes the file path for cross-platform compatibility.
- * - Extracts file metadata (last modified time).
- * - Performs an UPSERT into the `files` table with the latest summary/type/timestamp.
+ * - Normalizes the file path to prevent OS-specific path bugs.
+ * - Stores metadata like summary, type, lastModified, etc.
+ * - Sets `embedding` to null initially — computed later.
  */
 export function indexFile(filePath, summary, type) {
     const stats = fs.statSync(filePath);
     const lastModified = stats.mtime.toISOString();
     const indexedAt = new Date().toISOString();
     const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
-    // Index the file metadata
     db.prepare(sqlTemplates.upsertFileTemplate).run({
         path: normalizedPath,
         summary,
@@ -28,7 +27,13 @@ export function indexFile(filePath, summary, type) {
     console.log(`📄 Indexed: ${normalizedPath}`);
 }
 /**
- * Perform a raw keyword-based full-text search using the FTS5 index.
+ * 🔍 FTS5 keyword-based search using SQLite's full-text index.
+ *
+ * Use this when you want:
+ * - A simple keyword search.
+ * - Fast fallback search that doesn't rely on embeddings.
+ *
+ * Returns a full `FileRow[]` (all known metadata, but no similarity score).
  */
 export function queryFiles(safeQuery, limit = 10) {
     console.log(`Executing search query: ${safeQuery}`);
@@ -42,7 +47,16 @@ export function queryFiles(safeQuery, limit = 10) {
     return results;
 }
 /**
- * Perform a hybrid semantic + keyword-based search.
+ * 🧠 Combined semantic + FTS search (Hybrid).
+ *
+ * 1. Convert user query to embedding vector using OpenAI / Ollama etc.
+ * 2. Perform a keyword-based FTS match to limit candidates.
+ * 3. For each candidate, compute cosine similarity between query + file embedding.
+ * 4. Blend the BM25 score and cosine similarity to produce a final score.
+ * 5. Return top K most relevant results.
+ *
+ * ⚠️ This returns a **lighter-weight type** than `FileRow` — doesn't include id, timestamps, etc.
+ * Use a wrapper type like `SearchResultWithScore` in calling code if you need both.
  */
 export async function searchFiles(query, topK = 5) {
     console.log(`🧠 Searching for query: "${query}"`);
@@ -53,6 +67,7 @@ export async function searchFiles(query, topK = 5) {
     }
     const safeQuery = sanitizeQueryForFts(query);
     console.log(`Executing search query in FTS5: ${safeQuery}`);
+    // Step 1: Narrow candidate set using fast keyword match
     const ftsResults = db.prepare(`
     SELECT fts.rowid AS id, f.path, f.summary, f.type, bm25(files_fts) AS bm25Score, f.embedding
     FROM files f
@@ -65,6 +80,7 @@ export async function searchFiles(query, topK = 5) {
     if (ftsResults.length === 0) {
         return [];
     }
+    // Step 2: Compute score based on embedding similarity + BM25 score
     const bm25Min = Math.min(...ftsResults.map(r => r.bm25Score));
     const bm25Max = Math.max(...ftsResults.map(r => r.bm25Score));
     const scored = ftsResults.map(result => {
@@ -79,13 +95,15 @@ export async function searchFiles(query, topK = 5) {
             }
             catch (err) {
                 console.error(`❌ Failed to parse embedding for ${result.path}:`, err);
-                finalScore = 0; // fallback
+                finalScore = 0;
             }
         }
         else {
+            // Fallback to BM25-only score
             finalScore = 1 - ((result.bm25Score - bm25Min) / (bm25Max - bm25Min + 1e-5));
         }
         return {
+            id: result.id, // Ensure the id is included here
             path: result.path,
             summary: result.summary,
             score: finalScore,
@@ -99,9 +117,41 @@ export async function searchFiles(query, topK = 5) {
     console.log(`Returning top ${topK} results based on combined score`);
     return sorted;
 }
+/**
+ * 🔢 Cosine similarity between two vectors.
+ */
 function cosineSimilarity(a, b) {
     const dot = a.reduce((sum, ai, i) => sum + ai * b[i], 0);
     const magA = Math.sqrt(a.reduce((sum, ai) => sum + ai * ai, 0));
     const magB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));
     return dot / (magA * magB);
 }
+/**
+ * 🧠 Get all extracted functions for a given list of file IDs.
+ *
+ * - Used in `askCmd.ts` to fetch code snippets from relevant files.
+ * - Grouped by file_id.
+ */
+export function getFunctionsForFiles(fileIds) {
+    if (!fileIds.length)
+        return {};
+    const placeholders = fileIds.map(() => '?').join(',');
+    const stmt = db.prepare(`
+    SELECT f.file_id, f.name, f.start_line, f.end_line, f.content
+    FROM functions f
+    WHERE f.file_id IN (${placeholders})
+  `);
+    const rows = stmt.all(...fileIds);
+    const grouped = {};
+    for (const row of rows) {
+        if (!grouped[row.file_id])
+            grouped[row.file_id] = [];
+        grouped[row.file_id].push({
+            name: row.name,
+            start_line: row.start_line,
+            end_line: row.end_line,
+            content: row.content,
+        });
+    }
+    return grouped;
+}

package/dist/db/functionExtractors/extractFromJs.js

@@ -1,14 +1,22 @@
 import { parse } from 'acorn';
-import { simple as walkSimple } from 'acorn-walk';
+import { ancestor as walkAncestor } from 'acorn-walk';
 import { generateEmbedding } from '../../lib/generateEmbedding.js';
 import { db } from '../client.js';
 import path from 'path';
 import { log } from '../../utils/log.js';
-/**
- * Parses a JavaScript/TypeScript file, extracts all top-level functions,
- * generates embeddings, and indexes both the functions and any calls made
- * within each function into the database.
- */
+function getFunctionName(node, parent, fileName) {
+    if (node.id?.name)
+        return node.id.name;
+    if (parent?.type === 'VariableDeclarator' && parent.id?.name)
+        return parent.id.name;
+    if (parent?.type === 'Property' && parent.key?.name)
+        return parent.key.name;
+    if (parent?.type === 'AssignmentExpression' && parent.left?.name)
+        return parent.left.name;
+    if (parent?.type === 'MethodDefinition' && parent.key?.name)
+        return parent.key.name;
+    return `${fileName}:<anon>`;
+}
 export async function extractFromJS(filePath, content, fileId) {
     const ast = parse(content, {
         ecmaVersion: 'latest',
@@ -16,27 +24,36 @@ export async function extractFromJS(filePath, content, fileId) {
         locations: true,
     });
     const functions = [];
-    walkSimple(ast, {
-        FunctionDeclaration(node) {
-            const name = node.id?.name || `${path.basename(filePath)}:<anon>`;
-            const start_line = node?.loc?.start.line ?? -1;
-            const end_line = node?.loc?.end.line ?? -1;
-            const body = content.slice(node.start, node.end);
-            functions.push({ name, start_line, end_line, body });
+    walkAncestor(ast, {
+        FunctionDeclaration(node, ancestors) {
+            const parent = ancestors[ancestors.length - 2];
+            const name = getFunctionName(node, parent, path.basename(filePath));
+            functions.push({
+                name,
+                start_line: node.loc?.start.line ?? -1,
+                end_line: node.loc?.end.line ?? -1,
+                content: content.slice(node.start, node.end),
+            });
         },
-        FunctionExpression(node) {
-            const name = `${path.basename(filePath)}:<anon>`;
-            const start_line = node?.loc?.start.line ?? -1;
-            const end_line = node?.loc?.end.line ?? -1;
-            const body = content.slice(node.start, node.end);
-            functions.push({ name, start_line, end_line, body });
+        FunctionExpression(node, ancestors) {
+            const parent = ancestors[ancestors.length - 2];
+            const name = getFunctionName(node, parent, path.basename(filePath));
+            functions.push({
+                name,
+                start_line: node.loc?.start.line ?? -1,
+                end_line: node.loc?.end.line ?? -1,
+                content: content.slice(node.start, node.end),
+            });
         },
-        ArrowFunctionExpression(node) {
-            const name = `${path.basename(filePath)}:<anon>`;
-            const start_line = node?.loc?.start.line ?? -1;
-            const end_line = node?.loc?.end.line ?? -1;
-            const body = content.slice(node.start, node.end);
-            functions.push({ name, start_line, end_line, body });
+        ArrowFunctionExpression(node, ancestors) {
+            const parent = ancestors[ancestors.length - 2];
+            const name = getFunctionName(node, parent, path.basename(filePath));
+            functions.push({
+                name,
+                start_line: node.loc?.start.line ?? -1,
+                end_line: node.loc?.end.line ?? -1,
+                content: content.slice(node.start, node.end),
+            });
         },
     });
     if (functions.length === 0) {
@@ -45,7 +62,7 @@ export async function extractFromJS(filePath, content, fileId) {
     }
     log(`🔍 Found ${functions.length} functions in ${filePath}`);
     for (const fn of functions) {
-        const embedding = await generateEmbedding(fn.body);
+        const embedding = await generateEmbedding(fn.content);
         const result = db.prepare(`
       INSERT INTO functions (
         file_id, name, start_line, end_line, content, embedding, lang
@@ -57,20 +74,20 @@ export async function extractFromJS(filePath, content, fileId) {
             name: fn.name,
             start_line: fn.start_line,
             end_line: fn.end_line,
-            content: fn.body,
+            content: fn.content,
             embedding: JSON.stringify(embedding),
             lang: 'js'
         });
         const callerId = result.lastInsertRowid;
-        const fnAst = parse(content, {
+        const fnAst = parse(fn.content, {
             ecmaVersion: 'latest',
             sourceType: 'module',
             locations: true,
         });
         const calls = [];
-        walkSimple(fnAst, {
+        walkAncestor(fnAst, {
             CallExpression(node) {
-                if (node.callee.type === 'Identifier' && node.callee.name) {
+                if (node.callee?.type === 'Identifier' && node.callee.name) {
                     calls.push({ calleeName: node.callee.name });
                 }
             }
@@ -86,7 +103,6 @@ export async function extractFromJS(filePath, content, fileId) {
         }
         log(`📌 Indexed function: ${fn.name} with ${calls.length} calls`);
     }
-    // ✅ Mark as extracted using new processing_status column
     db.prepare(`
     UPDATE files 
     SET processing_status = 'extracted'

package/dist/db/schema.js

@@ -46,6 +46,8 @@ export function initSchema() {
       lang TEXT
     );
 
+    CREATE INDEX IF NOT EXISTS idx_file_id ON functions(file_id);
+
     CREATE TABLE IF NOT EXISTS function_calls (
       caller_id INTEGER REFERENCES functions(id),
       callee_name TEXT

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.43",
+  "version": "0.1.45",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"