scai 0.1.23 → 0.1.25

package/dist/commands/AskCmd.js

@@ -1,66 +1,63 @@
+import readline from 'readline';
 import { searchFiles } from "../db/fileIndex.js";
 import { generate } from "../lib/generate.js";
-import { summaryModule } from "../pipeline/modules/summaryModule.js";
 export async function runAskCommand(query) {
     if (!query) {
-        console.error('❌ Please provide a search query.\n👉 Usage: scai ask "keyword"');
+        query = await promptOnce('🧠 Ask your question:\n> ');
+    }
+    query = query.trim();
+    if (!query) {
+        console.error('❌ No question provided.\n👉 Usage: scai ask "your question"');
         return;
     }
     console.log(`🔍 Searching for: "${query}"\n`);
-    // Use vector-based search
-    const results = await searchFiles(query, 5); // Or 3 if you want fewer
-    if (results.length === 0) {
-        console.log('⚠️ No similar embeddings found. Asking the model for context instead...');
-    }
-    else {
+    const start = Date.now();
+    const results = await searchFiles(query, 5);
+    const duration = Date.now() - start;
+    console.log(`⏱️ searchFiles took ${duration}ms and returned ${results.length} result(s)`);
+    if (results.length > 0) {
         console.log('📊 Closest files based on semantic similarity:');
-        results.forEach(file => {
-            console.log(`📄 Path: ${file?.path}`);
+        results.forEach((file, i) => {
+            console.log(`  ${i + 1}. 📄 Path: ${file?.path} | Score: ${file?.score?.toFixed(3)}`);
         });
     }
+    else {
+        console.log('⚠️ No similar embeddings found. Asking the model for context instead...');
+    }
+    // 🧠 Use stored summaries directly
     let allSummaries = '';
     for (const file of results) {
-        try {
-            if (!file?.summary) {
-                console.warn(`⚠️ No summary available for file: ${file?.path}`);
-                continue;
-            }
-            console.log(`📝 Using cached summary for file: ${file?.path}`);
-            const summaryResponse = await summaryModule.run({ content: file?.summary ? file.summary : '', filepath: file?.path });
-            if (summaryResponse.summary) {
-                allSummaries += `\n${summaryResponse.summary}`;
-            }
-        }
-        catch (err) {
-            console.error(`❌ Error processing file: ${file?.path}`, err instanceof Error ? err.message : err);
+        if (!file?.summary) {
+            console.warn(`⚠️ No summary available for file: ${file?.path}`);
+            continue;
         }
+        console.log(`📝 Using stored summary for: ${file.path}`);
+        allSummaries += `\n${file.summary}`;
     }
-    if (allSummaries.trim()) {
-        console.log('🧠 Summaries found, sending them to the model for synthesis...');
-        try {
-            const input = {
-                content: `${query}\n\n${allSummaries}`,
-                filepath: '',
-            };
-            const modelResponse = await generate(input, 'llama3');
-            console.log(`\n📝 Model response:\n${modelResponse.content}`);
-        }
-        catch (err) {
-            console.error('❌ Model request failed:', err);
-        }
+    const input = {
+        content: allSummaries ? `${query}\n\n${allSummaries}` : query,
+        filepath: '',
+    };
+    try {
+        console.log(allSummaries.trim()
+            ? '🧠 Summaries found, sending them to the model for synthesis...'
+            : '⚠️ No summaries found. Asking the model for context only...');
+        const modelResponse = await generate(input, 'llama3');
+        console.log(`\n📝 Model response:\n${modelResponse.content}`);
     }
-    else {
-        console.log('⚠️ No summaries found. Asking the model for context only...');
-        try {
-            const input = {
-                content: query,
-                filepath: '',
-            };
-            const modelResponse = await generate(input, 'llama3');
-            console.log(`\n📝 Model response:\n${modelResponse.content}`);
-        }
-        catch (err) {
-            console.error('❌ Model request failed:', err);
-        }
+    catch (err) {
+        console.error('❌ Model request failed:', err);
     }
 }
+function promptOnce(promptText) {
+    return new Promise(resolve => {
+        const rl = readline.createInterface({
+            input: process.stdin,
+            output: process.stdout
+        });
+        rl.question(promptText, answer => {
+            rl.close();
+            resolve(answer.trim());
+        });
+    });
+}

package/dist/commands/DaemonCmd.js

@@ -1,86 +1,32 @@
-import { summaryModule } from '../pipeline/modules/summaryModule.js';
-import { db } from '../db/client.js';
-import fs from 'fs/promises';
 import fsSync from 'fs';
+import { LOG_PATH, PID_PATH } from '../constants.js';
+import { log } from '../utils/log.js';
+import { spawn } from 'child_process';
+import { fileURLToPath } from 'url';
 import path from 'path';
-import { generateEmbedding } from '../lib/generateEmbedding.js';
-import { IGNORED_EXTENSIONS } from '../config/IgnoredExtensions.js';
-import { PID_PATH, SCAI_HOME } from '../constants.js';
-const LOG_PATH = path.join(SCAI_HOME, 'daemon.log');
-const SLEEP_MS = 30 * 1000; // 💤 Pause between batches
-const IDLE_SLEEP_MS = 4 * SLEEP_MS; // 💤 Longer pause if idle
-const MAX_FILES_PER_BATCH = 5; // 🎛️ Throttle indexing per cycle
-// 💤 Utility
-function sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
-// 🪵 Append to log file
-function log(message) {
-    const timestamp = new Date().toISOString();
-    fsSync.appendFileSync(LOG_PATH, `[${timestamp}] ${message}\n`);
-}
-// ❌ Skip unwanted file types
-function shouldIgnoreFile(filePath) {
-    const ext = path.extname(filePath).toLowerCase();
-    return IGNORED_EXTENSIONS.includes(ext);
-}
-// 🧠 One summarization batch
-export async function runDaemonBatch() {
-    const rows = db.prepare(`
-    SELECT path, type FROM files
-    WHERE summary IS NULL OR summary = ''
-    ORDER BY last_modified DESC
-    LIMIT ?
-  `).all(MAX_FILES_PER_BATCH);
-    if (rows.length === 0) {
-        log('✅ No files left to summarize.');
-        return false; // 💤 Idle
-    }
-    for (const row of rows) {
-        if (!fsSync.existsSync(row.path)) {
-            log(`⚠️ Skipped missing file: ${row.path}`);
-            continue;
-        }
-        if (shouldIgnoreFile(row.path)) {
-            log(`⚠️ Skipped (extension): ${row.path}`);
-            continue;
-        }
+export async function startDaemon() {
+    // If there's a PID file, check if the process is still running
+    if (fsSync.existsSync(PID_PATH)) {
+        const pid = parseInt(fsSync.readFileSync(PID_PATH, 'utf8'));
         try {
-            const content = await fs.readFile(row.path, 'utf-8');
-            const result = await summaryModule.run({ content, filepath: row.path });
-            const summary = result?.summary?.trim() || null;
-            let embedding = null;
-            if (summary) {
-                const vector = await generateEmbedding(summary);
-                if (vector)
-                    embedding = JSON.stringify(vector);
-            }
-            db.prepare(`
-        UPDATE files
-        SET summary = @summary, embedding = @embedding, indexed_at = datetime('now')
-        WHERE path = @path
-      `).run({ summary, embedding, path: row.path });
-            log(`📝 Summarized: ${row.path}`);
-        }
-        catch (err) {
-            log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}`);
+            process.kill(pid, 0); // Check if process exists
+            log(`⚠️ Daemon already running with PID ${pid}.`);
+            return;
         }
-        await sleep(200); // 🧘 Micro delay between each file
-    }
-    return true; // ✅ Work was done
-}
-// 🚀 Daemon loop: runs until killed
-export async function runDaemonScheduler() {
-    fsSync.mkdirSync(SCAI_HOME, { recursive: true });
-    fsSync.writeFileSync(PID_PATH, process.pid.toString(), 'utf-8');
-    fsSync.appendFileSync(LOG_PATH, `\n\n🧠 Daemon started at ${new Date().toISOString()} — PID ${process.pid}\n`);
-    let cycles = 0;
-    while (true) {
-        const didWork = await runDaemonBatch();
-        cycles++;
-        if (cycles % 20 === 0) {
-            log(`🌀 Still running. Cycles: ${cycles}`);
+        catch {
+            log(`⚠️ Stale PID file found. Removing and restarting daemon...`);
+            fsSync.unlinkSync(PID_PATH);
         }
-        await sleep(didWork ? SLEEP_MS : IDLE_SLEEP_MS);
     }
+    log('🚀 Starting summarizer daemon in background mode...');
+    log(`📝 Logs will be saved to: ${LOG_PATH}`);
+    process.env.BACKGROUND_MODE = 'true';
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
+    const daemonWorkerPath = path.join(__dirname, '../daemon/daemonWorker.js');
+    const child = spawn(process.execPath, [daemonWorkerPath], {
+        detached: true,
+        stdio: ['ignore', 'ignore', 'ignore'],
+    });
+    child.unref();
 }

package/dist/commands/IndexCmd.js

@@ -1,14 +1,26 @@
 import fg from 'fast-glob';
-import fs from 'fs';
 import path from 'path';
 import { initSchema } from '../db/schema.js';
 import { indexFile } from '../db/fileIndex.js';
 import { shouldIgnoreFile } from '../utils/shouldIgnoreFiles.js';
 import { detectFileType } from '../utils/detectFileType.js';
-import { runDaemonScheduler } from './DaemonCmd.js';
+import { startDaemon } from './DaemonCmd.js';
 import { IGNORED_FOLDER_GLOBS } from '../config/IgnoredPaths.js';
 import { Config } from '../config.js';
-import { PID_PATH } from '../constants.js';
+import { DB_PATH } from '../constants.js';
+import { log } from '../utils/log.js';
+import lockfile from 'proper-lockfile';
+// 🧠 Lock the database to prevent simultaneous access
+async function lockDb() {
+    try {
+        const lock = await lockfile.lock(DB_PATH); // DB_PATH from constants.ts
+        return lock;
+    }
+    catch (err) {
+        log('❌ Failed to acquire DB lock: ' + err);
+        throw err;
+    }
+}
 export async function runIndexCommand(targetDir, options = {}) {
     try {
         initSchema();
@@ -33,7 +45,7 @@ export async function runIndexCommand(targetDir, options = {}) {
         resolvedDir = Config.getIndexDir() || process.cwd();
         Config.setIndexDir(resolvedDir); // persist if not yet saved
     }
-    console.log(`📂 Indexing files in: ${resolvedDir}`);
+    log(`📂 Indexing files in: ${resolvedDir}`);
     const files = await fg('**/*.*', {
         cwd: resolvedDir,
         ignore: IGNORED_FOLDER_GLOBS,
@@ -41,6 +53,7 @@ export async function runIndexCommand(targetDir, options = {}) {
     });
     const countByExt = {};
     let count = 0;
+    const release = await lockDb(); // Lock the DB before starting
     for (const file of files) {
         if (shouldIgnoreFile(file))
             continue;
@@ -49,22 +62,16 @@ export async function runIndexCommand(targetDir, options = {}) {
             indexFile(file, null, type); // Index file without summary
             const ext = path.extname(file);
             countByExt[ext] = (countByExt[ext] || 0) + 1;
-            console.log(`📄 Indexed: ${path.relative(resolvedDir, file)}`);
+            log(`📄 Indexed: ${path.relative(resolvedDir, file)}`);
             count++;
         }
         catch (err) {
-            console.warn(`⚠️ Skipped in indexCmd ${file}:`, err instanceof Error ? err.message : err);
-        }
-    }
-    console.log('📊 Indexed files by extension:', countByExt);
-    console.log(`✅ Done. Indexed ${count} files.`);
-    if (options.detached) {
-        if (fs.existsSync(PID_PATH)) {
-            console.warn(`⚠️ Daemon already running (PID file found at ${PID_PATH}). Skipping launch.`);
-        }
-        else {
-            console.log('🚀 Starting summarizer daemon in background mode...');
-            runDaemonScheduler();
+            log(`⚠️ Skipped in indexCmd ${file}: ${err instanceof Error ? err.message : err}`);
         }
     }
+    log('📊 Indexed files by extension:', JSON.stringify(countByExt, null, 2));
+    log(`✅ Done. Indexed ${count} files.`);
+    await release(); // Release the DB lock after indexing is done
+    // Auto-start daemon if not already running
+    startDaemon();
 }

package/dist/commands/ResetDbCmd.js

@@ -1,13 +1,52 @@
 import fs from 'fs';
+import fsp from 'fs/promises';
 import { db } from '../db/client.js';
-import { DB_PATH } from '../constants.js';
-export function resetDatabase() {
+import { DB_PATH, SCAI_HOME } from '../constants.js';
+import lockfile from 'proper-lockfile';
+import path from 'path';
+function getBackupDir() {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    return path.join(SCAI_HOME, `backup-${timestamp}`);
+}
+async function backupScaiFolder() {
+    const backupDir = getBackupDir();
+    try {
+        await fsp.mkdir(backupDir, { recursive: true });
+        const files = await fsp.readdir(SCAI_HOME);
+        for (const file of files) {
+            const srcPath = path.join(SCAI_HOME, file);
+            const destPath = path.join(backupDir, file);
+            const stat = await fsp.stat(srcPath);
+            if (stat.isFile()) {
+                await fsp.copyFile(srcPath, destPath);
+            }
+        }
+        console.log(`📦 Backed up .scai folder to: ${backupDir}`);
+        return backupDir;
+    }
+    catch (err) {
+        console.warn('⚠️ Failed to back up .scai folder:', err instanceof Error ? err.message : err);
+        return null;
+    }
+}
+export async function resetDatabase() {
+    console.log('🔁 Backing up existing .scai folder...');
+    await backupScaiFolder();
     try {
-        db.close(); // 🔒 Ensure the DB connection is closed
+        db.close();
         console.log('🔒 Closed SQLite database connection.');
     }
     catch (err) {
-        console.warn('⚠️ Could not close database:', err);
+        console.warn('⚠️ Could not close database:', err instanceof Error ? err.message : err);
+    }
+    try {
+        const releaseLock = await lockfile.unlock(DB_PATH).catch(() => null);
+        if (releaseLock) {
+            console.log('🔓 Released database lock.');
+        }
+    }
+    catch (err) {
+        console.warn('⚠️ Failed to release database lock:', err instanceof Error ? err.message : err);
     }
     if (fs.existsSync(DB_PATH)) {
         try {
@@ -22,5 +61,12 @@ export function resetDatabase() {
     else {
         console.log('ℹ️ No existing database found at:', DB_PATH);
     }
+    try {
+        fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
+        console.log('📁 Ensured that the database directory exists.');
+    }
+    catch (err) {
+        console.warn('⚠️ Could not ensure DB directory exists:', err instanceof Error ? err.message : err);
+    }
     console.log('✅ Database has been reset. You can now re-run: scai index');
 }

package/dist/commands/StopDaemonCmd.js

@@ -9,15 +9,26 @@ export async function runStopDaemonCommand() {
     }
     const pid = parseInt(fs.readFileSync(PID_PATH, 'utf-8'), 10);
     if (isNaN(pid)) {
-        console.error('⚠️ Invalid PID file.');
+        console.error('⚠️ Invalid PID file. Removing it...');
+        fs.unlinkSync(PID_PATH);
+        return;
+    }
+    try {
+        // Check if process exists
+        process.kill(pid, 0);
+    }
+    catch {
+        console.warn(`⚠️ No running process with PID ${pid}. Removing stale PID file.`);
+        fs.unlinkSync(PID_PATH);
         return;
     }
     try {
-        process.kill(pid);
+        // Attempt to terminate the process
+        process.kill(pid, 'SIGTERM');
         fs.unlinkSync(PID_PATH);
         console.log(`✅ Daemon process ${pid} stopped.`);
     }
     catch (err) {
-        console.error(`❌ Failed to stop process ${pid}:`, err instanceof Error ? err.message : err);
+        console.error(`❌ Failed to stop daemon process ${pid}:`, err instanceof Error ? err.message : err);
     }
 }

package/dist/config/IgnoredExtensions.js

@@ -1,4 +1,3 @@
-// src/config/IgnoredExtensions.ts
 export const IGNORED_EXTENSIONS = [
     // 🖼 Media
     '.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico',
@@ -43,4 +42,11 @@ export const IGNORED_EXTENSIONS = [
     '.sap', '.sappkg', '.qbw', '.qbb',
     // 🔒 Lock files (but NOT lock *configs*)
     '.lck', '.lockfile', '.db-lock', '.pid', '.socket',
+    // ⚠️ Added file types that are only kept by exception
+    '.xml', // Only specific XML files like pom.xml should be kept
+    '.json', // Kept only if exact filename is in exceptions
+    '.yaml', // Kept only if filename is explicitly whitelisted
+    '.yml',
+    '.md',
+    '.txt',
 ];

package/dist/config/StopWords.js

@@ -0,0 +1,28 @@
+/**
+ * These common words are ignored from search queries
+ * to reduce noise and improve FTS and embedding match quality.
+ */
+export const STOP_WORDS = new Set([
+    // Articles & conjunctions
+    'a', 'an', 'and', 'but', 'or', 'nor',
+    // Prepositions
+    'at', 'by', 'for', 'from', 'in', 'into', 'of', 'on', 'to', 'with', 'about', 'above', 'below', 'under', 'over', 'through',
+    // Pronouns
+    'i', 'me', 'my', 'mine', 'you', 'your', 'yours', 'he', 'him', 'his', 'she', 'her', 'hers', 'it', 'its',
+    'we', 'us', 'our', 'ours', 'they', 'them', 'their', 'theirs',
+    // Determiners
+    'this', 'that', 'these', 'those', 'some', 'any', 'each', 'every', 'either', 'neither',
+    // Auxiliary and modal verbs
+    'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+    'do', 'does', 'did',
+    'have', 'has', 'had',
+    'can', 'could', 'shall', 'should', 'will', 'would', 'may', 'might', 'must',
+    // Wh-words and generic question words
+    'what', 'which', 'who', 'whom', 'whose', 'where', 'when', 'why', 'how',
+    // Misc common functional words
+    'as', 'if', 'than', 'then', 'there', 'because', 'so', 'just', 'only', 'not', 'no',
+    // Very common verbs to strip from noisy queries
+    'use', 'get', 'make', 'need', 'want', 'let', 'help', 'work', 'see', 'look', 'like', 'know',
+    // Other frequent noise
+    'all', 'more', 'most', 'many', 'much', 'such', 'also', 'again'
+]);

package/dist/constants.js

@@ -21,6 +21,11 @@ export const PID_PATH = path.join(SCAI_HOME, 'daemon.pid');
  * ~/.scai/config.json
  */
 export const CONFIG_PATH = path.join(SCAI_HOME, 'config.json');
+/**
+ * Path to the daemon log file:
+ * ~/.scai/daemon.log
+ */
+export const LOG_PATH = path.join(SCAI_HOME, 'daemon.log');
 /**
  * Get the active index directory.
  *

package/dist/daemon/daemonBatch.js

@@ -0,0 +1,65 @@
+import { summaryModule } from '../pipeline/modules/summaryModule.js';
+import { db } from '../db/client.js';
+import fs from 'fs/promises';
+import fsSync from 'fs';
+import { generateEmbedding } from '../lib/generateEmbedding.js';
+import { DB_PATH } from '../constants.js';
+import { log } from '../utils/log.js';
+import lockfile from 'proper-lockfile';
+import { shouldIgnoreFile } from '../utils/shouldIgnoreFiles.js';
+const MAX_FILES_PER_BATCH = 5;
+async function lockDb() {
+    try {
+        return await lockfile.lock(DB_PATH);
+    }
+    catch (err) {
+        log('❌ Failed to acquire DB lock: ' + err);
+        throw err;
+    }
+}
+export async function runDaemonBatch() {
+    const rows = db.prepare(`
+    SELECT path, type FROM files
+    WHERE summary IS NULL OR summary = ''
+    ORDER BY last_modified DESC
+    LIMIT ?
+  `).all(MAX_FILES_PER_BATCH);
+    if (rows.length === 0) {
+        log('✅ No files left to summarize.');
+        return false;
+    }
+    const release = await lockDb();
+    for (const row of rows) {
+        if (!fsSync.existsSync(row.path)) {
+            log(`⚠️ Skipped missing file: ${row.path}`);
+            continue;
+        }
+        if (shouldIgnoreFile(row.path)) {
+            log(`⚠️ Skipped (extension): ${row.path}`);
+            continue;
+        }
+        try {
+            const content = await fs.readFile(row.path, 'utf-8');
+            const result = await summaryModule.run({ content, filepath: row.path });
+            const summary = result?.summary?.trim() || null;
+            let embedding = null;
+            if (summary) {
+                const vector = await generateEmbedding(summary);
+                if (vector)
+                    embedding = JSON.stringify(vector);
+            }
+            db.prepare(`
+        UPDATE files
+        SET summary = @summary, embedding = @embedding, indexed_at = datetime('now')
+        WHERE path = @path
+      `).run({ summary, embedding, path: row.path });
+            log(`📝 Summarized: ${row.path}`);
+        }
+        catch (err) {
+            log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        await new Promise(resolve => setTimeout(resolve, 200));
+    }
+    await release();
+    return true;
+}

package/dist/daemon/daemonWorker.js

@@ -0,0 +1,27 @@
+import fsSync from 'fs';
+import { LOG_PATH, PID_PATH, SCAI_HOME } from '../constants.js';
+import { log } from '../utils/log.js';
+import { runDaemonBatch } from '../daemon/daemonBatch.js'; // ✅ now from utils
+const SLEEP_MS = 30 * 1000;
+const IDLE_SLEEP_MS = 4 * SLEEP_MS;
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+async function runDaemonScheduler() {
+    fsSync.mkdirSync(SCAI_HOME, { recursive: true });
+    fsSync.writeFileSync(PID_PATH, process.pid.toString(), 'utf-8');
+    fsSync.appendFileSync(LOG_PATH, `\n\n🧠 Daemon started at ${new Date().toISOString()} — PID ${process.pid}\n`);
+    let cycles = 0;
+    while (true) {
+        const didWork = await runDaemonBatch();
+        cycles++;
+        if (cycles % 20 === 0) {
+            log(`🌀 Still running. Cycles: ${cycles}`);
+        }
+        await sleep(didWork ? SLEEP_MS : IDLE_SLEEP_MS);
+    }
+}
+runDaemonScheduler().catch(err => {
+    log(`❌ Daemon crashed: ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+});

package/dist/db/client.js

@@ -1,5 +1,9 @@
 import Database from 'better-sqlite3';
 import fs from 'fs';
 import { DB_PATH, SCAI_HOME } from '../constants.js';
+// Ensure the directory exists
 fs.mkdirSync(SCAI_HOME, { recursive: true });
+// Open the database connection
 export const db = new Database(DB_PATH);
+// Set journal_mode to WAL for better concurrency
+db.pragma('journal_mode = WAL');

package/dist/db/fileIndex.js

@@ -1,18 +1,15 @@
 import { db } from './client.js';
 import fs from 'fs';
+import path from 'path';
 import { generateEmbedding } from '../lib/generateEmbedding.js';
 import * as sqlTemplates from './sqlTemplates.js';
-import path from 'path';
+import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
 /**
  * Index a file into the local SQLite database.
  *
  * - Normalizes the file path for cross-platform compatibility.
  * - Extracts file metadata (last modified time).
  * - Performs an UPSERT into the `files` table with the latest summary/type/timestamp.
- *
- * @param filePath - Absolute path to the file being indexed
- * @param summary - Optional summary of the file content
- * @param type - File type or extension (e.g., 'md', 'ts')
  */
 export function indexFile(filePath, summary, type) {
     const stats = fs.statSync(filePath);
@@ -25,18 +22,12 @@ export function indexFile(filePath, summary, type) {
         type,
         lastModified,
         indexedAt,
+        embedding: null
     });
     console.log(`📄 Indexed: ${normalizedPath}`);
 }
 /**
  * Perform a raw keyword-based full-text search using the FTS5 index.
- *
- * - Tokenizes and sanitizes the input query string.
- * - Performs a ranked search using BM25 scoring via the virtual FTS table.
- * - Returns basic file metadata along with rank for ordering.
- *
- * @param query - The search query string (e.g., "api router config")
- * @param limit - Max number of results to return (default: 10)
  */
 export function queryFiles(query, limit = 10) {
     const safeQuery = query
@@ -48,50 +39,59 @@ export function queryFiles(query, limit = 10) {
     })
         .join(' OR ');
     console.log(`Executing search query: ${safeQuery}`);
-    const results = db.prepare(sqlTemplates.rawQueryTemplate).all({
-        query: safeQuery,
-        limit
+    const results = db.prepare(`
+    SELECT f.id, f.path, f.summary, f.type, f.last_modified, f.indexed_at
+    FROM files f
+    JOIN files_fts fts ON f.id = fts.rowid
+    WHERE fts.files_fts MATCH ?
+    LIMIT ?
+  `).all(safeQuery, limit);
+    console.log(`Search returned ${results.length} results.`);
+    results.forEach(result => {
+        console.log(`📄 Found in FTS search: ${result.path}`);
     });
     return results;
 }
-export function cosineSimilarity(a, b) {
-    const dot = a.reduce((sum, ai, i) => sum + ai * b[i], 0);
-    const magA = Math.sqrt(a.reduce((sum, ai) => sum + ai * ai, 0));
-    const magB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));
-    return dot / (magA * magB);
-}
 /**
  * Perform a hybrid semantic + keyword-based search.
- *
- * - Generates a vector embedding of the query.
- * - Runs an FTS search using BM25 ranking.
- * - Looks up file embeddings from the database and compares using cosine similarity.
- * - Combines similarity score and BM25 rank into a weighted final score.
- *
- * @param query - Natural language search query
- * @param topK - Max number of top-ranked results to return (default: 5)
  */
 export async function searchFiles(query, topK = 5) {
+    console.log(`🧠 Searching for query: "${query}"`);
     const embedding = await generateEmbedding(query);
-    if (!embedding)
+    if (!embedding) {
+        console.log('⚠️ Failed to generate embedding for query');
         return [];
-    const safeQuery = query
-        .trim()
-        .split(/\s+/)
-        .map(token => {
-        token = token.replace(/[?*\\"]/g, '').replace(/'/g, "''");
-        return token.includes(' ') ? `"${token}"` : `${token}*`;
-    })
-        .join(' OR ');
-    console.log(`Executing search query: ${safeQuery}`);
-    const ftsResults = db.prepare(sqlTemplates.fetchBm25ScoresTemplate).all({ query: safeQuery });
+    }
+    const safeQuery = sanitizeQueryForFts(query);
+    console.log(`Executing search query in FTS5: ${safeQuery}`);
+    const ftsResults = db.prepare(`
+  SELECT fts.rowid AS id, f.path, f.summary, f.type, bm25(files_fts) AS bm25Score
+  FROM files f
+  JOIN files_fts fts ON f.id = fts.rowid
+  WHERE fts.files_fts MATCH ?
+  AND f.embedding IS NOT NULL
+  ORDER BY bm25Score DESC
+  LIMIT ?
+`).all(safeQuery, topK);
+    console.log(`FTS search returned ${ftsResults.length} results`);
+    if (ftsResults.length === 0) {
+        console.log('⚠️ No results found from FTS search');
+        return [];
+    }
+    ftsResults.forEach(result => {
+        console.log(`📄 FTS found: ${result.path}`);
+    });
     const bm25Min = Math.min(...ftsResults.map(r => r.bm25Score));
     const bm25Max = Math.max(...ftsResults.map(r => r.bm25Score));
     const scored = ftsResults.map(result => {
         try {
-            const embResult = db.prepare(sqlTemplates.fetchEmbeddingTemplate).get({ path: result.path });
-            if (!embResult || typeof embResult.embedding !== 'string')
+            const embResult = db.prepare(sqlTemplates.fetchEmbeddingTemplate).get({
+                path: result.path,
+            });
+            if (!embResult || typeof embResult.embedding !== 'string') {
+                console.log(`⚠️ No embedding for file: ${result.path}`);
                 return null;
+            }
             const vector = JSON.parse(embResult.embedding);
             const sim = cosineSimilarity(embedding, vector);
             const normalizedBm25 = 1 - ((result.bm25Score - bm25Min) / (bm25Max - bm25Min + 1e-5));
@@ -101,15 +101,23 @@ export async function searchFiles(query, topK = 5) {
                 summary: result.summary,
                 score: finalScore,
                 sim,
-                bm25: normalizedBm25
+                bm25: normalizedBm25,
             };
         }
         catch (err) {
-            console.error(`Error processing embedding for file: ${result.path}`, err);
+            console.error(`❌ Error processing embedding for file: ${result.path}`, err);
             return null;
         }
-    }).filter(Boolean)
+    })
+        .filter((r) => r !== null)
         .sort((a, b) => b.score - a.score)
         .slice(0, topK);
+    console.log(`Returning top ${topK} results based on combined score`);
     return scored;
 }
+function cosineSimilarity(a, b) {
+    const dot = a.reduce((sum, ai, i) => sum + ai * b[i], 0);
+    const magA = Math.sqrt(a.reduce((sum, ai) => sum + ai * ai, 0));
+    const magB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));
+    return dot / (magA * magB);
+}

package/dist/db/schema.js

@@ -1,19 +1,35 @@
 import { db } from "./client.js";
 export function initSchema() {
     db.exec(`
-  CREATE TABLE IF NOT EXISTS files (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  path TEXT UNIQUE,
-  summary TEXT,
-  type TEXT,
-  indexed_at TEXT,
-  last_modified TEXT,
-  embedding TEXT
-);
+    CREATE TABLE IF NOT EXISTS files (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      path TEXT UNIQUE,
+      summary TEXT,
+      type TEXT,
+      indexed_at TEXT,
+      last_modified TEXT,
+      embedding TEXT
+    );
 
-CREATE VIRTUAL TABLE IF NOT EXISTS files_fts
-USING fts5(path, summary, content='files', content_rowid='id');
+    CREATE VIRTUAL TABLE IF NOT EXISTS files_fts
+    USING fts5(path, summary, content='files', content_rowid='id');
 
-`);
-    console.log('✅ SQLite schema initialized');
+    -- FTS Triggers to keep files_fts in sync
+    CREATE TRIGGER IF NOT EXISTS files_ai AFTER INSERT ON files BEGIN
+      INSERT INTO files_fts(rowid, path, summary)
+      VALUES (new.id, new.path, new.summary);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS files_au AFTER UPDATE ON files BEGIN
+      UPDATE files_fts SET
+        path = new.path,
+        summary = new.summary
+      WHERE rowid = new.id;
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS files_ad AFTER DELETE ON files BEGIN
+      DELETE FROM files_fts WHERE rowid = old.id;
+    END;
+  `);
+    console.log('✅ SQLite schema initialized with FTS5 triggers');
 }

package/dist/db/sqlTemplates.js

@@ -1,12 +1,21 @@
 // Upsert file metadata into `files`
 export const upsertFileTemplate = `
-  INSERT INTO files (path, summary, type, last_modified, indexed_at)
-  VALUES (:path, :summary, :type, :lastModified, :indexedAt)
+  INSERT INTO files (path, summary, type, last_modified, indexed_at, embedding)
+  VALUES (:path, :summary, :type, :lastModified, :indexedAt, :embedding)
   ON CONFLICT(path) DO UPDATE SET
-    summary = excluded.summary,
+    summary = CASE
+      WHEN excluded.summary IS NOT NULL AND excluded.summary != files.summary
+      THEN excluded.summary
+      ELSE files.summary
+    END,
     type = excluded.type,
     last_modified = excluded.last_modified,
-    indexed_at = excluded.indexed_at
+    indexed_at = excluded.indexed_at,
+    embedding = CASE
+      WHEN excluded.embedding IS NOT NULL AND excluded.embedding != files.embedding
+      THEN excluded.embedding
+      ELSE files.embedding
+    END
 `;
 // Fetch search results with BM25 ranking
 export const fetchBm25ScoresTemplate = `

package/dist/index.js

@@ -5,9 +5,6 @@ import { Config } from './config.js';
 import { createRequire } from 'module';
 const require = createRequire(import.meta.url);
 const { version } = require('../package.json');
-// 🧠 Commands
-import { checkEnv } from "./commands/EnvCmd.js";
-import { checkGit } from "./commands/GitCmd.js";
 import { suggestCommitMessage } from "./commands/CommitSuggesterCmd.js";
 import { handleRefactor } from "./commands/RefactorCmd.js";
 import { generateTests } from "./commands/TestGenCmd.js";
@@ -18,7 +15,7 @@ import { runModulePipelineFromCLI } from './commands/ModulePipelineCmd.js';
 import { runIndexCommand } from './commands/IndexCmd.js';
 import { resetDatabase } from './commands/ResetDbCmd.js';
 import { runQueryCommand } from './commands/QueryCmd.js';
-import { runDaemonBatch } from './commands/DaemonCmd.js';
+import { startDaemon } from './commands/DaemonCmd.js';
 import { runStopDaemonCommand } from "./commands/StopDaemonCmd.js";
 import { runAskCommand } from './commands/AskCmd.js';
 // 🎛️ CLI Setup
@@ -34,12 +31,14 @@ cmd
     await bootstrap();
     console.log('✅ Model initialization completed!');
 });
+cmd
+    .command('sugg')
+    .description('Suggest a commit message from staged changes')
+    .option('-c, --commit', 'Automatically commit with suggested message')
+    .action(suggestCommitMessage);
 // 🔧 Group: Git-related commands
 const git = cmd.command('git').description('Git utilities');
-git
-    .command('status')
-    .description('Check Git status')
-    .action(checkGit);
+// The sugg command under the 'git' group
 git
     .command('sugg')
     .description('Suggest a commit message from staged changes')
@@ -66,15 +65,6 @@ gen
     .command('tests <file>')
     .description('Generate a Jest test file for the specified JS/TS module')
     .action((file) => generateTests(file));
-// 🔍 Indexing
-cmd
-    .command('index [targetDir]')
-    .description('Index supported files in the given directory (or current folder if none)')
-    .option('-d, --detached', 'Run summarizer daemon after indexing')
-    .option('--force', 'Force indexing even if another folder has already been indexed')
-    .action((targetDir, options) => {
-    runIndexCommand(targetDir, { detached: options.detached, force: options.force });
-});
 // ⚙️ Group: Configuration settings
 const set = cmd.command('set').description('Set configuration values');
 set
@@ -99,35 +89,46 @@ set
     Config.show();
 });
 // 🧪 Diagnostics and info
-cmd
-    .command('env')
-    .description('Check environment variables')
-    .action(checkEnv);
 cmd
     .command('config')
     .description('Show the currently active model and language settings')
     .action(() => {
     Config.show();
 });
+// Add explanation about alpha features directly in the help menu
+cmd.addHelpText('after', `
+🚨 Alpha Features:
+  - The "index", "daemon", "stop-daemon", "reset-db" commands are considered alpha features.
+  - These commands are in active development and may change in the future.
+
+💡 Use with caution and expect possible changes or instability.
+`);
+// 🔍 Indexing
+cmd
+    .command('index [targetDir]')
+    .description('Index supported files in the given directory (or current folder if none)')
+    .option('--force', 'Force indexing even if another folder has already been indexed')
+    .action((targetDir, options) => {
+    runIndexCommand(targetDir, { force: options.force });
+});
 // 🧠 Query and assistant
 cmd
     .command('query <query>')
     .description('Search indexed files by keyword')
     .action(runQueryCommand);
 cmd
-    .command('ask')
-    .description('Ask a question using file summaries and a local model')
-    .argument('<question...>', 'The question to ask')
-    .action((question) => {
-    const q = question.join(' ');
-    runAskCommand(q);
+    .command('ask [question...]') // <- the ... makes it variadic
+    .description('Ask a question based on indexed files')
+    .action((questionParts) => {
+    const fullQuery = questionParts?.join(' ');
+    runAskCommand(fullQuery);
 });
 // 🛠️ Background tasks and maintenance
 cmd
     .command('daemon')
     .description('Run background summarization of indexed files')
     .action(async () => {
-    await runDaemonBatch(); // ignore the return value
+    await startDaemon(); // ignore the return value
 });
 cmd
     .command('stop-daemon')

package/dist/utils/log.js

@@ -0,0 +1,15 @@
+import fs from 'fs';
+import { LOG_PATH } from '../constants.js';
+export function log(...args) {
+    const timestamp = new Date().toISOString();
+    const message = args.map(arg => typeof arg === 'string' ? arg : JSON.stringify(arg, null, 2)).join(' ');
+    const isBackground = process.env.BACKGROUND_MODE === 'true';
+    if (isBackground) {
+        // If running in background, log to a file
+        fs.appendFileSync(LOG_PATH, `[${timestamp}] ${message}\n`);
+    }
+    else {
+        // Otherwise, log to the console
+        console.log(`[${timestamp}] ${message}`);
+    }
+}

package/dist/utils/removeIgnoredFiles.js

@@ -0,0 +1,43 @@
+import Database from 'better-sqlite3';
+import path from 'path';
+import os from 'os';
+import { IGNORED_EXTENSIONS } from '../config/IgnoredExtensions.js';
+import { specificFileExceptions } from './specificFileExceptions.js';
+// THIS FILE IS MEANT TO BE RUN AS A NODE JS SCRIPT. node dist/src/utilsremoveIgnoredFiles.js
+// It removes wrongly indexed files that don't add value to the model. 
+const DB_PATH = path.join(os.homedir(), '.scai', 'db.sqlite');
+const db = new Database(DB_PATH);
+console.log('🧹 Removing files with ignored extensions from the database...');
+// === Remove Files with Ignored Extensions, Excluding Specific Exceptions ===
+IGNORED_EXTENSIONS.forEach(ext => {
+    try {
+        const filesToDelete = db.prepare(`
+      SELECT path FROM files WHERE path LIKE ?
+    `).all(`%${ext}`);
+        ;
+        let deletedCount = 0;
+        filesToDelete.forEach(file => {
+            // Check if the file is in the exception list
+            if (!specificFileExceptions.includes(file.path)) {
+                // Delete the file from the database
+                const deleted = db.prepare(`DELETE FROM files WHERE path = ?`).run(file.path);
+                if (deleted.changes > 0) {
+                    deletedCount++;
+                }
+            }
+            else {
+                console.log(`⚠️ Skipped file (exception): ${file.path}`);
+            }
+        });
+        if (deletedCount > 0) {
+            console.log(`✅ Removed ${deletedCount} files with extension: ${ext}`);
+        }
+        else {
+            console.log(`⚠️ No deletions for files with extension: ${ext}`);
+        }
+    }
+    catch (err) {
+        console.error("❌ Failed to remove files with extension ${ext}:", err instanceof Error ? err.message : err);
+    }
+});
+console.log('🧹 Finished removing ignored files.');

package/dist/utils/sanitizeQuery.js

@@ -0,0 +1,16 @@
+// src/utils/sanitizeQuery.ts
+import { STOP_WORDS } from '../config/StopWords.js';
+export function sanitizeQueryForFts(input) {
+    const tokens = input
+        .trim()
+        .split(/\s+/)
+        .map(token => token.toLowerCase())
+        .filter(token => token.length > 2 &&
+        !STOP_WORDS.has(token) &&
+        /^[a-z0-9]+$/.test(token))
+        .map(token => token.replace(/[?*\\"]/g, '').replace(/'/g, "''") + '*');
+    // 👇 Prevent FTS syntax errors by returning a catch-all query
+    if (tokens.length === 0)
+        return '*';
+    return tokens.join(' OR ');
+}

package/dist/utils/shouldIgnoreFiles.js

@@ -1,6 +1,14 @@
 import path from 'path';
 import { IGNORED_EXTENSIONS } from '../config/IgnoredExtensions.js';
+import { specificFileExceptions } from '../utils/specificFileExceptions.js';
 export function shouldIgnoreFile(filePath) {
+    // Get file extension
     const ext = path.extname(filePath).toLowerCase();
+    // Check if the file is explicitly listed in the exceptions
+    const fileName = path.basename(filePath);
+    if (specificFileExceptions.includes(fileName)) {
+        return false; // Don't ignore if it's in the exceptions list
+    }
+    // If not in exceptions, check against ignored extensions
     return IGNORED_EXTENSIONS.includes(ext);
 }

package/dist/utils/specificFileExceptions.js

@@ -0,0 +1,90 @@
+export const specificFileExceptions = [
+    // 🧑‍💻 Project Configuration Files
+    'package.json', // Keep package.json for NPM/Yarn dependency management
+    'package-lock.json', // Keep package-lock.json for npm lockfile
+    'yarn.lock', // Keep yarn.lock for Yarn dependency lockfile
+    'pnpm-lock.yaml', // Keep pnpm-lock.yaml for pnpm lockfile
+    'tsconfig.json', // Keep TypeScript configuration file
+    'tsconfig.build.json', // Keep build-specific tsconfig file
+    'tsconfig.prod.json', // Keep production-specific tsconfig file
+    'tsconfig.dev.json', // Keep development-specific tsconfig file
+    'jsconfig.json', // Keep jsconfig.json for JavaScript projects
+    'eslint.json', // Keep eslint configuration
+    'eslint.config.js', // Keep eslint config file
+    'babel.config.js', // Keep Babel configuration
+    'webpack.config.js', // Keep Webpack configuration
+    'webpack.dev.config.js', // Keep development-specific Webpack config
+    'webpack.prod.config.js', // Keep production-specific Webpack config
+    'rollup.config.js', // Keep Rollup configuration file
+    'gulpfile.js', // Keep Gulp task runner file
+    'Makefile', // Keep Makefile for project builds
+    // 🧪 Docker & CI/CD
+    'Dockerfile', // Keep Dockerfile
+    'Dockerfile.dev', // Keep Dockerfile for development
+    'docker-compose.yaml', // Keep docker-compose.yaml for container orchestration
+    'docker-compose.yml', // Keep docker-compose.yml (common variation)
+    'ci.yml', // Keep CI configuration file (e.g., GitHub Actions)
+    'gitlab-ci.yml', // Keep GitLab CI configuration file
+    'Jenkinsfile', // Keep Jenkins pipeline file
+    'circleci/config.yml', // Keep CircleCI configuration file
+    // 📜 Documentation and Readme Files
+    'README.md', // Keep README file for project documentation
+    'README.rst', // Keep README in reStructuredText format
+    'CONTRIBUTING.md', // Keep contributing guidelines
+    'CHANGELOG.md', // Keep changelog for tracking project history
+    'LICENSE', // Keep project license
+    'LICENSE.txt', // Keep license in text format
+    'LICENSE.md', // Keep license in markdown format
+    'NOTICE.txt', // Keep NOTICE file
+    'INSTALL.md', // Keep installation instructions
+    // 🛠️ Build and Deployment Configuration Files
+    'build.gradle', // Keep Gradle build file
+    'pom.xml', // Keep Maven Project Object Model (POM) file
+    'settings.gradle', // Keep Gradle settings file
+    'build.sh', // Keep shell script for building the project
+    'build.bash', // Keep bash build script
+    'deploy.sh', // Keep shell script for deployment
+    'ci.sh', // Keep shell script for CI
+    // 🔧 Other Project Files
+    'Makefile.am', // Keep Automake Makefile
+    'config.yaml', // Keep general config file in YAML format
+    'config.json', // Keep general config file in JSON format
+    'config.toml', // Keep TOML configuration file
+    'settings.json', // Keep settings configuration file
+    'settings.yml', // Keep settings configuration file in YAML format
+    'secrets.json', // Keep secrets (make sure they are handled securely)
+    // 📂 Web Development & Frontend
+    'index.html', // Keep main HTML file
+    'index.php', // Keep main PHP file
+    'app.js', // Keep main JavaScript entry file
+    'app.ts', // Keep main TypeScript entry file
+    'styles.css', // Keep main CSS file
+    'main.scss', // Keep main SCSS file
+    'main.less', // Keep main LESS file
+    'style.css', // Keep style CSS
+    'app.vue', // Keep Vue.js file
+    'index.vue', // Keep Vue.js index file
+    // 🛠️ Miscellaneous Important Files
+    'README.txt', // Keep documentation in text format
+    'data.json', // Keep data JSON file
+    'data.yml', // Keep data YAML file
+    'env.json', // Keep environment JSON file
+    'env.yml', // Keep environment YAML file
+    '.env', // Keep environment variable files
+    '.env.local', // Keep local environment variables
+    '.env.production', // Keep production environment variables
+    '.env.development', // Keep development environment variables
+    // 🚧 Test-related files
+    'test.config.js', // Keep test config for testing frameworks
+    'test-utils.js', // Keep test utility files
+    'test.setup.js', // Keep setup files for tests
+    'jest.setup.js', // Keep Jest setup files
+    'mocha.setup.js', // Keep Mocha setup files
+    'karma.conf.js', // Keep Karma configuration for tests
+    'cypress.json', // Keep Cypress config for end-to-end testing
+    'karma.conf.js', // Keep Karma test runner config
+    'tests.js', // Keep test file
+    'tests.ts', // Keep TypeScript test file
+    'test.js', // Keep JavaScript test file
+    'test.ts', // Keep TypeScript test file
+];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.23",
+  "version": "0.1.25",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"
@@ -27,12 +27,14 @@
     "better-sqlite3": "^12.1.1",
     "commander": "^11.0.0",
     "fast-glob": "^3.3.3",
-    "ora": "^8.2.0"
+    "ora": "^8.2.0",
+    "proper-lockfile": "^4.1.2"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",
     "@types/jest": "^30.0.0",
     "@types/node": "^24.0.1",
+    "@types/proper-lockfile": "^4.1.4",
     "jest": "^30.0.2",
     "ts-jest": "^29.4.0",
     "typescript": "^5.8.3"