scai 0.1.22 → 0.1.23

package/dist/commands/DaemonCmd.js

@@ -2,79 +2,85 @@ import { summaryModule } from '../pipeline/modules/summaryModule.js';
 import { db } from '../db/client.js';
 import fs from 'fs/promises';
 import fsSync from 'fs';
-import os from 'os';
 import path from 'path';
 import { generateEmbedding } from '../lib/generateEmbedding.js';
 import { IGNORED_EXTENSIONS } from '../config/IgnoredExtensions.js';
-const MAX_FILES = 1000;
-const DAEMON_DURATION_MINUTES = 25;
-const DAEMON_INTERVAL_MINUTES = 30;
-const PID_PATH = path.join(os.homedir(), '.scai/daemon.pid');
-// Helper function to check if a file should be ignored
-const shouldIgnoreFile = (filePath) => {
+import { PID_PATH, SCAI_HOME } from '../constants.js';
+const LOG_PATH = path.join(SCAI_HOME, 'daemon.log');
+const SLEEP_MS = 30 * 1000; // 💤 Pause between batches
+const IDLE_SLEEP_MS = 4 * SLEEP_MS; // 💤 Longer pause if idle
+const MAX_FILES_PER_BATCH = 5; // 🎛️ Throttle indexing per cycle
+// 💤 Utility
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+// 🪵 Append to log file
+function log(message) {
+    const timestamp = new Date().toISOString();
+    fsSync.appendFileSync(LOG_PATH, `[${timestamp}] ${message}\n`);
+}
+// ❌ Skip unwanted file types
+function shouldIgnoreFile(filePath) {
     const ext = path.extname(filePath).toLowerCase();
     return IGNORED_EXTENSIONS.includes(ext);
-};
+}
+// 🧠 One summarization batch
 export async function runDaemonBatch() {
-    console.log('📥 Daemon batch: scanning for files to summarize...');
     const rows = db.prepare(`
     SELECT path, type FROM files
     WHERE summary IS NULL OR summary = ''
     ORDER BY last_modified DESC
     LIMIT ?
-    `).all(MAX_FILES);
+  `).all(MAX_FILES_PER_BATCH);
     if (rows.length === 0) {
-        console.log('✅ No files left to summarize.');
-        return;
+        log('✅ No files left to summarize.');
+        return false; // 💤 Idle
     }
     for (const row of rows) {
+        if (!fsSync.existsSync(row.path)) {
+            log(`⚠️ Skipped missing file: ${row.path}`);
+            continue;
+        }
         if (shouldIgnoreFile(row.path)) {
-            console.log(`⚠️ Ignored file (unwanted extension): ${row.path}`);
+            log(`⚠️ Skipped (extension): ${row.path}`);
             continue;
         }
         try {
             const content = await fs.readFile(row.path, 'utf-8');
             const result = await summaryModule.run({ content, filepath: row.path });
-            const summary = result?.summary?.trim() ? result.summary : null;
+            const summary = result?.summary?.trim() || null;
             let embedding = null;
             if (summary) {
                 const vector = await generateEmbedding(summary);
                 if (vector)
                     embedding = JSON.stringify(vector);
             }
-            // Using named parameters for better readability and flexibility
             db.prepare(`
-                UPDATE files
-                SET summary = @summary, embedding = @embedding, indexed_at = datetime('now')
-                WHERE path = @path
-            `).run({ summary, embedding, path: row.path });
-            console.log(`📝 Summarized: ${row.path}`);
-            console.log(`🔢 Embedded: ${row.path}`);
+        UPDATE files
+        SET summary = @summary, embedding = @embedding, indexed_at = datetime('now')
+        WHERE path = @path
+      `).run({ summary, embedding, path: row.path });
+            log(`📝 Summarized: ${row.path}`);
         }
         catch (err) {
-            console.warn(`⚠️ Failed: ${row.path}`, err instanceof Error ? err.message : err);
+            log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}`);
         }
+        await sleep(200); // 🧘 Micro delay between each file
     }
+    return true; // ✅ Work was done
 }
+// 🚀 Daemon loop: runs until killed
 export async function runDaemonScheduler() {
-    // Write PID to file
-    fsSync.mkdirSync(path.dirname(PID_PATH), { recursive: true });
+    fsSync.mkdirSync(SCAI_HOME, { recursive: true });
     fsSync.writeFileSync(PID_PATH, process.pid.toString(), 'utf-8');
-    console.log('🧠 Daemon started. PID:', process.pid);
-    console.log('⏱️ Will run every 30 minutes for 10 minutes.');
-    console.log('🧠 Background summarizer started. Will run every 30 minutes for 10 minutes.');
-    const startDaemonCycle = async () => {
-        const startTime = Date.now();
-        const endTime = startTime + DAEMON_DURATION_MINUTES * 60 * 1000;
-        while (Date.now() < endTime) {
-            await runDaemonBatch();
-            await new Promise(res => setTimeout(res, 60 * 1000)); // 1 min pause between mini-batches
-        }
-        console.log(`⏱️ Daemon completed 10-minute cycle. Next in ${DAEMON_INTERVAL_MINUTES} min.`);
-    };
-    // Repeat every 30 minutes
+    fsSync.appendFileSync(LOG_PATH, `\n\n🧠 Daemon started at ${new Date().toISOString()} — PID ${process.pid}\n`);
+    let cycles = 0;
     while (true) {
-        await startDaemonCycle();
-        await new Promise(res => setTimeout(res, DAEMON_INTERVAL_MINUTES * 60 * 1000));
+        const didWork = await runDaemonBatch();
+        cycles++;
+        if (cycles % 20 === 0) {
+            log(`🌀 Still running. Cycles: ${cycles}`);
+        }
+        await sleep(didWork ? SLEEP_MS : IDLE_SLEEP_MS);
     }
 }

package/dist/commands/IndexCmd.js

@@ -1,4 +1,5 @@
 import fg from 'fast-glob';
+import fs from 'fs';
 import path from 'path';
 import { initSchema } from '../db/schema.js';
 import { indexFile } from '../db/fileIndex.js';
@@ -6,35 +7,33 @@ import { shouldIgnoreFile } from '../utils/shouldIgnoreFiles.js';
 import { detectFileType } from '../utils/detectFileType.js';
 import { runDaemonScheduler } from './DaemonCmd.js';
 import { IGNORED_FOLDER_GLOBS } from '../config/IgnoredPaths.js';
-import { db } from '../db/client.js';
-const IGNORE = [
-    '**/node_modules/**', '**/dist/**', '**/build/**',
-    '**/coverage/**', '**/.git/**', '**/*.test.*'
-];
-export async function runIndexCommand(targetDir = process.cwd(), options = {}) {
-    const resolvedDir = path.resolve(targetDir);
-    console.log(`📂 Indexing files in: ${resolvedDir}`);
-    initSchema();
-    // 🧠 Check if another directory has already been indexed
-    const indexedPaths = db.prepare(`
-    SELECT DISTINCT path FROM files LIMIT 100
-  `).all();
-    const knownRoot = indexedPaths.length > 0
-        ? path.dirname(indexedPaths[0].path)
-        : null;
-    if (knownRoot && !resolvedDir.startsWith(knownRoot) && !options.force) {
-        console.warn(`⚠️ You're indexing a different folder than before:
-- Previously: ${knownRoot}
-- Now:        ${resolvedDir}
-
-This will add more files into the existing index and may reduce accuracy or performance.
-
-Use --force to continue, or consider clearing the index:
-  scai reset-db
-
-Aborting.`);
+import { Config } from '../config.js';
+import { PID_PATH } from '../constants.js';
+export async function runIndexCommand(targetDir, options = {}) {
+    try {
+        initSchema();
+    }
+    catch (err) {
+        console.error('❌ Failed to initialize schema:', err);
         process.exit(1);
     }
+    let resolvedDir;
+    if (options.force) {
+        // Force: use passed dir or fallback to cwd, no config updates
+        resolvedDir = path.resolve(targetDir || process.cwd());
+        console.warn('⚠️ Running in --force mode. Config will not be updated.');
+    }
+    else if (targetDir) {
+        // User provided a directory: resolve and persist to config
+        resolvedDir = path.resolve(targetDir);
+        Config.setIndexDir(resolvedDir);
+    }
+    else {
+        // Use configured indexDir or fallback to cwd
+        resolvedDir = Config.getIndexDir() || process.cwd();
+        Config.setIndexDir(resolvedDir); // persist if not yet saved
+    }
+    console.log(`📂 Indexing files in: ${resolvedDir}`);
     const files = await fg('**/*.*', {
         cwd: resolvedDir,
         ignore: IGNORED_FOLDER_GLOBS,
@@ -47,20 +46,25 @@ Aborting.`);
             continue;
         try {
             const type = detectFileType(file);
-            indexFile(file, null, type); // empty summary for now
+            indexFile(file, null, type); // Index file without summary
             const ext = path.extname(file);
             countByExt[ext] = (countByExt[ext] || 0) + 1;
             console.log(`📄 Indexed: ${path.relative(resolvedDir, file)}`);
             count++;
         }
         catch (err) {
-            console.warn(`⚠️ Skipped ${file}:`, err instanceof Error ? err.message : err);
+            console.warn(`⚠️ Skipped in indexCmd ${file}:`, err instanceof Error ? err.message : err);
         }
     }
     console.log('📊 Indexed files by extension:', countByExt);
     console.log(`✅ Done. Indexed ${count} files.`);
     if (options.detached) {
-        console.log('🚀 Starting summarizer daemon in background mode...');
-        runDaemonScheduler();
+        if (fs.existsSync(PID_PATH)) {
+            console.warn(`⚠️ Daemon already running (PID file found at ${PID_PATH}). Skipping launch.`);
+        }
+        else {
+            console.log('🚀 Starting summarizer daemon in background mode...');
+            runDaemonScheduler();
+        }
     }
 }

package/dist/commands/ResetDbCmd.js

@@ -1,19 +1,18 @@
 import fs from 'fs';
-import path from 'path';
 import { db } from '../db/client.js';
+import { DB_PATH } from '../constants.js';
 export function resetDatabase() {
-    const dbPath = path.resolve(process.cwd(), '.scai/db.sqlite');
     try {
-        db.close(); // 🔒 Make sure the DB is closed
+        db.close(); // 🔒 Ensure the DB connection is closed
         console.log('🔒 Closed SQLite database connection.');
     }
     catch (err) {
         console.warn('⚠️ Could not close database:', err);
     }
-    if (fs.existsSync(dbPath)) {
+    if (fs.existsSync(DB_PATH)) {
         try {
-            fs.unlinkSync(dbPath);
-            console.log('🧹 Deleted existing database.');
+            fs.unlinkSync(DB_PATH);
+            console.log(`🧹 Deleted existing database at ${DB_PATH}`);
         }
         catch (err) {
             console.error('❌ Failed to delete DB file:', err instanceof Error ? err.message : err);
@@ -21,7 +20,7 @@ export function resetDatabase() {
         }
     }
     else {
-        console.log('ℹ️ No existing database found.');
+        console.log('ℹ️ No existing database found at:', DB_PATH);
     }
     console.log('✅ Database has been reset. You can now re-run: scai index');
 }

package/dist/constants.js

@@ -1,19 +1,44 @@
 import os from 'os';
 import path from 'path';
 import fs from 'fs';
+/**
+ * The base directory where internal SCAI config/state is stored:
+ * ~/.scai
+ */
 export const SCAI_HOME = path.join(os.homedir(), '.scai');
+/**
+ * Full path to the SQLite database used by SCAI:
+ * ~/.scai/db.sqlite
+ */
 export const DB_PATH = path.join(SCAI_HOME, 'db.sqlite');
+/**
+ * Path to the daemon process ID file (if running in background mode):
+ * ~/.scai/daemon.pid
+ */
 export const PID_PATH = path.join(SCAI_HOME, 'daemon.pid');
+/**
+ * Path to the config file that stores user settings like model, language, indexDir, etc.:
+ * ~/.scai/config.json
+ */
 export const CONFIG_PATH = path.join(SCAI_HOME, 'config.json');
-// Function to read config and get the indexDir on-demand
+/**
+ * Get the active index directory.
+ *
+ * - If the user has configured an `indexDir`, use it.
+ * - If not, default to the user’s home directory (`~`), not `.scai`.
+ */
 export function getIndexDir() {
     try {
         const config = JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf-8'));
-        return config.indexDir || path.join(os.homedir(), 'defaultIndex'); // Default if not set
+        return config.indexDir || os.homedir(); // 🏠 Default: ~
     }
     catch (e) {
-        return path.join(os.homedir(), 'defaultIndex'); // Fallback if no config file
+        return os.homedir(); // 🏠 Fallback if config file is missing or invalid
     }
 }
-// On-demand index directory retrieval
+/**
+ * On-demand index directory to scan for files.
+ *
+ * Used by indexing logic (`scai index`) to determine what folder to scan.
+ */
 export const INDEX_DIR = getIndexDir();

package/dist/db/fileIndex.js

@@ -1,51 +1,57 @@
-// File: src/db/fileIndex.ts
 import { db } from './client.js';
 import fs from 'fs';
 import { generateEmbedding } from '../lib/generateEmbedding.js';
-import * as sqlTemplates from './sqlTemplates.js'; // Import the SQL templates
+import * as sqlTemplates from './sqlTemplates.js';
+import path from 'path';
+/**
+ * Index a file into the local SQLite database.
+ *
+ * - Normalizes the file path for cross-platform compatibility.
+ * - Extracts file metadata (last modified time).
+ * - Performs an UPSERT into the `files` table with the latest summary/type/timestamp.
+ *
+ * @param filePath - Absolute path to the file being indexed
+ * @param summary - Optional summary of the file content
+ * @param type - File type or extension (e.g., 'md', 'ts')
+ */
 export function indexFile(filePath, summary, type) {
     const stats = fs.statSync(filePath);
     const lastModified = stats.mtime.toISOString();
-    // 1) INSERT new rows (only when path not present)
-    const insertStmt = db.prepare(sqlTemplates.insertFileTemplate);
-    insertStmt.run({ path: filePath, summary, type, lastModified });
-    // 2) UPDATE metadata if file already existed and changed
-    const updateStmt = db.prepare(sqlTemplates.updateFileTemplate);
-    updateStmt.run({ path: filePath, type, lastModified });
-    // Step 1: Delete from FTS where the path matches
-    db.prepare(sqlTemplates.deleteFromFtsTemplate).run({ path: filePath });
-    // Step 2: Insert into FTS with the same id
-    db.prepare(sqlTemplates.insertIntoFtsTemplate).run({ path: filePath, summary });
+    const indexedAt = new Date().toISOString();
+    const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
+    db.prepare(sqlTemplates.upsertFileTemplate).run({
+        path: normalizedPath,
+        summary,
+        type,
+        lastModified,
+        indexedAt,
+    });
+    console.log(`📄 Indexed: ${normalizedPath}`);
 }
+/**
+ * Perform a raw keyword-based full-text search using the FTS5 index.
+ *
+ * - Tokenizes and sanitizes the input query string.
+ * - Performs a ranked search using BM25 scoring via the virtual FTS table.
+ * - Returns basic file metadata along with rank for ordering.
+ *
+ * @param query - The search query string (e.g., "api router config")
+ * @param limit - Max number of results to return (default: 10)
+ */
 export function queryFiles(query, limit = 10) {
-    // Sanitize the query by removing or escaping special characters
     const safeQuery = query
         .trim()
         .split(/\s+/)
         .map(token => {
-        token = token
-            .replace(/[?*\\"]/g, '') // Remove question marks, asterisks, backslashes, and double quotes
-            .replace(/'/g, "''"); // Escape single quotes for SQL safety
-        // For multi-word queries, wrap the token in quotes for exact phrase matching
-        if (token.includes(' ')) {
-            return `"${token}"`; // Exact phrase match for multi-word tokens
-        }
-        return `${token}*`; // Prefix match for single tokens
+        token = token.replace(/[?*\\"]/g, '').replace(/'/g, "''");
+        return token.includes(' ') ? `"${token}"` : `${token}*`;
     })
         .join(' OR ');
-    // Log the constructed query for debugging purposes
     console.log(`Executing search query: ${safeQuery}`);
-    // Execute the query with safeQuery and limit as parameters
-    const sql = `
-    SELECT f.path, f.summary, f.type, f.last_modified, f.indexed_at,
-           bm25(files_fts) AS rank
-    FROM files_fts
-    JOIN files f ON files_fts.rowid = f.id
-    WHERE files_fts MATCH :query
-    ORDER BY rank
-    LIMIT :limit
-  `;
-    const results = db.prepare(sql).all({ query: safeQuery, limit });
+    const results = db.prepare(sqlTemplates.rawQueryTemplate).all({
+        query: safeQuery,
+        limit
+    });
     return results;
 }
 export function cosineSimilarity(a, b) {
@@ -54,52 +60,47 @@ export function cosineSimilarity(a, b) {
     const magB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));
     return dot / (magA * magB);
 }
+/**
+ * Perform a hybrid semantic + keyword-based search.
+ *
+ * - Generates a vector embedding of the query.
+ * - Runs an FTS search using BM25 ranking.
+ * - Looks up file embeddings from the database and compares using cosine similarity.
+ * - Combines similarity score and BM25 rank into a weighted final score.
+ *
+ * @param query - Natural language search query
+ * @param topK - Max number of top-ranked results to return (default: 5)
+ */
 export async function searchFiles(query, topK = 5) {
-    // Generate the query embedding
     const embedding = await generateEmbedding(query);
     if (!embedding)
         return [];
-    // Sanitize the query by removing or escaping special characters
     const safeQuery = query
         .trim()
         .split(/\s+/)
         .map(token => {
-        token = token
-            .replace(/[?*\\"]/g, '') // Remove question marks, asterisks, backslashes, and double quotes
-            .replace(/'/g, "''"); // Escape single quotes for SQL safety
-        // For multi-word queries, wrap the token in quotes for exact phrase matching
-        if (token.includes(' ')) {
-            return `"${token}"`; // Exact phrase match for multi-word tokens
-        }
-        return `${token}*`; // Prefix match for single tokens
+        token = token.replace(/[?*\\"]/g, '').replace(/'/g, "''");
+        return token.includes(' ') ? `"${token}"` : `${token}*`;
     })
         .join(' OR ');
-    // Log the constructed query for debugging purposes
     console.log(`Executing search query: ${safeQuery}`);
-    // Fetch BM25 scores from the FTS using the safeQuery string directly
     const ftsResults = db.prepare(sqlTemplates.fetchBm25ScoresTemplate).all({ query: safeQuery });
     const bm25Min = Math.min(...ftsResults.map(r => r.bm25Score));
     const bm25Max = Math.max(...ftsResults.map(r => r.bm25Score));
-    // Calculate final score combining BM25 and cosine similarity
     const scored = ftsResults.map(result => {
         try {
-            // Fetch embedding for each file from the `files` table
             const embResult = db.prepare(sqlTemplates.fetchEmbeddingTemplate).get({ path: result.path });
-            // Check if embedding exists and has the correct structure
             if (!embResult || typeof embResult.embedding !== 'string')
                 return null;
-            // Parse the embedding
             const vector = JSON.parse(embResult.embedding);
             const sim = cosineSimilarity(embedding, vector);
-            // Normalize BM25 scores
             const normalizedBm25 = 1 - ((result.bm25Score - bm25Min) / (bm25Max - bm25Min + 1e-5));
-            const normalizedSim = sim; // cosineSimilarity is already 0–1
-            const finalScore = 0.7 * normalizedSim + 0.3 * normalizedBm25;
+            const finalScore = 0.7 * sim + 0.3 * normalizedBm25;
             return {
                 path: result.path,
                 summary: result.summary,
                 score: finalScore,
-                sim: normalizedSim,
+                sim,
                 bm25: normalizedBm25
             };
         }

package/dist/db/schema.js

@@ -1,19 +1,19 @@
 import { db } from "./client.js";
 export function initSchema() {
     db.exec(`
-    CREATE TABLE IF NOT EXISTS files (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      path TEXT UNIQUE,
-      summary TEXT,
-      type TEXT,
-      indexed_at TEXT,
-      last_modified TEXT,
-      embedding TEXT
-    );
+  CREATE TABLE IF NOT EXISTS files (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  path TEXT UNIQUE,
+  summary TEXT,
+  type TEXT,
+  indexed_at TEXT,
+  last_modified TEXT,
+  embedding TEXT
+);
 
-    -- FTS5 table for fast full‑text search of summaries and paths
-    CREATE VIRTUAL TABLE IF NOT EXISTS files_fts
-    USING fts5(path, summary, content='');
-  `);
+CREATE VIRTUAL TABLE IF NOT EXISTS files_fts
+USING fts5(path, summary, content='files', content_rowid='id');
+
+`);
     console.log('✅ SQLite schema initialized');
 }

package/dist/db/sqlTemplates.js

@@ -1,29 +1,14 @@
-// Template for inserting or ignoring new file entries
-export const insertFileTemplate = `
-  INSERT OR IGNORE INTO files
-    (path, summary, type, indexed_at, last_modified)
-  VALUES (:path, :summary, :type, datetime('now'), :lastModified)
+// Upsert file metadata into `files`
+export const upsertFileTemplate = `
+  INSERT INTO files (path, summary, type, last_modified, indexed_at)
+  VALUES (:path, :summary, :type, :lastModified, :indexedAt)
+  ON CONFLICT(path) DO UPDATE SET
+    summary = excluded.summary,
+    type = excluded.type,
+    last_modified = excluded.last_modified,
+    indexed_at = excluded.indexed_at
 `;
-// Template for updating file metadata if it has changed
-export const updateFileTemplate = `
-  UPDATE files
-  SET type = :type,
-      last_modified = :lastModified,
-      indexed_at = datetime('now')
-  WHERE path = :path
-    AND last_modified != :lastModified
-`;
-// Template for deleting a file from FTS
-export const deleteFromFtsTemplate = `
-  DELETE FROM files_fts
-  WHERE rowid = (SELECT id FROM files WHERE path = :path)
-`;
-// Template for inserting a file into FTS with its ID
-export const insertIntoFtsTemplate = `
-  INSERT INTO files_fts(rowid, path, summary)
-  VALUES((SELECT id FROM files WHERE path = :path), :path, :summary)
-`;
-// Template for fetching BM25 scores from FTS
+// Fetch search results with BM25 ranking
 export const fetchBm25ScoresTemplate = `
   SELECT f.path, f.summary, f.type, bm25(files_fts) AS bm25Score
   FROM files_fts
@@ -31,7 +16,17 @@ export const fetchBm25ScoresTemplate = `
   WHERE files_fts MATCH :query
   LIMIT 50
 `;
-// Template for fetching embedding for a specific file
+// Fetch embedding vector for a file
 export const fetchEmbeddingTemplate = `
   SELECT embedding FROM files WHERE path = :path
 `;
+// Used for non-embedding query in `queryFiles()`
+export const rawQueryTemplate = `
+  SELECT f.path, f.summary, f.type, f.last_modified, f.indexed_at,
+         bm25(files_fts) AS rank
+  FROM files_fts
+  JOIN files f ON files_fts.rowid = f.id
+  WHERE files_fts MATCH :query
+  ORDER BY rank
+  LIMIT :limit
+`;

package/dist/index.js

@@ -73,8 +73,7 @@ cmd
     .option('-d, --detached', 'Run summarizer daemon after indexing')
     .option('--force', 'Force indexing even if another folder has already been indexed')
     .action((targetDir, options) => {
-    const resolvedDir = targetDir ? path.resolve(targetDir) : process.cwd();
-    runIndexCommand(resolvedDir, { detached: options.detached, force: options.force });
+    runIndexCommand(targetDir, { detached: options.detached, force: options.force });
 });
 // ⚙️ Group: Configuration settings
 const set = cmd.command('set').description('Set configuration values');
@@ -127,7 +126,9 @@ cmd
 cmd
     .command('daemon')
     .description('Run background summarization of indexed files')
-    .action(runDaemonBatch);
+    .action(async () => {
+    await runDaemonBatch(); // ignore the return value
+});
 cmd
     .command('stop-daemon')
     .description('Stop the background summarizer daemon')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.22",
+  "version": "0.1.23",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"