scai 0.1.19 → 0.1.20

package/dist/commands/AskCmd.js

@@ -0,0 +1,66 @@
+import { searchFiles } from "../db/fileIndex.js";
+import { generate } from "../lib/generate.js";
+import { summaryModule } from "../pipeline/modules/summaryModule.js";
+export async function runAskCommand(query) {
+    if (!query) {
+        console.error('❌ Please provide a search query.\n👉 Usage: scai ask "keyword"');
+        return;
+    }
+    console.log(`🔍 Searching for: "${query}"\n`);
+    // Use vector-based search
+    const results = await searchFiles(query, 5); // Or 3 if you want fewer
+    if (results.length === 0) {
+        console.log('⚠️ No similar embeddings found. Asking the model for context instead...');
+    }
+    else {
+        console.log('📊 Closest files based on semantic similarity:');
+        results.forEach(file => {
+            console.log(`📄 Path: ${file?.path}`);
+        });
+    }
+    let allSummaries = '';
+    for (const file of results) {
+        try {
+            if (!file?.summary) {
+                console.warn(`⚠️ No summary available for file: ${file?.path}`);
+                continue;
+            }
+            console.log(`📝 Using cached summary for file: ${file?.path}`);
+            const summaryResponse = await summaryModule.run({ content: file?.summary ? file.summary : '', filepath: file?.path });
+            if (summaryResponse.summary) {
+                allSummaries += `\n${summaryResponse.summary}`;
+            }
+        }
+        catch (err) {
+            console.error(`❌ Error processing file: ${file?.path}`, err instanceof Error ? err.message : err);
+        }
+    }
+    if (allSummaries.trim()) {
+        console.log('🧠 Summaries found, sending them to the model for synthesis...');
+        try {
+            const input = {
+                content: `${query}\n\n${allSummaries}`,
+                filepath: '',
+            };
+            const modelResponse = await generate(input, 'llama3');
+            console.log(`\n📝 Model response:\n${modelResponse.content}`);
+        }
+        catch (err) {
+            console.error('❌ Model request failed:', err);
+        }
+    }
+    else {
+        console.log('⚠️ No summaries found. Asking the model for context only...');
+        try {
+            const input = {
+                content: query,
+                filepath: '',
+            };
+            const modelResponse = await generate(input, 'llama3');
+            console.log(`\n📝 Model response:\n${modelResponse.content}`);
+        }
+        catch (err) {
+            console.error('❌ Model request failed:', err);
+        }
+    }
+}

package/dist/commands/DaemonCmd.js

@@ -4,10 +4,17 @@ import fs from 'fs/promises';
 import fsSync from 'fs';
 import os from 'os';
 import path from 'path';
-const MAX_FILES = 500;
-const DAEMON_DURATION_MINUTES = 10;
+import { generateEmbedding } from '../lib/generateEmbedding.js';
+import { IGNORED_EXTENSIONS } from '../config/IgnoredExtensions.js';
+const MAX_FILES = 1000;
+const DAEMON_DURATION_MINUTES = 25;
 const DAEMON_INTERVAL_MINUTES = 30;
 const PID_PATH = path.join(os.homedir(), '.scai/daemon.pid');
+// Helper function to check if a file should be ignored
+const shouldIgnoreFile = (filePath) => {
+    const ext = path.extname(filePath).toLowerCase();
+    return IGNORED_EXTENSIONS.includes(ext);
+};
 export async function runDaemonBatch() {
     console.log('📥 Daemon batch: scanning for files to summarize...');
     const rows = db.prepare(`
@@ -15,19 +22,34 @@ export async function runDaemonBatch() {
     WHERE summary IS NULL OR summary = ''
     ORDER BY last_modified DESC
     LIMIT ?
-`).all(MAX_FILES);
+    `).all(MAX_FILES);
     if (rows.length === 0) {
         console.log('✅ No files left to summarize.');
         return;
     }
     for (const row of rows) {
+        if (shouldIgnoreFile(row.path)) {
+            console.log(`⚠️ Ignored file (unwanted extension): ${row.path}`);
+            continue;
+        }
         try {
             const content = await fs.readFile(row.path, 'utf-8');
             const result = await summaryModule.run({ content, filepath: row.path });
             const summary = result?.summary?.trim() ? result.summary : null;
-            db.prepare(`UPDATE files SET summary = ?, indexed_at = datetime('now') WHERE path = ?`)
-                .run(summary, row.path);
+            let embedding = null;
+            if (summary) {
+                const vector = await generateEmbedding(summary);
+                if (vector)
+                    embedding = JSON.stringify(vector);
+            }
+            // Using named parameters for better readability and flexibility
+            db.prepare(`
+                UPDATE files
+                SET summary = @summary, embedding = @embedding, indexed_at = datetime('now')
+                WHERE path = @path
+            `).run({ summary, embedding, path: row.path });
             console.log(`📝 Summarized: ${row.path}`);
+            console.log(`🔢 Embedded: ${row.path}`);
         }
         catch (err) {
             console.warn(`⚠️ Failed: ${row.path}`, err instanceof Error ? err.message : err);

package/dist/commands/QueryCmd.js

@@ -1,4 +1,4 @@
-import { searchFiles } from '../db/fileIndex.js';
+import { queryFiles } from '../db/fileIndex.js';
 import path from 'path';
 export async function runQueryCommand(query) {
     if (!query) {
@@ -6,7 +6,7 @@ export async function runQueryCommand(query) {
         return;
     }
     console.log(`🔍 Searching for: "${query}"\n`);
-    const results = searchFiles(query);
+    const results = queryFiles(query);
     if (results.length === 0) {
         console.log('⚠️ No matching files found.');
         return;

package/dist/config/IgnoredExtensions.js

@@ -3,23 +3,44 @@ export const IGNORED_EXTENSIONS = [
     // 🖼 Media
     '.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico',
     '.mp4', '.mp3', '.mov', '.avi', '.mkv', '.flv', '.wav', '.flac',
+    '.aac', '.m4a', '.wma', '.3gp', '.webm', '.ogg', '.aiff', '.au',
     // 📦 Archives & install packages
     '.zip', '.tar', '.gz', '.bz2', '.xz', '.rar', '.7z',
-    '.jar', '.war', '.ear', // Java packaging
-    '.deb', '.rpm', '.pkg', '.msi', '.dmg', '.cab', '.xz',
+    '.jar', '.war', '.ear', '.deb', '.rpm', '.pkg', '.msi', '.dmg', '.cab', '.apk',
+    '.tar.gz', '.tar.bz2', '.tar.xz', '.tar.lzma', '.tar.zst',
     // 🧱 Binaries & executables
     '.exe', '.dll', '.bin', '.so', '.dylib', '.a', '.lib',
-    '.iso', '.img', '.elf', '.o', '.obj',
+    '.iso', '.img', '.elf', '.o', '.obj', '.msm', '.vbs', '.jscript',
+    '.cmd', '.bat', '.ps1', '.sh', '.bash', '.run',
     // 🧪 Runtime / build / cache
-    '.log', '.lock', '.tmp', '.map',
+    '.log', '.tmp', '.map',
     '.db', '.sqlite', '.pkl', '.sav', '.rdb', '.ldb',
-    '.pyc', '.class', '.tsbuildinfo', '.coverage',
+    '.pyc', '.class', '.tsbuildinfo', '.coverage', '.eslintcache',
+    '.yarn', '.webpack', '.babel', '.compilercache',
     // 🔤 Fonts & styles
     '.woff', '.woff2', '.ttf', '.eot', '.otf', '.css.map',
+    '.scss', '.sass', '.less', '.styl',
     // 🔐 Certs, keys, credentials
-    '.crt', '.key', '.pem', '.pub', '.asc', '.gpg',
+    '.crt', '.key', '.pem', '.pub', '.asc', '.gpg', '.p12', '.csr', '.der', '.pfx',
     // ♻️ Backups / temp
-    '.bak', '.old', '.swp', '.swo', '.tmp', '.orig',
-    // 🌐 Misc
-    '.torrent', '.DS_Store', '.env.local', '.env.production', '.env.development',
+    '.bak', '.old', '.swp', '.swo', '.orig',
+    '.sublime-workspace', '.sublime-project', '.db-shm', '.db-wal',
+    // 🌐 System/config folders (still ignored by path, not extension)
+    '.DS_Store', '.bundle', '.npmrc',
+    // 🗺️ GIS / Geospatial
+    '.shp', '.shx', '.dbf', '.prj', '.qix', '.sbn', '.sbx', '.shp.xml', '.cpg', '.gpkg', '.mif', '.mid',
+    // 📊 Enterprise BI / Reporting
+    '.pbix', '.rdl', '.rpt', '.bqy', '.iqy',
+    // 🧪 ETL / DWH / Modeling
+    '.abf', '.dtsx', '.bim', '.xmi',
+    // 🏗️ CAD / Engineering
+    '.dwg', '.dxf', '.step', '.stp', '.sldprt', '.sldasm',
+    '.iges', '.igs', '.3ds', '.fbx',
+    // 🧾 Forms / Print / Publishing
+    '.xps', '.afpub', '.pub', '.indd', '.qxd', '.frm', '.frx', '.frl',
+    // 💰 ERP / Finance / Legacy DB
+    '.mbd', '.fdb', '.nav', '.accdb', '.mdb', '.gdb',
+    '.sap', '.sappkg', '.qbw', '.qbb',
+    // 🔒 Lock files (but NOT lock *configs*)
+    '.lck', '.lockfile', '.db-lock', '.pid', '.socket',
 ];

package/dist/config/IgnoredPaths.js

@@ -18,4 +18,28 @@ export const IGNORED_FOLDER_GLOBS = [
     '**/.output/**',
     '**/tmp/**',
     '**/*.test.*',
+    '**/.m2/**',
+    '**/.gradle/**',
+    '**/.tox/**',
+    '**/.nox/**',
+    '**/.hypothesis/**',
+    '**/.npm/**',
+    '**/.yarn/**',
+    '**/*.o',
+    '**/*.out',
+    '**/*.exe',
+    '**/*.dll',
+    '**/.cache/**',
+    '**/.pylint.d/**',
+    '**/.eslintcache/**',
+    '**/.cache-loader/**',
+    '**/.serverless/**',
+    '**/.docker/**',
+    '**/.sublime-workspace',
+    '**/.sublime-project',
+    '**/*.log',
+    '**/npm-debug.log',
+    '**/yarn-error.log',
+    '**/debug.log',
+    '**/Dockerfile',
 ];

package/dist/config/ModelConfig.js

@@ -19,5 +19,5 @@ export class ModelConfig {
         console.log(`   Language: ${this.language}`);
     }
 }
-ModelConfig.model = 'codellama:7b';
+ModelConfig.model = 'llama3';
 ModelConfig.language = 'ts';

package/dist/db/fileIndex.js

@@ -1,51 +1,114 @@
+// File: src/db/fileIndex.ts
 import { db } from './client.js';
 import fs from 'fs';
+import { generateEmbedding } from '../lib/generateEmbedding.js';
+import * as sqlTemplates from './sqlTemplates.js'; // Import the SQL templates
 export function indexFile(filePath, summary, type) {
     const stats = fs.statSync(filePath);
     const lastModified = stats.mtime.toISOString();
     // 1) INSERT new rows (only when path not present)
-    const insertStmt = db.prepare(`
-    INSERT OR IGNORE INTO files
-      (path, summary, type, indexed_at, last_modified)
-    VALUES (?, ?, ?, datetime('now'), ?)
-  `);
-    insertStmt.run(filePath, summary, type, lastModified);
+    const insertStmt = db.prepare(sqlTemplates.insertFileTemplate);
+    insertStmt.run({ path: filePath, summary, type, lastModified });
     // 2) UPDATE metadata if file already existed and changed
-    const updateStmt = db.prepare(`
-    UPDATE files
-    SET type       = ?,
-        last_modified = ?,
-        indexed_at = datetime('now')
-    WHERE path = ?
-      AND last_modified != ?
-  `);
-    updateStmt.run(type, lastModified, filePath, lastModified);
+    const updateStmt = db.prepare(sqlTemplates.updateFileTemplate);
+    updateStmt.run({ path: filePath, type, lastModified });
     // Step 1: Delete from FTS where the path matches
-    db.prepare(`
-  DELETE FROM files_fts
-  WHERE rowid = (SELECT id FROM files WHERE path = ?)
-`).run(filePath);
+    db.prepare(sqlTemplates.deleteFromFtsTemplate).run({ path: filePath });
     // Step 2: Insert into FTS with the same id
-    db.prepare(`
-  INSERT INTO files_fts(rowid, path, summary)
-  VALUES((SELECT id FROM files WHERE path = ?), ?, ?)
-`).run(filePath, filePath, summary);
+    db.prepare(sqlTemplates.insertIntoFtsTemplate).run({ path: filePath, summary });
 }
-export function searchFiles(query, limit = 10) {
-    // Use FTS5 MATCH for relevance-ranked results
-    const stmt = db.prepare(`
+export function queryFiles(query, limit = 3) {
+    // Sanitize the query by removing or escaping special characters
+    const safeQuery = query
+        .trim()
+        .split(/\s+/)
+        .map(token => {
+        token = token
+            .replace(/[?*\\"]/g, '') // Remove question marks, asterisks, backslashes, and double quotes
+            .replace(/'/g, "''"); // Escape single quotes for SQL safety
+        // For multi-word queries, wrap the token in quotes for exact phrase matching
+        if (token.includes(' ')) {
+            return `"${token}"`; // Exact phrase match for multi-word tokens
+        }
+        return `${token}*`; // Prefix match for single tokens
+    })
+        .join(' OR ');
+    // Log the constructed query for debugging purposes
+    console.log(`Executing search query: ${safeQuery}`);
+    // Execute the query with safeQuery and limit as parameters
+    const sql = `
     SELECT f.path, f.summary, f.type, f.last_modified, f.indexed_at,
            bm25(files_fts) AS rank
     FROM files_fts
     JOIN files f ON files_fts.rowid = f.id
-    WHERE files_fts MATCH ?
+    WHERE files_fts MATCH :query
     ORDER BY rank
-    LIMIT ?
-  `);
-    const matchQuery = query
+    LIMIT :limit
+  `;
+    const results = db.prepare(sql).all({ query: safeQuery, limit });
+    return results;
+}
+export function cosineSimilarity(a, b) {
+    const dot = a.reduce((sum, ai, i) => sum + ai * b[i], 0);
+    const magA = Math.sqrt(a.reduce((sum, ai) => sum + ai * ai, 0));
+    const magB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));
+    return dot / (magA * magB);
+}
+export async function searchFiles(query, topK = 5) {
+    // Generate the query embedding
+    const embedding = await generateEmbedding(query);
+    if (!embedding)
+        return [];
+    // Sanitize the query by removing or escaping special characters
+    const safeQuery = query
         .trim()
         .split(/\s+/)
-        .map(token => `${token}*`) // prefix search
-        .join(' ');
-    return stmt.all(matchQuery, limit);
+        .map(token => {
+        token = token
+            .replace(/[?*\\"]/g, '') // Remove question marks, asterisks, backslashes, and double quotes
+            .replace(/'/g, "''"); // Escape single quotes for SQL safety
+        // For multi-word queries, wrap the token in quotes for exact phrase matching
+        if (token.includes(' ')) {
+            return `"${token}"`; // Exact phrase match for multi-word tokens
+        }
+        return `${token}*`; // Prefix match for single tokens
+    })
+        .join(' OR ');
+    // Log the constructed query for debugging purposes
+    console.log(`Executing search query: ${safeQuery}`);
+    // Fetch BM25 scores from the FTS using the safeQuery string directly
+    const ftsResults = db.prepare(sqlTemplates.fetchBm25ScoresTemplate).all({ query: safeQuery });
+    const bm25Min = Math.min(...ftsResults.map(r => r.bm25Score));
+    const bm25Max = Math.max(...ftsResults.map(r => r.bm25Score));
+    // Calculate final score combining BM25 and cosine similarity
+    const scored = ftsResults.map(result => {
+        try {
+            // Fetch embedding for each file from the `files` table
+            const embResult = db.prepare(sqlTemplates.fetchEmbeddingTemplate).get({ path: result.path });
+            // Check if embedding exists and has the correct structure
+            if (!embResult || typeof embResult.embedding !== 'string')
+                return null;
+            // Parse the embedding
+            const vector = JSON.parse(embResult.embedding);
+            const sim = cosineSimilarity(embedding, vector);
+            // Normalize BM25 scores
+            const normalizedBm25 = 1 - ((result.bm25Score - bm25Min) / (bm25Max - bm25Min + 1e-5));
+            const normalizedSim = sim; // cosineSimilarity is already 0–1
+            const finalScore = 0.7 * normalizedSim + 0.3 * normalizedBm25;
+            return {
+                path: result.path,
+                summary: result.summary,
+                score: finalScore,
+                sim: normalizedSim,
+                bm25: normalizedBm25
+            };
+        }
+        catch (err) {
+            console.error(`Error processing embedding for file: ${result.path}`, err);
+            return null;
+        }
+    }).filter(Boolean)
+        .sort((a, b) => b.score - a.score)
+        .slice(0, topK);
+    return scored;
 }

package/dist/db/schema.js

@@ -7,7 +7,8 @@ export function initSchema() {
       summary TEXT,
       type TEXT,
       indexed_at TEXT,
-      last_modified TEXT
+      last_modified TEXT,
+      embedding TEXT
     );
 
     -- FTS5 table for fast full‑text search of summaries and paths

package/dist/db/sqlTemplates.js

@@ -0,0 +1,37 @@
+// Template for inserting or ignoring new file entries
+export const insertFileTemplate = `
+  INSERT OR IGNORE INTO files
+    (path, summary, type, indexed_at, last_modified)
+  VALUES (:path, :summary, :type, datetime('now'), :lastModified)
+`;
+// Template for updating file metadata if it has changed
+export const updateFileTemplate = `
+  UPDATE files
+  SET type = :type,
+      last_modified = :lastModified,
+      indexed_at = datetime('now')
+  WHERE path = :path
+    AND last_modified != :lastModified
+`;
+// Template for deleting a file from FTS
+export const deleteFromFtsTemplate = `
+  DELETE FROM files_fts
+  WHERE rowid = (SELECT id FROM files WHERE path = :path)
+`;
+// Template for inserting a file into FTS with its ID
+export const insertIntoFtsTemplate = `
+  INSERT INTO files_fts(rowid, path, summary)
+  VALUES((SELECT id FROM files WHERE path = :path), :path, :summary)
+`;
+// Template for fetching BM25 scores from FTS
+export const fetchBm25ScoresTemplate = `
+  SELECT f.path, f.summary, f.type, bm25(files_fts) AS bm25Score
+  FROM files_fts
+  JOIN files f ON files_fts.rowid = f.id
+  WHERE files_fts MATCH :query
+  LIMIT 50
+`;
+// Template for fetching embedding for a specific file
+export const fetchEmbeddingTemplate = `
+  SELECT embedding FROM files WHERE path = :path
+`;

package/dist/index.js

@@ -20,6 +20,7 @@ import { resetDatabase } from './commands/ResetDbCmd.js';
 import { runQueryCommand } from './commands/QueryCmd.js';
 import { runDaemonBatch } from './commands/DaemonCmd.js';
 import { runStopDaemonCommand } from "./commands/StopDaemonCmd.js";
+import { runAskCommand } from './commands/AskCmd.js';
 // Create the CLI instance
 const cmd = new Command('scai')
     .version(version)
@@ -90,6 +91,15 @@ cmd
     .command('query <query>')
     .description('Search indexed files by keyword')
     .action(runQueryCommand);
+// Command structure using Commander
+cmd
+    .command('ask')
+    .description('Ask a question using file summaries and a local model')
+    .argument('<question...>', 'The question to ask')
+    .action((question) => {
+    const q = question.join(' ');
+    runAskCommand(q); // No model option, just pass the question
+});
 cmd
     .command('reset-db')
     .description('Delete and reset the SQLite database')

package/dist/lib/generateEmbedding.js

@@ -0,0 +1,22 @@
+export async function generateEmbedding(text) {
+    try {
+        const res = await fetch('http://localhost:11434/api/embeddings', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                model: 'mistral', // or 'llama3' — whatever works best
+                prompt: text,
+            }),
+        });
+        if (!res.ok) {
+            console.error('❌ Failed to generate embedding:', await res.text());
+            return null;
+        }
+        const data = await res.json();
+        return data.embedding;
+    }
+    catch (err) {
+        console.error('❌ Embedding error:', err instanceof Error ? err.message : err);
+        return null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.19",
+  "version": "0.1.20",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"