scai 0.1.38 → 0.1.39

package/dist/commands/IndexCmd.js

@@ -2,14 +2,14 @@ import fg from 'fast-glob';
 import path from 'path';
 import { initSchema } from '../db/schema.js';
 import { indexFile } from '../db/fileIndex.js';
-import { shouldIgnoreFile } from '../utils/shouldIgnoreFiles.js';
-import { detectFileType } from '../utils/detectFileType.js';
+import { detectFileType } from '../fileRules/detectFileType.js';
 import { startDaemon } from './DaemonCmd.js';
-import { IGNORED_FOLDER_GLOBS } from '../config/IgnoredPaths.js';
+import { IGNORED_FOLDER_GLOBS } from '../fileRules/ignoredPaths.js';
 import { Config } from '../config.js';
 import { DB_PATH } from '../constants.js';
 import { log } from '../utils/log.js';
 import lockfile from 'proper-lockfile';
+import { classifyFile } from '../fileRules/classifyFile.js';
 // 🧠 Lock the database to prevent simultaneous access
 async function lockDb() {
     try {
@@ -55,8 +55,11 @@ export async function runIndexCommand(targetDir, options = {}) {
     let count = 0;
     const release = await lockDb(); // Lock the DB before starting
     for (const file of files) {
-        if (shouldIgnoreFile(file))
+        const classification = classifyFile(file);
+        if (classification !== 'valid') {
+            log(`⏭️ Skipping (${classification}): ${file}`);
             continue;
+        }
         try {
             const type = detectFileType(file);
             indexFile(file, null, type); // Index file without summary

package/dist/commands/MigrateCmd.js

@@ -0,0 +1,15 @@
+// src/commands/MigrateCmd.ts
+import path from 'path';
+import { pathToFileURL } from 'url';
+import { fileURLToPath } from 'url';
+export async function runMigrateCommand() {
+    const scriptPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../dist/scripts/migrateDb.js' // compiled output
+    );
+    try {
+        await import(pathToFileURL(scriptPath).href);
+    }
+    catch (err) {
+        console.error('❌ Failed to run migration script:', err);
+        process.exit(1);
+    }
+}

package/dist/commands/ModulePipelineCmd.js

@@ -1,5 +1,5 @@
 import { readFileSync } from 'fs';
-import { getModuleByName } from '../registry/moduleRegistry.js';
+import { getModuleByName } from '../pipeline/registry/moduleRegistry.js';
 import { runModulePipeline } from '../pipeline/runModulePipeline.js';
 export async function runModulePipelineFromCLI(file, options) {
     if (!options.modules) {

package/dist/commands/SummaryCmd.js

@@ -4,7 +4,7 @@ import readline from 'readline';
 import { queryFiles, indexFile } from '../db/fileIndex.js';
 import { summaryModule } from '../pipeline/modules/summaryModule.js';
 import { summarizeCode } from '../utils/summarizer.js';
-import { detectFileType } from '../utils/detectFileType.js';
+import { detectFileType } from '../fileRules/detectFileType.js';
 import { generateEmbedding } from '../lib/generateEmbedding.js';
 import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
 import { db } from '../db/client.js';

package/dist/daemon/daemonBatch.js

@@ -6,9 +6,8 @@ import { generateEmbedding } from '../lib/generateEmbedding.js';
 import { DB_PATH } from '../constants.js';
 import { log } from '../utils/log.js';
 import lockfile from 'proper-lockfile';
-import { shouldIgnoreFile } from '../utils/shouldIgnoreFiles.js';
 import { summaryModule } from '../pipeline/modules/summaryModule.js';
-import { isGeneratedOrBundledFile } from '../utils/fileClassifier.js';
+import { classifyFile } from '../fileRules/classifyFile.js';
 const MAX_FILES_PER_BATCH = 5;
 async function lockDb() {
     try {
@@ -22,14 +21,14 @@ async function lockDb() {
 export async function runDaemonBatch() {
     log('🟡 Starting daemon batch...');
     const rows = db.prepare(`
-    SELECT path, type, functions_extracted FROM files
+    SELECT path, type, processing_status FROM files
     WHERE (summary IS NULL OR summary = '') 
-      OR (functions_extracted IS NULL OR functions_extracted = 0)
+      OR processing_status = 'unprocessed'
     ORDER BY last_modified DESC
     LIMIT ?
   `).all(MAX_FILES_PER_BATCH);
     if (rows.length === 0) {
-        log('✅ No files left to summarize.');
+        log('✅ No files left to process.');
         return false;
     }
     const release = await lockDb();
@@ -37,45 +36,37 @@ export async function runDaemonBatch() {
         log(`📂 Processing file: ${row.path}`);
         if (!fsSync.existsSync(row.path)) {
             log(`⚠️ Skipped missing file: ${row.path}`);
+            db.prepare(`UPDATE files SET processing_status = 'skipped' WHERE path = @path`).run({ path: row.path });
             continue;
         }
-        if (shouldIgnoreFile(row.path)) {
-            log(`⚠️ Skipped (extension): ${row.path}`);
-            continue;
-        }
-        if (isGeneratedOrBundledFile(row.path)) {
-            log(`🚫 Skipping likely generated file: ${row.path}`);
+        const classification = classifyFile(row.path);
+        if (classification !== 'valid') {
+            log(`⏭️ Skipping (${classification}): ${row.path}`);
+            db.prepare(`UPDATE files SET processing_status = 'skipped' WHERE path = @path`).run({ path: row.path });
             continue;
         }
         try {
             const content = await fs.readFile(row.path, 'utf-8');
-            // 1. Lookup file ID in the database using its path
-            const fileResult = db
-                .prepare(`SELECT id FROM files WHERE path = @path`) // Replaced `?` with `@path`
-                .get({ path: row.path });
+            const fileResult = db.prepare(`SELECT id FROM files WHERE path = @path`).get({ path: row.path });
             const fileId = fileResult?.id;
             if (fileId) {
-                // 2. Extract and index functions for this file
                 await indexFunctionsForFile(row.path, fileId);
-                // 3. Mark the file as having functions extracted in the database
-                db.prepare(`UPDATE files SET functions_extracted = 1 WHERE id = @id`).run({ id: fileId });
+                // Update processing_status to 'extracted' after function extraction attempt
+                db.prepare(`UPDATE files SET processing_status = 'extracted' WHERE id = @id`).run({ id: fileId });
             }
             else {
                 log(`⚠️ Could not find fileId for ${row.path}`);
             }
-            // 4. Summarize the file content
             log(`📝 Generating summary for ${row.path}...`);
             const summaryResult = await summaryModule.run({ content, filepath: row.path });
             const summary = summaryResult?.summary?.trim() || null;
             let embedding = null;
             if (summary) {
-                // 5. Generate embedding for the summary
                 const vector = await generateEmbedding(summary);
                 if (vector) {
                     embedding = JSON.stringify(vector);
                 }
             }
-            // 6. Save the summary and embedding in the database
             db.prepare(`
         UPDATE files
         SET summary = @summary, embedding = @embedding, indexed_at = datetime('now')
@@ -84,10 +75,9 @@ export async function runDaemonBatch() {
             log(`✅ Updated summary & embedding for ${row.path}\n`);
         }
         catch (err) {
-            // 7. Error handling: Log the error message if the process fails
             log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}\n`);
+            db.prepare(`UPDATE files SET processing_status = 'failed' WHERE path = @path`).run({ path: row.path });
         }
-        // 8. Optional delay between file processing to reduce load on the system
         await new Promise(resolve => setTimeout(resolve, 200));
     }
     await release();

package/dist/db/functionExtractors/extractFromJs.js

@@ -78,12 +78,11 @@ export async function extractFromJS(filePath, content, fileId) {
         }
         log(`📌 Indexed function: ${fn.name} with ${calls.length} calls`);
     }
-    // Mark the file as successfully extracted
+    // ✅ Mark as extracted using new processing_status column
     db.prepare(`
     UPDATE files 
-    SET functions_extracted = 1,
-        functions_extracted_at = datetime('now')
+    SET processing_status = 'extracted'
     WHERE id = @fileId
-`).run({ fileId });
+  `).run({ fileId });
     log(`✅ Marked functions as extracted for ${filePath}`);
 }

package/dist/db/functionExtractors/index.js

@@ -1,9 +1,9 @@
 import { log } from '../../utils/log.js';
-import { detectFileType } from '../../utils/detectFileType.js';
+import { detectFileType } from '../../fileRules/detectFileType.js';
 import { extractFromJava } from './extractFromJava.js';
 import { extractFromJS } from './extractFromJs.js';
 import { extractFromXML } from './extractFromXML.js';
-import { db } from '../client.js'; // Assuming db is imported from your DB utility
+import { db } from '../client.js';
 /**
  * Detects file type and delegates to the appropriate extractor.
  */
@@ -13,42 +13,36 @@ export async function extractFunctionsFromFile(filePath, content, fileId) {
         if (type === 'js' || type === 'ts' || type === 'javascript' || type === 'typescript') {
             log(`✅ Attempting to extract JS functions from ${filePath}\n`);
             await extractFromJS(filePath, content, fileId);
-            // Mark the file as "extracted" even if the extraction succeeds
-            db.prepare(`
-        UPDATE files SET functions_extracted = 1 WHERE id = @id
-      `).run({ id: fileId });
             return true;
         }
         if (type === 'java') {
             log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
             await extractFromJava(filePath, content, fileId);
-            // Mark the file as "extracted" after the attempt
+            // move into extract file
             db.prepare(`
-        UPDATE files SET functions_extracted = 1 WHERE id = @id
+        UPDATE files SET processing_status = 'skipped' WHERE id = @id
       `).run({ id: fileId });
             return false;
         }
         if (type === 'xml') {
             log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
             await extractFromXML(filePath, content, fileId);
-            // Mark the file as "extracted" after the attempt
+            // move into extract file
             db.prepare(`
-        UPDATE files SET functions_extracted = 1 WHERE id = @id
+        UPDATE files SET processing_status = 'skipped' WHERE id = @id
       `).run({ id: fileId });
             return false;
         }
         log(`⚠️ Unsupported file type: ${type} for function extraction. Skipping ${filePath}`);
-        // Mark unsupported file types as extracted to prevent future retries
         db.prepare(`
-      UPDATE files SET functions_extracted = 1 WHERE id = @id
+      UPDATE files SET processing_status = 'skipped' WHERE id = @id
     `).run({ id: fileId });
         return false;
     }
     catch (error) {
         log(`❌ Failed to extract functions from ${filePath}: ${error instanceof Error ? error.message : error}`);
-        // Mark as extracted on failure to ensure the system keeps moving forward
         db.prepare(`
-      UPDATE files SET functions_extracted = 1 WHERE id = @id
+      UPDATE files SET processing_status = 'failed' WHERE id = @id
     `).run({ id: fileId });
         return false;
     }

package/dist/db/schema.js

@@ -8,7 +8,8 @@ export function initSchema() {
       type TEXT,
       indexed_at TEXT,
       last_modified TEXT,
-      embedding TEXT
+      embedding TEXT,
+      processing_status TEXT
     );
 
     CREATE VIRTUAL TABLE IF NOT EXISTS files_fts

package/dist/fileRules/classifyFile.js

@@ -0,0 +1,10 @@
+// src/fileRules/classifyFile.ts
+import { isGeneratedOrBundledFile } from "./fileClassifier.js";
+import { shouldIgnoreFile } from "./shouldIgnoreFiles.js";
+export function classifyFile(filepath) {
+    if (shouldIgnoreFile(filepath))
+        return 'ignore';
+    if (isGeneratedOrBundledFile(filepath))
+        return 'generated';
+    return 'valid';
+}

package/dist/{utils → fileRules}/shouldIgnoreFiles.js

@@ -1,6 +1,6 @@
 import path from 'path';
-import { IGNORED_EXTENSIONS } from '../config/IgnoredExtensions.js';
-import { specificFileExceptions } from '../config/specificFileExceptions.js';
+import { IGNORED_EXTENSIONS } from './ignoredExtensions.js';
+import { specificFileExceptions } from './fileExceptions.js';
 export function shouldIgnoreFile(filePath) {
     // Get file extension
     const ext = path.extname(filePath).toLowerCase();

package/dist/index.js

@@ -20,6 +20,7 @@ import { startDaemon } from './commands/DaemonCmd.js';
 import { runStopDaemonCommand } from "./commands/StopDaemonCmd.js";
 import { runAskCommand } from './commands/AskCmd.js';
 import { runBackupCommand } from './commands/BackupCmd.js';
+import { runMigrateCommand } from "./commands/MigrateCmd.js";
 // 🎛️ CLI Setup
 const cmd = new Command('scai')
     .version(version)
@@ -133,6 +134,10 @@ cmd
     .command('stop-daemon')
     .description('Stop the background summarizer daemon')
     .action(runStopDaemonCommand);
+cmd
+    .command('migrate')
+    .description('Run DB migration scripts')
+    .action(runMigrateCommand);
 cmd
     .command('reset-db')
     .description('Delete and reset the SQLite database')

package/dist/pipeline/registry/moduleRegistry.js

@@ -0,0 +1,21 @@
+import { addCommentsModule } from '../modules/commentModule.js';
+import { cleanupModule } from '../modules/cleanupModule.js';
+import { summaryModule } from '../modules/summaryModule.js';
+import { generateTestsModule } from '../modules/generateTestsModule.js';
+import { commitSuggesterModule } from '../modules/commitSuggesterModule.js';
+import { changelogModule } from '../modules/changeLogModule.js';
+// Add more as needed...
+const builtInModules = {
+    comments: addCommentsModule,
+    cleanup: cleanupModule,
+    summary: summaryModule,
+    tests: generateTestsModule,
+    suggest: commitSuggesterModule,
+    changelog: changelogModule,
+};
+export function getModuleByName(name) {
+    return builtInModules[name];
+}
+export function listAvailableModules() {
+    return Object.keys(builtInModules);
+}

package/dist/scripts/migrateDb.js

@@ -0,0 +1,18 @@
+// scripts/migrateDb.ts
+import { db } from "../db/client.js";
+import { log } from "../utils/log.js";
+try {
+    db.prepare(`
+    ALTER TABLE files ADD COLUMN processing_status TEXT DEFAULT 'unprocessed'
+  `).run();
+    log('✅ Added processing_status column.');
+}
+catch (e) {
+    if (e instanceof Error && e.message.includes('duplicate column name')) {
+        log('ℹ️ Column processing_status already exists. Skipping.');
+    }
+    else {
+        log(`❌ Migration failed: ${e instanceof Error ? e.message : e}`);
+    }
+}
+// Add more migration steps here as needed.

package/dist/utils/sanitizeQuery.js

@@ -1,5 +1,5 @@
 // src/utils/sanitizeQuery.ts
-import { STOP_WORDS } from '../config/StopWords.js';
+import { STOP_WORDS } from '../fileRules/stopWords.js';
 export function sanitizeQueryForFts(input) {
     input = input.trim().toLowerCase();
     // If it's a single filename-like string (includes dots or slashes), quote it

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.38",
+  "version": "0.1.39",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"

package/dist/registry/moduleRegistry.js

@@ -1,21 +0,0 @@
-import { addCommentsModule } from '../pipeline/modules/commentModule.js';
-import { cleanupModule } from '../pipeline/modules/cleanupModule.js';
-import { summaryModule } from '../pipeline/modules/summaryModule.js';
-import { generateTestsModule } from '../pipeline/modules/generateTestsModule.js';
-import { commitSuggesterModule } from '../pipeline/modules/commitSuggesterModule.js';
-import { changelogModule } from '../pipeline/modules/changeLogModule.js';
-// Add more as needed...
-const builtInModules = {
-    comments: addCommentsModule,
-    cleanup: cleanupModule,
-    summary: summaryModule,
-    tests: generateTestsModule,
-    suggest: commitSuggesterModule,
-    changelog: changelogModule,
-};
-export function getModuleByName(name) {
-    return builtInModules[name];
-}
-export function listAvailableModules() {
-    return Object.keys(builtInModules);
-}

package/dist/scripts/addFunctionsExtractedColumn.js

@@ -1,29 +0,0 @@
-import { db } from '../db/client.js';
-try {
-    db.prepare(`
-    ALTER TABLE files ADD COLUMN functions_extracted BOOLEAN DEFAULT 0
-  `).run();
-    console.log('✅ Added functions_extracted column.');
-}
-catch (e) {
-    if (e instanceof Error ? e.message.includes('duplicate column name') : e) {
-        console.log('ℹ️ Column functions_extracted already exists. Skipping.');
-    }
-    else {
-        console.error('❌ Migration failed:', e instanceof Error ? e.message : e);
-    }
-}
-try {
-    db.prepare(`
-    ALTER TABLE files ADD COLUMN functions_extracted_at DATETIME
-  `).run();
-    console.log('✅ Added functions_extracted_at column.');
-}
-catch (e) {
-    if (e instanceof Error ? e.message.includes('duplicate column name') : e) {
-        console.log('ℹ️ Column functions_extracted_at already exists. Skipping.');
-    }
-    else {
-        console.error('❌ Migration failed:', e instanceof Error ? e.message : e);
-    }
-}

package/dist/utils/removeIgnoredFiles.js

@@ -1,43 +0,0 @@
-import Database from 'better-sqlite3';
-import path from 'path';
-import os from 'os';
-import { IGNORED_EXTENSIONS } from '../config/IgnoredExtensions.js';
-import { specificFileExceptions } from '../config/specificFileExceptions.js';
-// THIS FILE IS MEANT TO BE RUN AS A NODE JS SCRIPT. node dist/src/utilsremoveIgnoredFiles.js
-// It removes wrongly indexed files that don't add value to the model. 
-const DB_PATH = path.join(os.homedir(), '.scai', 'db.sqlite');
-const db = new Database(DB_PATH);
-console.log('🧹 Removing files with ignored extensions from the database...');
-// === Remove Files with Ignored Extensions, Excluding Specific Exceptions ===
-IGNORED_EXTENSIONS.forEach(ext => {
-    try {
-        const filesToDelete = db.prepare(`
-      SELECT path FROM files WHERE path LIKE ?
-    `).all(`%${ext}`);
-        ;
-        let deletedCount = 0;
-        filesToDelete.forEach(file => {
-            // Check if the file is in the exception list
-            if (!specificFileExceptions.includes(file.path)) {
-                // Delete the file from the database
-                const deleted = db.prepare(`DELETE FROM files WHERE path = ?`).run(file.path);
-                if (deleted.changes > 0) {
-                    deletedCount++;
-                }
-            }
-            else {
-                console.log(`⚠️ Skipped file (exception): ${file.path}`);
-            }
-        });
-        if (deletedCount > 0) {
-            console.log(`✅ Removed ${deletedCount} files with extension: ${ext}`);
-        }
-        else {
-            console.log(`⚠️ No deletions for files with extension: ${ext}`);
-        }
-    }
-    catch (err) {
-        console.error("❌ Failed to remove files with extension ${ext}:", err instanceof Error ? err.message : err);
-    }
-});
-console.log('🧹 Finished removing ignored files.');