scai 0.1.36 → 0.1.38

package/dist/daemon/daemonBatch.js

@@ -8,6 +8,7 @@ import { log } from '../utils/log.js';
 import lockfile from 'proper-lockfile';
 import { shouldIgnoreFile } from '../utils/shouldIgnoreFiles.js';
 import { summaryModule } from '../pipeline/modules/summaryModule.js';
+import { isGeneratedOrBundledFile } from '../utils/fileClassifier.js';
 const MAX_FILES_PER_BATCH = 5;
 async function lockDb() {
     try {
@@ -42,30 +43,39 @@ export async function runDaemonBatch() {
             log(`⚠️ Skipped (extension): ${row.path}`);
             continue;
         }
+        if (isGeneratedOrBundledFile(row.path)) {
+            log(`🚫 Skipping likely generated file: ${row.path}`);
+            continue;
+        }
         try {
             const content = await fs.readFile(row.path, 'utf-8');
-            // 1. Extract functions and function calls
+            // 1. Lookup file ID in the database using its path
             const fileResult = db
-                .prepare(`SELECT id FROM files WHERE path = ?`)
-                .get(row.path);
+                .prepare(`SELECT id FROM files WHERE path = @path`) // Replaced `?` with `@path`
+                .get({ path: row.path });
             const fileId = fileResult?.id;
-            if (!fileId) {
-                log(`⚠️ Could not find fileId for ${row.path}`);
+            if (fileId) {
+                // 2. Extract and index functions for this file
+                await indexFunctionsForFile(row.path, fileId);
+                // 3. Mark the file as having functions extracted in the database
+                db.prepare(`UPDATE files SET functions_extracted = 1 WHERE id = @id`).run({ id: fileId });
             }
             else {
-                await indexFunctionsForFile(row.path, fileId);
+                log(`⚠️ Could not find fileId for ${row.path}`);
             }
-            // 2. Summarize the file 
+            // 4. Summarize the file content
             log(`📝 Generating summary for ${row.path}...`);
             const summaryResult = await summaryModule.run({ content, filepath: row.path });
             const summary = summaryResult?.summary?.trim() || null;
             let embedding = null;
             if (summary) {
+                // 5. Generate embedding for the summary
                 const vector = await generateEmbedding(summary);
                 if (vector) {
                     embedding = JSON.stringify(vector);
                 }
             }
+            // 6. Save the summary and embedding in the database
             db.prepare(`
         UPDATE files
         SET summary = @summary, embedding = @embedding, indexed_at = datetime('now')
@@ -74,8 +84,10 @@ export async function runDaemonBatch() {
             log(`✅ Updated summary & embedding for ${row.path}\n`);
         }
         catch (err) {
+            // 7. Error handling: Log the error message if the process fails
             log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}\n`);
         }
+        // 8. Optional delay between file processing to reduce load on the system
         await new Promise(resolve => setTimeout(resolve, 200));
     }
     await release();

package/dist/db/functionExtractors/index.js

@@ -3,27 +3,53 @@ import { detectFileType } from '../../utils/detectFileType.js';
 import { extractFromJava } from './extractFromJava.js';
 import { extractFromJS } from './extractFromJs.js';
 import { extractFromXML } from './extractFromXML.js';
+import { db } from '../client.js'; // Assuming db is imported from your DB utility
 /**
  * Detects file type and delegates to the appropriate extractor.
  */
 export async function extractFunctionsFromFile(filePath, content, fileId) {
     const type = detectFileType(filePath).trim().toLowerCase();
-    ;
-    if (type === 'js' || type === 'ts') {
-        log(`✅ Attempting to extract JS functions from ${filePath}\n`);
-        await extractFromJS(filePath, content, fileId);
-        return true;
-    }
-    if (type === 'java') {
-        log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
-        await extractFromJava(filePath, content, fileId);
+    try {
+        if (type === 'js' || type === 'ts' || type === 'javascript' || type === 'typescript') {
+            log(`✅ Attempting to extract JS functions from ${filePath}\n`);
+            await extractFromJS(filePath, content, fileId);
+            // Mark the file as "extracted" even if the extraction succeeds
+            db.prepare(`
+        UPDATE files SET functions_extracted = 1 WHERE id = @id
+      `).run({ id: fileId });
+            return true;
+        }
+        if (type === 'java') {
+            log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
+            await extractFromJava(filePath, content, fileId);
+            // Mark the file as "extracted" after the attempt
+            db.prepare(`
+        UPDATE files SET functions_extracted = 1 WHERE id = @id
+      `).run({ id: fileId });
+            return false;
+        }
+        if (type === 'xml') {
+            log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
+            await extractFromXML(filePath, content, fileId);
+            // Mark the file as "extracted" after the attempt
+            db.prepare(`
+        UPDATE files SET functions_extracted = 1 WHERE id = @id
+      `).run({ id: fileId });
+            return false;
+        }
+        log(`⚠️ Unsupported file type: ${type} for function extraction. Skipping ${filePath}`);
+        // Mark unsupported file types as extracted to prevent future retries
+        db.prepare(`
+      UPDATE files SET functions_extracted = 1 WHERE id = @id
+    `).run({ id: fileId });
         return false;
     }
-    if (type === 'xml') {
-        log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
-        await extractFromXML(filePath, content, fileId);
+    catch (error) {
+        log(`❌ Failed to extract functions from ${filePath}: ${error instanceof Error ? error.message : error}`);
+        // Mark as extracted on failure to ensure the system keeps moving forward
+        db.prepare(`
+      UPDATE files SET functions_extracted = 1 WHERE id = @id
+    `).run({ id: fileId });
         return false;
     }
-    log(`⚠️ Unsupported file type: ${type} for function extraction. Skipping ${filePath}`);
-    return false;
 }

package/dist/utils/fileClassifier.js

@@ -0,0 +1,8 @@
+import path from "path";
+// utils/fileClassifier.ts
+export function isGeneratedOrBundledFile(filePath) {
+    const base = path.basename(filePath);
+    const isHashNamed = /[-_.](worker|bundle|chunk|[a-f0-9]{6,})\.(js|ts)$/.test(base);
+    const isInOutputFolder = /[\\/]dist[\\/]|[\\/]build[\\/]|[\\/]assets[\\/]|[\\/]node_modules[\\/]/i.test(filePath);
+    return isHashNamed || isInOutputFolder;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.36",
+  "version": "0.1.38",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"