scai 0.1.108 → 0.1.110

package/dist/CHANGELOG.md

@@ -166,4 +166,27 @@ Type handling with the module pipeline
 
 ## 2025-09-01
 
-* Improve handling of GitHub repository URLs and paths by extracting the owner and name separately
+* Improve handling of GitHub repository URLs and paths by extracting the owner and name separately
+
+## 2025-09-02
+
+• Added test configuration for project and generated tests
+• Add runTestsModule and repairTestsModule for testing pipeline
+
+## 2025-09-05
+
+• Enable execution of files as executable files in the scripts
+• Remove context failure if models not installed
+• Add ability to set global model
+
+## 2025-09-08
+
+### Requires DB reset ('scai db reset' followed by 'scai index start')
+
+1. Improved daemon batch processing by skipping missing files, classifying unknown file types, and persisting entities/tags in the database.
+2. Invoke kgModule in daemonBatch to build knowledge graphs after indexing.
+3. Improved data modeling and extraction logic for functions and classes in TypeScript files.
+4. Updated Edge/Table schema for better query performance.
+5. Update package-lock.json to caniuse-lite@1.0.30001741.
+6. Enable execution of  as an executable file in the scripts.
+7. Remove context failure if models not installed. Add ability to set global model.

package/dist/__tests__/example.test.js

@@ -0,0 +1,10 @@
+import { SCAI_HOME } from "../constants"; // example constant
+// cli/src/__tests__/example.test.ts
+describe("CLI src basic test", () => {
+    it("should pass a simple truthy test", () => {
+        expect(true).toBe(true);
+    });
+    it("should import a constant from cli/src/constants.ts", () => {
+        expect(typeof SCAI_HOME).not.toBe("undefined");
+    });
+});

package/dist/agent/agentManager.js

@@ -18,8 +18,20 @@ export class Agent {
         // Resolve modules (with before/after dependencies)
         const modules = this.resolveModules(this.goals);
         console.log(chalk.green("📋 Modules to run:"), modules.map((m) => m.name).join(" → "));
-        // Read file content (optional, could be used by modules in workflow)
-        await fs.readFile(filepath, "utf-8");
+        try {
+            // Check that the file exists before trying to read it
+            await fs.access(filepath);
+            // Read file content (optional, could be used by modules in workflow)
+            await fs.readFile(filepath, "utf-8");
+        }
+        catch (err) {
+            if (err.code === "ENOENT") {
+                console.error(chalk.redBright("❌ Error:"), `File not found: ${chalk.yellow(filepath)}`);
+                console.error(`Make sure the path is correct. (cwd: ${chalk.gray(process.cwd())})`);
+                process.exit(1);
+            }
+            throw err; // rethrow for unexpected errors
+        }
         // Delegate everything to handleAgentRun (like CLI commands do)
         await handleAgentRun(filepath, modules);
         console.log(chalk.green("✅ Agent finished!"));

package/dist/agent/workflowManager.js

@@ -76,6 +76,11 @@ export async function handleAgentRun(filepath, modules) {
                     baseChunks.length = 0;
                     baseChunks.push(...reset);
                     break;
+                case 'skip':
+                    console.log(chalk.gray(`⏭️ Skipped writing for module ${mod.name}`));
+                    // don’t touch files, but keep chunks flowing
+                    workingChunks = processed;
+                    break;
                 default:
                     console.log(chalk.yellow(`⚠️ Unknown mode; skipping write`));
                     // still move pipeline forward with processed

package/dist/commands/DaemonCmd.js

@@ -23,9 +23,11 @@ export async function startDaemon() {
     const __filename = fileURLToPath(import.meta.url);
     const __dirname = path.dirname(__filename);
     const daemonWorkerPath = path.join(__dirname, '../daemon/daemonWorker.js');
+    const out = fsSync.openSync(LOG_PATH, 'a');
+    const err = fsSync.openSync(LOG_PATH, 'a');
     const child = spawn(process.execPath, [daemonWorkerPath], {
         detached: true,
-        stdio: ['ignore', 'ignore', 'ignore'],
+        stdio: ['ignore', out, err], // stdout/stderr -> log file
         env: {
             ...process.env,
             BACKGROUND_MODE: 'true',

package/dist/config.js

@@ -6,7 +6,7 @@ import { normalizePath } from './utils/contentUtils.js';
 import chalk from 'chalk';
 import { getHashedRepoKey } from './utils/repoKey.js';
 const defaultConfig = {
-    model: 'codellama:13b',
+    model: 'llama3:8b',
     contextLength: 4096,
     language: 'ts',
     indexDir: '',
@@ -55,18 +55,23 @@ export const Config = {
         const repoCfg = cfg.repos?.[cfg.activeRepo ?? ''];
         return repoCfg?.model || cfg.model;
     },
-    setModel(model) {
+    setModel(model, scope = 'repo') {
         const cfg = readConfig();
-        const active = cfg.activeRepo;
-        if (active) {
+        if (scope === 'repo') {
+            const active = cfg.activeRepo;
+            if (!active) {
+                console.error("❌ No active repo to set model for.");
+                return;
+            }
             cfg.repos[active] = { ...cfg.repos[active], model };
-            writeConfig(cfg);
-            console.log(`📦 Model set to: ${model}`);
+            console.log(`📦 Model set for repo '${active}': ${model}`);
         }
         else {
-            writeConfig({ model });
-            console.log(`📦 Default model set to: ${model}`);
+            // Set global default model
+            cfg.model = model;
+            console.log(`📦 Global default model set to: ${model}`);
         }
+        writeConfig(cfg);
     },
     getLanguage() {
         const cfg = readConfig();

package/dist/context.js

@@ -5,12 +5,25 @@ import { getHashedRepoKey } from "./utils/repoKey.js";
 import { getDbForRepo, getDbPathForRepo } from "./db/client.js";
 import fs from "fs";
 import chalk from "chalk";
+import { execSync } from "child_process";
+function modelExists(model) {
+    try {
+        const output = execSync("ollama list", { encoding: "utf-8" });
+        return output
+            .split("\n")
+            .map(line => line.trim())
+            .filter(Boolean)
+            .some(line => line.toLowerCase().startsWith(model.toLowerCase() + " ") || line.toLowerCase() === model.toLowerCase());
+    }
+    catch (err) {
+        console.error(chalk.red("❌ Failed to check models with `ollama list`"));
+        return false;
+    }
+}
 export async function updateContext() {
     const cwd = normalizePath(process.cwd());
     const cfg = readConfig();
-    // 🔑 Find repoKey by matching indexDir to cwd
     let repoKey = Object.keys(cfg.repos || {}).find((key) => normalizePath(cfg.repos[key]?.indexDir || "") === cwd);
-    // Initialize new repo config if not found
     let isNewRepo = false;
     if (!repoKey) {
         repoKey = getHashedRepoKey(cwd);
@@ -19,28 +32,23 @@ export async function updateContext() {
         cfg.repos[repoKey].indexDir = cwd;
         isNewRepo = true;
     }
-    // Check if active repo has changed
     const activeRepoChanged = cfg.activeRepo !== repoKey;
-    // Always set this as active repo
     cfg.activeRepo = repoKey;
     writeConfig(cfg);
     const repoCfg = cfg.repos[repoKey];
     let ok = true;
-    // Only log detailed info if new repo or active repo changed
     if (isNewRepo || activeRepoChanged) {
         console.log(chalk.yellow("\n🔁 Updating context...\n"));
         console.log(`✅ Active repo: ${chalk.green(repoKey)}`);
         console.log(`✅ Index dir: ${chalk.cyan(repoCfg.indexDir || cwd)}`);
     }
-    // GitHub token is optional
     const token = repoCfg.githubToken || cfg.githubToken;
     if (!token) {
-        console.log(`ℹ️ No GitHub token found. You can set one with the: ${chalk.bold(chalk.bgGreen("scai auth set"))} command`);
+        console.log(`ℹ️ No GitHub token found. You can set one with: ${chalk.bold(chalk.bgGreen("scai auth set"))}`);
     }
     else if (isNewRepo || activeRepoChanged) {
         console.log(`✅ GitHub token present`);
     }
-    // Ensure DB exists
     const dbPath = getDbPathForRepo();
     if (!fs.existsSync(dbPath)) {
         console.log(chalk.yellow(`📦 Initializing DB at ${dbPath}`));
@@ -48,13 +56,31 @@ export async function updateContext() {
             getDbForRepo();
         }
         catch {
-            ok = false; // DB init failed
+            ok = false;
         }
     }
     else if (isNewRepo || activeRepoChanged) {
         console.log(chalk.green("✅ Database present"));
     }
-    // Final context status
+    // 🧠 Model check
+    const model = cfg.model;
+    if (!model) {
+        console.log(chalk.red("❌ No model configured.") +
+            "\n➡️ Set one with: " +
+            chalk.bold(chalk.bgGreen("scai config set-model <model>")));
+        ok = false;
+    }
+    else if (!modelExists(model)) {
+        console.log(chalk.red(`❌ Model '${model}' not installed in Ollama.`) +
+            "\n➡️ Install with: " +
+            chalk.bold(chalk.yellow(`ollama pull ${model}`)) +
+            " or choose another with: " +
+            chalk.bold(chalk.yellow("scai config set-model <model>")));
+        ok = false;
+    }
+    else {
+        console.log(chalk.green(`✅ Model '${model}' available`));
+    }
     if (ok) {
         console.log(chalk.bold.green("\n✅ Context OK\n"));
     }

package/dist/daemon/daemonBatch.js

@@ -1,4 +1,4 @@
-import { indexFunctionsForFile } from '../db/functionIndex.js';
+import { indexCodeForFile } from '../db/functionIndex.js';
 import fs from 'fs/promises';
 import fsSync from 'fs';
 import { generateEmbedding } from '../lib/generateEmbedding.js';
@@ -8,6 +8,7 @@ import { summaryModule } from '../pipeline/modules/summaryModule.js';
 import { classifyFile } from '../fileRules/classifyFile.js';
 import { getDbForRepo, getDbPathForRepo } from '../db/client.js';
 import { markFileAsSkippedByPath, selectUnprocessedFiles, updateFileWithSummaryAndEmbedding, } from '../db/sqlTemplates.js';
+import { kgModule } from '../pipeline/modules/kgModule.js';
 const MAX_FILES_PER_BATCH = 5;
 /**
  * Acquires a lock on the database to ensure that only one daemon batch
@@ -32,7 +33,6 @@ async function lockDb() {
  */
 export async function runDaemonBatch() {
     log('🟡 Starting daemon batch...');
-    // Selects up to MAX_FILES_PER_BATCH files that haven't been processed yet
     const db = getDbForRepo();
     const rows = db.prepare(selectUnprocessedFiles).all(MAX_FILES_PER_BATCH);
     if (rows.length === 0) {
@@ -42,13 +42,11 @@ export async function runDaemonBatch() {
     const release = await lockDb();
     for (const row of rows) {
         log(`📂 Processing file: ${row.path}`);
-        // Skip if file is missing from the file system
         if (!fsSync.existsSync(row.path)) {
             log(`⚠️ Skipped missing file: ${row.path}`);
             db.prepare(markFileAsSkippedByPath).run({ path: row.path });
             continue;
         }
-        // Skip if file is classified as something we don't process
         const classification = classifyFile(row.path);
         if (classification !== 'valid') {
             log(`⏭️ Skipping (${classification}): ${row.path}`);
@@ -57,24 +55,20 @@ export async function runDaemonBatch() {
         }
         try {
             const content = await fs.readFile(row.path, 'utf-8');
-            // Determine whether the file needs to be re-summarized
             const needsResummary = !row.summary ||
                 !row.indexed_at ||
                 (row.last_modified && new Date(row.last_modified) > new Date(row.indexed_at));
             if (needsResummary) {
                 log(`📝 Generating summary for ${row.path}...`);
-                // Generate a summary using the summary pipeline
                 const summaryResult = await summaryModule.run({ content, filepath: row.path });
                 const summary = summaryResult?.summary?.trim() || null;
                 let embedding = null;
-                // Generate an embedding from the summary (if present)
                 if (summary) {
                     const vector = await generateEmbedding(summary);
                     if (vector) {
                         embedding = JSON.stringify(vector);
                     }
                 }
-                // Update the file record with the new summary and embedding
                 db.prepare(updateFileWithSummaryAndEmbedding).run({
                     summary,
                     embedding,
@@ -85,19 +79,79 @@ export async function runDaemonBatch() {
             else {
                 log(`⚡ Skipped summary (up-to-date) for ${row.path}`);
             }
-            // Extract top-level functions from the file and update the DB
-            const extracted = await indexFunctionsForFile(row.path, row.id);
-            if (extracted) {
-                log(`✅ Function extraction complete for ${row.path}\n`);
+            const success = await indexCodeForFile(row.path, row.id);
+            if (success) {
+                log(`✅ Indexed code for ${row.path}`);
+                try {
+                    log(`🔗 Building Knowledge Graph for ${row.path}...`);
+                    const kgInput = {
+                        fileId: row.id,
+                        filepath: row.path,
+                        summary: row.summary || undefined,
+                    };
+                    const kgResult = await kgModule.run(kgInput, content);
+                    log(`✅ Knowledge Graph built for ${row.path}`);
+                    log(`Entities: ${kgResult.entities.length}, Edges: ${kgResult.edges.length}`);
+                    // Persist KG entities + tags only if there are any
+                    if (kgResult.entities.length > 0) {
+                        const insertTag = db.prepare(`
+    INSERT OR IGNORE INTO tags_master (name) VALUES (:name)
+  `);
+                        const getTagId = db.prepare(`
+    SELECT id FROM tags_master WHERE name = :name
+  `);
+                        const insertEntityTag = db.prepare(`
+    INSERT OR IGNORE INTO entity_tags (entity_type, entity_id, tag_id)
+    VALUES (:entity_type, :entity_id, :tag_id)
+  `);
+                        for (const entity of kgResult.entities) {
+                            // Skip entity if type or tags are missing
+                            if (!entity.type || !Array.isArray(entity.tags) || entity.tags.length === 0) {
+                                console.warn(`⚠ Skipping entity due to missing type or tags:`, entity);
+                                continue;
+                            }
+                            for (const tag of entity.tags) {
+                                // Skip empty or invalid tags
+                                if (!tag || typeof tag !== 'string') {
+                                    console.warn(`⚠ Skipping invalid tag for entity ${entity.type}:`, tag);
+                                    continue;
+                                }
+                                try {
+                                    // ✅ Use :name in SQL and plain key in object
+                                    insertTag.run({ name: tag });
+                                    const tagRow = getTagId.get({ name: tag });
+                                    if (!tagRow) {
+                                        console.warn(`⚠ Could not find tag ID for: ${tag}`);
+                                        continue;
+                                    }
+                                    insertEntityTag.run({
+                                        entity_type: entity.type,
+                                        entity_id: row.id,
+                                        tag_id: tagRow.id,
+                                    });
+                                }
+                                catch (err) {
+                                    console.error(`❌ Failed to persist entity/tag:`, { entity, tag, error: err });
+                                }
+                            }
+                        }
+                        log(`✅ Persisted entities + tags for ${row.path}`);
+                    }
+                    else {
+                        log(`⚠️ No entities found for ${row.path}, skipping DB inserts`);
+                    }
+                }
+                catch (kgErr) {
+                    log(`❌ KG build failed for ${row.path}: ${kgErr instanceof Error ? kgErr.message : String(kgErr)}`);
+                }
             }
             else {
-                log(`ℹ️ No functions extracted for ${row.path}\n`);
+                log(`ℹ️ No code elements extracted for ${row.path}`);
             }
         }
         catch (err) {
             log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}\n`);
         }
-        // Add a small delay to throttle processing
         await new Promise(resolve => setTimeout(resolve, 200));
     }
     await release();

package/dist/daemon/daemonWorker.js

@@ -31,9 +31,26 @@ export async function daemonWorker() {
     while (true) {
         try {
             log('🔄 Running daemon batch...');
-            const didWork = await runDaemonBatch();
+            // Wrap the batch in debug
+            let didWork = false;
+            try {
+                log('🔹 Running runDaemonBatch()...');
+                didWork = await runDaemonBatch();
+                log('✅ runDaemonBatch() completed successfully');
+            }
+            catch (batchErr) {
+                log('🔥 Error inside runDaemonBatch():', batchErr);
+            }
             if (!didWork) {
-                const queueEmpty = await isQueueEmpty();
+                let queueEmpty = false;
+                try {
+                    log('🔹 Checking if queue is empty...');
+                    queueEmpty = await isQueueEmpty();
+                    log(`🔹 Queue empty status: ${queueEmpty}`);
+                }
+                catch (queueErr) {
+                    log('🔥 Error checking queue status:', queueErr);
+                }
                 if (queueEmpty) {
                     log('🕊️ No work found. Idling...');
                     await sleep(IDLE_SLEEP_MS * 3);

package/dist/db/functionExtractors/extractFromJs.js

@@ -29,6 +29,7 @@ export async function extractFromJS(filePath, content, fileId) {
             locations: true,
         });
         const functions = [];
+        const classes = [];
         walkAncestor(ast, {
             FunctionDeclaration(node, ancestors) {
                 const parent = ancestors[ancestors.length - 2];
@@ -60,31 +61,63 @@ export async function extractFromJS(filePath, content, fileId) {
                     content: content.slice(node.start, node.end),
                 });
             },
+            ClassDeclaration(node) {
+                const className = node.id?.name || `${path.basename(filePath)}:<anon-class>`;
+                classes.push({
+                    name: className,
+                    start_line: node.loc?.start.line ?? -1,
+                    end_line: node.loc?.end.line ?? -1,
+                    content: content.slice(node.start, node.end),
+                    superClass: node.superClass?.name ?? null,
+                });
+            },
+            ClassExpression(node) {
+                const className = node.id?.name || `${path.basename(filePath)}:<anon-class>`;
+                classes.push({
+                    name: className,
+                    start_line: node.loc?.start.line ?? -1,
+                    end_line: node.loc?.end.line ?? -1,
+                    content: content.slice(node.start, node.end),
+                    superClass: node.superClass?.name ?? null,
+                });
+            },
         });
-        if (functions.length === 0) {
-            log(`⚠️ No functions found in: ${filePath}`);
+        if (functions.length === 0 && classes.length === 0) {
+            log(`⚠️ No functions/classes found in: ${filePath}`);
             db.prepare(markFileAsSkippedTemplate).run({ id: fileId });
             return false;
         }
-        log(`🔍 Found ${functions.length} functions in ${filePath}`);
+        log(`🔍 Found ${functions.length} functions and ${classes.length} classes in ${filePath}`);
+        // Insert functions
         for (const fn of functions) {
             const embedding = await generateEmbedding(fn.content);
-            const result = db.prepare(`
+            const result = db
+                .prepare(`
         INSERT INTO functions (
           file_id, name, start_line, end_line, content, embedding, lang
         ) VALUES (
           @file_id, @name, @start_line, @end_line, @content, @embedding, @lang
         )
-      `).run({
+      `)
+                .run({
                 file_id: fileId,
                 name: fn.name,
                 start_line: fn.start_line,
                 end_line: fn.end_line,
                 content: fn.content,
                 embedding: JSON.stringify(embedding),
-                lang: 'js'
+                lang: 'js',
+            });
+            const functionId = result.lastInsertRowid;
+            // file → function edge
+            db.prepare(`INSERT INTO edges (source_type, source_id, target_type, target_id, relation)
+         VALUES (@source_type, @source_id, @target_type, @target_id, 'contains')`).run({
+                source_type: 'file',
+                source_id: fileId,
+                target_type: 'function',
+                target_id: functionId,
             });
-            const callerId = result.lastInsertRowid;
+            // Walk inside function to find calls
             const fnAst = parse(fn.content, {
                 ecmaVersion: 'latest',
                 sourceType: 'module',
@@ -96,26 +129,73 @@ export async function extractFromJS(filePath, content, fileId) {
                     if (node.callee?.type === 'Identifier' && node.callee.name) {
                         calls.push({ calleeName: node.callee.name });
                     }
-                }
+                },
             });
             for (const call of calls) {
-                db.prepare(`
-          INSERT INTO function_calls (caller_id, callee_name)
-          VALUES (@caller_id, @callee_name)
-        `).run({
-                    caller_id: callerId,
-                    callee_name: call.calleeName
+                // Store name for later resolution
+                db.prepare(`INSERT INTO function_calls (caller_id, callee_name) VALUES (@caller_id, @callee_name)`).run({ caller_id: functionId, callee_name: call.calleeName });
+                // Optional unresolved edge
+                db.prepare(`INSERT INTO edges (source_type, source_id, target_type, target_id, relation)
+           VALUES (@source_type, @source_id, @target_type, @target_id, 'calls')`).run({
+                    source_type: 'function',
+                    source_id: functionId,
+                    target_type: 'function',
+                    target_id: 0, // unresolved callee
                 });
             }
             log(`📌 Indexed function: ${fn.name} with ${calls.length} calls`);
         }
+        // Insert classes
+        for (const cls of classes) {
+            const embedding = await generateEmbedding(cls.content);
+            const result = db
+                .prepare(`
+    INSERT INTO classes (
+      file_id, name, start_line, end_line, content, embedding, lang
+    ) VALUES (
+      @file_id, @name, @start_line, @end_line, @content, @embedding, @lang
+    )
+  `)
+                .run({
+                file_id: fileId,
+                name: cls.name,
+                start_line: cls.start_line,
+                end_line: cls.end_line,
+                content: cls.content,
+                embedding: JSON.stringify(embedding),
+                lang: 'js',
+            });
+            const classId = result.lastInsertRowid;
+            // file → class edge
+            db.prepare(`INSERT INTO edges (source_type, source_id, target_type, target_id, relation)
+     VALUES (@source_type, @source_id, @target_type, @target_id, 'contains')`).run({
+                source_type: 'file',
+                source_id: fileId,
+                target_type: 'class',
+                target_id: classId,
+            });
+            // superclass → store unresolved reference
+            if (cls.superClass) {
+                db.prepare(`INSERT INTO edges (source_type, source_id, target_type, target_id, relation)
+       VALUES (@source_type, @source_id, @target_type, @target_id, 'inherits')`).run({
+                    source_type: 'class',
+                    source_id: classId,
+                    target_type: 'class',
+                    target_id: 0, // unresolved superclass
+                });
+                console.log(`🔗 Class ${cls.name} inherits ${cls.superClass} (edge stored for later resolution)`);
+            }
+            console.log(`🏷 Indexed class: ${cls.name} (id=${classId})`);
+        }
+        // Optional summary after extraction
+        console.log(`📊 Extraction summary for ${filePath}: ${functions.length} functions, ${classes.length} classes`);
         db.prepare(markFileAsExtractedTemplate).run({ id: fileId });
-        log(`✅ Marked functions as extracted for ${filePath}`);
+        log(`✅ Marked functions/classes as extracted for ${filePath}`);
         return true;
     }
     catch (err) {
         log(`❌ Failed to extract from: ${filePath}`);
-        log(`   ↳ ${String(err.message)}`);
+        log(`   ↳ ${err.message}`);
         db.prepare(markFileAsFailedTemplate).run({ id: fileId });
         return false;
     }