scai 0.1.109 → 0.1.111

package/dist/CHANGELOG.md

@@ -171,4 +171,22 @@ Type handling with the module pipeline
 ## 2025-09-02
 
 • Added test configuration for project and generated tests
-• Add runTestsModule and repairTestsModule for testing pipeline
+• Add runTestsModule and repairTestsModule for testing pipeline
+
+## 2025-09-05
+
+• Enable execution of files as executable files in the scripts
+• Remove context failure if models not installed
+• Add ability to set global model
+
+## 2025-09-08
+
+### Requires DB reset ('scai db reset' followed by 'scai index start')
+
+1. Improved daemon batch processing by skipping missing files, classifying unknown file types, and persisting entities/tags in the database.
+2. Invoke kgModule in daemonBatch to build knowledge graphs after indexing.
+3. Improved data modeling and extraction logic for functions and classes in TypeScript files.
+4. Updated Edge/Table schema for better query performance.
+5. Update package-lock.json to caniuse-lite@1.0.30001741.
+6. Enable execution of  as an executable file in the scripts.
+7. Remove context failure if models not installed. Add ability to set global model.

package/dist/commands/AskCmd.js

@@ -1,13 +1,12 @@
 import fs from 'fs';
 import path from 'path';
 import readline from 'readline';
-import { searchFiles, queryFiles, getFunctionsForFiles } from '../db/fileIndex.js';
+import { searchFiles, queryFiles } from '../db/fileIndex.js';
 import { sanitizeQueryForFts } from '../utils/sanitizeQuery.js';
 import { generate } from '../lib/generate.js';
 import { buildContextualPrompt } from '../utils/buildContextualPrompt.js';
-import { generateFocusedFileTree } from '../utils/fileTree.js';
 import { log } from '../utils/log.js';
-import { PROMPT_LOG_PATH, SCAI_HOME, RELATED_FILES_LIMIT, MAX_SUMMARY_LINES, getIndexDir, MAX_FUNCTION_LINES } from '../constants.js';
+import { PROMPT_LOG_PATH, SCAI_HOME, RELATED_FILES_LIMIT, getIndexDir } from '../constants.js';
 import chalk from 'chalk';
 export async function runAskCommand(query) {
     if (!query) {
@@ -19,22 +18,21 @@ export async function runAskCommand(query) {
         return;
     }
     console.log(`📁 Using index root: ${getIndexDir()}`);
-    console.log(`🔍 Searching for: "${query}"\n`);
-    // 🟩 STEP 1: Semantic Search
+    // Semantic Search
     const start = Date.now();
-    const semanticResults = await searchFiles(query, RELATED_FILES_LIMIT); // RankedFile[]
+    const semanticResults = await searchFiles(query, RELATED_FILES_LIMIT);
     const duration = Date.now() - start;
     console.log(`⏱️ searchFiles took ${duration}ms and returned ${semanticResults.length} result(s)`);
     semanticResults.forEach((file, i) => {
         console.log(`  ${i + 1}. 📄 Path: ${file.path} | Score: ${file.score?.toFixed(3) ?? 'n/a'}`);
     });
-    // 🟩 STEP 1.5: Fallback FTS search
+    // Fallback FTS search
     const safeQuery = sanitizeQueryForFts(query);
-    const fallbackResults = queryFiles(safeQuery, 10); // FileRow[]
+    const fallbackResults = queryFiles(safeQuery, 10);
     fallbackResults.forEach((file, i) => {
         console.log(`  ${i + 1}. 🔎 Fallback Match: ${file.path}`);
     });
-    // 🟩 STEP 2: Merge results
+    // Merge results
     const seen = new Set();
     const combinedResults = [];
     for (const file of semanticResults) {
@@ -56,7 +54,20 @@ export async function runAskCommand(query) {
             });
         }
     }
-    // 🟩 STEP 3: Log results
+    // Exact match prioritization
+    const queryFilenameRaw = path.basename(query).toLowerCase();
+    const queryFilenameNoExt = queryFilenameRaw.replace(/\.[^/.]+$/, '');
+    const exactMatchIndex = combinedResults.findIndex(f => {
+        const base = path.basename(f.path).toLowerCase();
+        const baseNoExt = base.replace(/\.[^/.]+$/, '');
+        return base === queryFilenameRaw || baseNoExt === queryFilenameNoExt;
+    });
+    if (exactMatchIndex !== -1) {
+        const [exactMatch] = combinedResults.splice(exactMatchIndex, 1);
+        combinedResults.unshift(exactMatch);
+        console.log(`🎯 Exact match prioritized: ${exactMatch.path}`);
+    }
+    // Log combined results
     if (combinedResults.length) {
         console.log('\n📊 Final Related Files:');
         combinedResults.forEach((f, i) => {
@@ -66,79 +77,38 @@ export async function runAskCommand(query) {
     else {
         console.log('⚠️ No similar files found. Using query only.');
     }
-    // 🟩 STEP 4: Load top file code + metadata
-    const topFile = combinedResults[0];
-    const filepath = topFile?.path || '';
-    let code = '';
-    let topSummary = topFile.summary || '(No summary available)';
-    let topFunctions = [];
-    const fileFunctions = {};
-    // Truncate summary
-    topSummary = topSummary.split('\n').slice(0, MAX_SUMMARY_LINES).join('\n');
-    const allFileIds = combinedResults
-        .map(file => file.id)
-        .filter((id) => typeof id === 'number');
-    const allFunctionsMap = getFunctionsForFiles(allFileIds); // Record<number, Function[]>
-    try {
-        code = fs.readFileSync(filepath, 'utf-8');
-        const topFileId = topFile.id;
-        topFunctions = allFunctionsMap[topFileId]?.map(fn => {
-            const content = fn.content
-                ? fn.content.split('\n').slice(0, MAX_FUNCTION_LINES).join('\n')
-                : '(No content available)';
-            return {
-                name: fn.name,
-                content,
-            };
-        }) || [];
-    }
-    catch (err) {
-        console.warn(`⚠️ Failed to read or analyze top file (${filepath}):`, err);
+    // STEP 4+: Build contextual prompt using topFile + combinedResults
+    if (combinedResults.length === 0) {
+        throw new Error('❌ No search results found. Cannot build contextual prompt.');
     }
-    // 🟩 STEP 5: Build relatedFiles with functions and fileFunctions
-    const relatedFiles = combinedResults.slice(0, RELATED_FILES_LIMIT).map(file => {
-        const fileId = file.id;
-        let summary = file.summary || '(No summary available)';
-        if (summary) {
-            summary = summary.split('\n').slice(0, MAX_SUMMARY_LINES).join('\n');
-        }
-        const functions = allFunctionsMap[fileId]?.map(fn => {
-            const content = fn.content
-                ? fn.content.split('\n').slice(0, MAX_FUNCTION_LINES).join('\n')
-                : '(No content available)';
-            return {
-                name: fn.name,
-                content,
-            };
-        }) || [];
-        return {
-            path: file.path,
-            summary,
-            functions,
-        };
-    });
-    // 🟩 STEP 6: Generate file tree
-    let fileTree = '';
+    const file = combinedResults[0];
+    let code = "";
+    // STEP 4++: Add code to params
     try {
-        fileTree = generateFocusedFileTree(filepath, 2);
+        code = fs.readFileSync(file.path, 'utf-8');
+        if (!code) {
+            console.warn(`⚠️ No code loaded for top file: ${file.path}`);
+        }
     }
     catch (e) {
-        console.warn('⚠️ Could not generate file tree:', e);
+        console.log("Error reading code from selected file: ", e instanceof Error ? e.message : e);
     }
-    // 🟩 STEP 7: Build prompt
-    console.log(chalk.blueBright('\n📦 Building contextual prompt...'));
-    const promptContent = buildContextualPrompt({
-        baseInstruction: query,
+    const topFile = {
+        id: file.id,
+        path: file.path,
+        summary: file.summary ?? "",
         code,
-        summary: topSummary,
-        functions: topFunctions,
-        relatedFiles,
-        projectFileTree: fileTree || undefined,
-        fileFunctions,
-    });
+    };
+    const promptArgs = {
+        topFile,
+        relatedFiles: combinedResults,
+        query,
+    };
+    console.log(chalk.blueBright('\n📦 Building contextual prompt...'));
+    const promptContent = await buildContextualPrompt(promptArgs);
     console.log(chalk.greenBright('✅ Prompt built successfully.'));
     console.log(chalk.cyan(`[runAskCommand] Prompt token estimate: ~${Math.round(promptContent.length / 4)} tokens`));
-    // 🟩 STEP 8: Save prompt
+    // STEP 5: Save prompt
     try {
         if (!fs.existsSync(SCAI_HOME))
             fs.mkdirSync(SCAI_HOME, { recursive: true });
@@ -148,21 +118,21 @@ export async function runAskCommand(query) {
     catch (err) {
         log('❌ Failed to write prompt log:', err);
     }
-    // 🟩 STEP 9: Ask model
+    // STEP 6: Ask model
     try {
         console.log('\n🤖 Asking the model...');
         const input = {
             content: promptContent,
-            filepath,
+            filepath: topFile.path,
         };
-        const modelResponse = await generate(input, 'llama3');
+        const modelResponse = await generate(input);
         console.log(`\n🧠 Model Response:\n${modelResponse.content}`);
     }
     catch (err) {
         console.error('❌ Model request failed:', err);
     }
 }
-// 🟩 Helper: Prompt once
+// Helper: Prompt once
 function promptOnce(promptText) {
     return new Promise(resolve => {
         console.log(promptText);

package/dist/commands/DaemonCmd.js

@@ -23,9 +23,11 @@ export async function startDaemon() {
     const __filename = fileURLToPath(import.meta.url);
     const __dirname = path.dirname(__filename);
     const daemonWorkerPath = path.join(__dirname, '../daemon/daemonWorker.js');
+    const out = fsSync.openSync(LOG_PATH, 'a');
+    const err = fsSync.openSync(LOG_PATH, 'a');
     const child = spawn(process.execPath, [daemonWorkerPath], {
         detached: true,
-        stdio: ['ignore', 'ignore', 'ignore'],
+        stdio: ['ignore', out, err], // stdout/stderr -> log file
         env: {
             ...process.env,
             BACKGROUND_MODE: 'true',

package/dist/config.js

@@ -6,7 +6,7 @@ import { normalizePath } from './utils/contentUtils.js';
 import chalk from 'chalk';
 import { getHashedRepoKey } from './utils/repoKey.js';
 const defaultConfig = {
-    model: 'codellama:13b',
+    model: 'llama3:8b',
     contextLength: 4096,
     language: 'ts',
     indexDir: '',
@@ -55,18 +55,23 @@ export const Config = {
         const repoCfg = cfg.repos?.[cfg.activeRepo ?? ''];
         return repoCfg?.model || cfg.model;
     },
-    setModel(model) {
+    setModel(model, scope = 'repo') {
         const cfg = readConfig();
-        const active = cfg.activeRepo;
-        if (active) {
+        if (scope === 'repo') {
+            const active = cfg.activeRepo;
+            if (!active) {
+                console.error("❌ No active repo to set model for.");
+                return;
+            }
             cfg.repos[active] = { ...cfg.repos[active], model };
-            writeConfig(cfg);
-            console.log(`📦 Model set to: ${model}`);
+            console.log(`📦 Model set for repo '${active}': ${model}`);
         }
         else {
-            writeConfig({ model });
-            console.log(`📦 Default model set to: ${model}`);
+            // Set global default model
+            cfg.model = model;
+            console.log(`📦 Global default model set to: ${model}`);
         }
+        writeConfig(cfg);
     },
     getLanguage() {
         const cfg = readConfig();

package/dist/context.js

@@ -5,12 +5,25 @@ import { getHashedRepoKey } from "./utils/repoKey.js";
 import { getDbForRepo, getDbPathForRepo } from "./db/client.js";
 import fs from "fs";
 import chalk from "chalk";
+import { execSync } from "child_process";
+function modelExists(model) {
+    try {
+        const output = execSync("ollama list", { encoding: "utf-8" });
+        return output
+            .split("\n")
+            .map(line => line.trim())
+            .filter(Boolean)
+            .some(line => line.toLowerCase().startsWith(model.toLowerCase() + " ") || line.toLowerCase() === model.toLowerCase());
+    }
+    catch (err) {
+        console.error(chalk.red("❌ Failed to check models with `ollama list`"));
+        return false;
+    }
+}
 export async function updateContext() {
     const cwd = normalizePath(process.cwd());
     const cfg = readConfig();
-    // 🔑 Find repoKey by matching indexDir to cwd
     let repoKey = Object.keys(cfg.repos || {}).find((key) => normalizePath(cfg.repos[key]?.indexDir || "") === cwd);
-    // Initialize new repo config if not found
     let isNewRepo = false;
     if (!repoKey) {
         repoKey = getHashedRepoKey(cwd);
@@ -19,28 +32,23 @@ export async function updateContext() {
         cfg.repos[repoKey].indexDir = cwd;
         isNewRepo = true;
     }
-    // Check if active repo has changed
     const activeRepoChanged = cfg.activeRepo !== repoKey;
-    // Always set this as active repo
     cfg.activeRepo = repoKey;
     writeConfig(cfg);
     const repoCfg = cfg.repos[repoKey];
     let ok = true;
-    // Only log detailed info if new repo or active repo changed
     if (isNewRepo || activeRepoChanged) {
         console.log(chalk.yellow("\n🔁 Updating context...\n"));
         console.log(`✅ Active repo: ${chalk.green(repoKey)}`);
         console.log(`✅ Index dir: ${chalk.cyan(repoCfg.indexDir || cwd)}`);
     }
-    // GitHub token is optional
     const token = repoCfg.githubToken || cfg.githubToken;
     if (!token) {
-        console.log(`ℹ️ No GitHub token found. You can set one with the: ${chalk.bold(chalk.bgGreen("scai auth set"))} command`);
+        console.log(`ℹ️ No GitHub token found. You can set one with: ${chalk.bold(chalk.bgGreen("scai auth set"))}`);
     }
     else if (isNewRepo || activeRepoChanged) {
         console.log(`✅ GitHub token present`);
     }
-    // Ensure DB exists
     const dbPath = getDbPathForRepo();
     if (!fs.existsSync(dbPath)) {
         console.log(chalk.yellow(`📦 Initializing DB at ${dbPath}`));
@@ -48,13 +56,31 @@ export async function updateContext() {
             getDbForRepo();
         }
         catch {
-            ok = false; // DB init failed
+            ok = false;
         }
     }
     else if (isNewRepo || activeRepoChanged) {
         console.log(chalk.green("✅ Database present"));
     }
-    // Final context status
+    // 🧠 Model check
+    const model = cfg.model;
+    if (!model) {
+        console.log(chalk.red("❌ No model configured.") +
+            "\n➡️ Set one with: " +
+            chalk.bold(chalk.bgGreen("scai config set-model <model>")));
+        ok = false;
+    }
+    else if (!modelExists(model)) {
+        console.log(chalk.red(`❌ Model '${model}' not installed in Ollama.`) +
+            "\n➡️ Install with: " +
+            chalk.bold(chalk.yellow(`ollama pull ${model}`)) +
+            " or choose another with: " +
+            chalk.bold(chalk.yellow("scai config set-model <model>")));
+        ok = false;
+    }
+    else {
+        console.log(chalk.green(`✅ Model '${model}' available`));
+    }
     if (ok) {
         console.log(chalk.bold.green("\n✅ Context OK\n"));
     }

package/dist/daemon/daemonBatch.js

@@ -1,4 +1,4 @@
-import { indexFunctionsForFile } from '../db/functionIndex.js';
+import { indexCodeForFile } from '../db/functionIndex.js';
 import fs from 'fs/promises';
 import fsSync from 'fs';
 import { generateEmbedding } from '../lib/generateEmbedding.js';
@@ -8,6 +8,7 @@ import { summaryModule } from '../pipeline/modules/summaryModule.js';
 import { classifyFile } from '../fileRules/classifyFile.js';
 import { getDbForRepo, getDbPathForRepo } from '../db/client.js';
 import { markFileAsSkippedByPath, selectUnprocessedFiles, updateFileWithSummaryAndEmbedding, } from '../db/sqlTemplates.js';
+import { kgModule } from '../pipeline/modules/kgModule.js';
 const MAX_FILES_PER_BATCH = 5;
 /**
  * Acquires a lock on the database to ensure that only one daemon batch
@@ -32,7 +33,6 @@ async function lockDb() {
  */
 export async function runDaemonBatch() {
     log('🟡 Starting daemon batch...');
-    // Selects up to MAX_FILES_PER_BATCH files that haven't been processed yet
     const db = getDbForRepo();
     const rows = db.prepare(selectUnprocessedFiles).all(MAX_FILES_PER_BATCH);
     if (rows.length === 0) {
@@ -42,13 +42,11 @@ export async function runDaemonBatch() {
     const release = await lockDb();
     for (const row of rows) {
         log(`📂 Processing file: ${row.path}`);
-        // Skip if file is missing from the file system
         if (!fsSync.existsSync(row.path)) {
             log(`⚠️ Skipped missing file: ${row.path}`);
             db.prepare(markFileAsSkippedByPath).run({ path: row.path });
             continue;
         }
-        // Skip if file is classified as something we don't process
         const classification = classifyFile(row.path);
         if (classification !== 'valid') {
             log(`⏭️ Skipping (${classification}): ${row.path}`);
@@ -57,24 +55,20 @@ export async function runDaemonBatch() {
         }
         try {
             const content = await fs.readFile(row.path, 'utf-8');
-            // Determine whether the file needs to be re-summarized
             const needsResummary = !row.summary ||
                 !row.indexed_at ||
                 (row.last_modified && new Date(row.last_modified) > new Date(row.indexed_at));
             if (needsResummary) {
                 log(`📝 Generating summary for ${row.path}...`);
-                // Generate a summary using the summary pipeline
                 const summaryResult = await summaryModule.run({ content, filepath: row.path });
                 const summary = summaryResult?.summary?.trim() || null;
                 let embedding = null;
-                // Generate an embedding from the summary (if present)
                 if (summary) {
                     const vector = await generateEmbedding(summary);
                     if (vector) {
                         embedding = JSON.stringify(vector);
                     }
                 }
-                // Update the file record with the new summary and embedding
                 db.prepare(updateFileWithSummaryAndEmbedding).run({
                     summary,
                     embedding,
@@ -85,19 +79,79 @@ export async function runDaemonBatch() {
             else {
                 log(`⚡ Skipped summary (up-to-date) for ${row.path}`);
             }
-            // Extract top-level functions from the file and update the DB
-            const extracted = await indexFunctionsForFile(row.path, row.id);
-            if (extracted) {
-                log(`✅ Function extraction complete for ${row.path}\n`);
+            const success = await indexCodeForFile(row.path, row.id);
+            if (success) {
+                log(`✅ Indexed code for ${row.path}`);
+                try {
+                    log(`🔗 Building Knowledge Graph for ${row.path}...`);
+                    const kgInput = {
+                        fileId: row.id,
+                        filepath: row.path,
+                        summary: row.summary || undefined,
+                    };
+                    const kgResult = await kgModule.run(kgInput, content);
+                    log(`✅ Knowledge Graph built for ${row.path}`);
+                    log(`Entities: ${kgResult.entities.length}, Edges: ${kgResult.edges.length}`);
+                    // Persist KG entities + tags only if there are any
+                    if (kgResult.entities.length > 0) {
+                        const insertTag = db.prepare(`
+    INSERT OR IGNORE INTO tags_master (name) VALUES (:name)
+  `);
+                        const getTagId = db.prepare(`
+    SELECT id FROM tags_master WHERE name = :name
+  `);
+                        const insertEntityTag = db.prepare(`
+    INSERT OR IGNORE INTO entity_tags (entity_type, entity_id, tag_id)
+    VALUES (:entity_type, :entity_id, :tag_id)
+  `);
+                        for (const entity of kgResult.entities) {
+                            // Skip entity if type or tags are missing
+                            if (!entity.type || !Array.isArray(entity.tags) || entity.tags.length === 0) {
+                                console.warn(`⚠ Skipping entity due to missing type or tags:`, entity);
+                                continue;
+                            }
+                            for (const tag of entity.tags) {
+                                // Skip empty or invalid tags
+                                if (!tag || typeof tag !== 'string') {
+                                    console.warn(`⚠ Skipping invalid tag for entity ${entity.type}:`, tag);
+                                    continue;
+                                }
+                                try {
+                                    // ✅ Use :name in SQL and plain key in object
+                                    insertTag.run({ name: tag });
+                                    const tagRow = getTagId.get({ name: tag });
+                                    if (!tagRow) {
+                                        console.warn(`⚠ Could not find tag ID for: ${tag}`);
+                                        continue;
+                                    }
+                                    insertEntityTag.run({
+                                        entity_type: entity.type,
+                                        entity_id: row.id,
+                                        tag_id: tagRow.id,
+                                    });
+                                }
+                                catch (err) {
+                                    console.error(`❌ Failed to persist entity/tag:`, { entity, tag, error: err });
+                                }
+                            }
+                        }
+                        log(`✅ Persisted entities + tags for ${row.path}`);
+                    }
+                    else {
+                        log(`⚠️ No entities found for ${row.path}, skipping DB inserts`);
+                    }
+                }
+                catch (kgErr) {
+                    log(`❌ KG build failed for ${row.path}: ${kgErr instanceof Error ? kgErr.message : String(kgErr)}`);
+                }
             }
             else {
-                log(`ℹ️ No functions extracted for ${row.path}\n`);
+                log(`ℹ️ No code elements extracted for ${row.path}`);
             }
         }
         catch (err) {
             log(`❌ Failed: ${row.path}: ${err instanceof Error ? err.message : String(err)}\n`);
         }
-        // Add a small delay to throttle processing
         await new Promise(resolve => setTimeout(resolve, 200));
     }
     await release();

package/dist/daemon/daemonWorker.js

@@ -31,9 +31,26 @@ export async function daemonWorker() {
     while (true) {
         try {
             log('🔄 Running daemon batch...');
-            const didWork = await runDaemonBatch();
+            // Wrap the batch in debug
+            let didWork = false;
+            try {
+                log('🔹 Running runDaemonBatch()...');
+                didWork = await runDaemonBatch();
+                log('✅ runDaemonBatch() completed successfully');
+            }
+            catch (batchErr) {
+                log('🔥 Error inside runDaemonBatch():', batchErr);
+            }
             if (!didWork) {
-                const queueEmpty = await isQueueEmpty();
+                let queueEmpty = false;
+                try {
+                    log('🔹 Checking if queue is empty...');
+                    queueEmpty = await isQueueEmpty();
+                    log(`🔹 Queue empty status: ${queueEmpty}`);
+                }
+                catch (queueErr) {
+                    log('🔥 Error checking queue status:', queueErr);
+                }
                 if (queueEmpty) {
                     log('🕊️ No work found. Idling...');
                     await sleep(IDLE_SLEEP_MS * 3);

package/dist/db/fileIndex.js

@@ -6,6 +6,7 @@ import * as sqlTemplates from './sqlTemplates.js';
 import { CANDIDATE_LIMIT } from '../constants.js';
 import { getDbForRepo } from './client.js';
 import { scoreFiles } from '../fileRules/scoreFiles.js'; // 👈 NEW
+import chalk from 'chalk';
 export function indexFile(filePath, summary, type) {
     const stats = fs.statSync(filePath);
     const lastModified = stats.mtime.toISOString();
@@ -44,7 +45,7 @@ export function queryFiles(safeQuery, limit = 10) {
   `).all(safeQuery, limit);
 }
 export async function searchFiles(query, topK = 5) {
-    console.log(`🧠 Searching for query: "${query}"`);
+    console.log(chalk.yellow(`🧠 Searching for query: "${query}"`));
     const embedding = await generateEmbedding(query);
     if (!embedding) {
         console.log('⚠️ Failed to generate embedding for query');