scai 0.1.109 → 0.1.111

package/dist/pipeline/modules/cleanupModule.js

@@ -29,25 +29,21 @@ function isTopOrBottomNoise(line) {
 }
 export const cleanupModule = {
     name: 'cleanup',
-    description: 'Remove markdown fences and fluff from top/bottom of each chunk with colored logging',
+    description: 'Remove markdown fences, fluff, and non-JSON lines with colored logging',
     async run(input) {
-        // Normalize line endings to \n to avoid issues with \r\n
+        // Normalize line endings to \n
         let content = input.content.replace(/\r\n/g, '\n');
         let lines = content.split('\n');
         // --- CLEAN TOP ---
-        // Remove noise lines before the first triple tick or end
         while (lines.length && (lines[0].trim() === '' || isTopOrBottomNoise(lines[0]))) {
             if (/^```(?:\w+)?$/.test(lines[0].trim()))
-                break; // Stop if opening fence found
+                break;
             console.log(chalk.red(`[cleanupModule] Removing noise from top:`), chalk.yellow(`"${lines[0].trim()}"`));
             lines.shift();
         }
-        // If opening fence found at top, find matching closing fence
         if (lines.length && /^```(?:\w+)?$/.test(lines[0].trim())) {
             console.log(chalk.red(`[cleanupModule] Found opening fenced block at top.`));
-            // Remove opening fence line
             lines.shift();
-            // Find closing fence index
             let closingIndex = -1;
             for (let i = 0; i < lines.length; i++) {
                 if (/^```(?:\w+)?$/.test(lines[i].trim())) {
@@ -57,26 +53,22 @@ export const cleanupModule = {
             }
             if (closingIndex !== -1) {
                 console.log(chalk.red(`[cleanupModule] Found closing fenced block at line ${closingIndex + 1}, removing fence lines.`));
-                // Remove closing fence line
                 lines.splice(closingIndex, 1);
             }
             else {
                 console.log(chalk.yellow(`[cleanupModule] No closing fenced block found, only removed opening fence.`));
             }
-            // NO removal of noise lines after fenced block here (to keep new comments intact)
         }
         // --- CLEAN BOTTOM ---
-        // If closing fence found at bottom, remove only that triple tick line
         if (lines.length && /^```(?:\w+)?$/.test(lines[lines.length - 1].trim())) {
             console.log(chalk.red(`[cleanupModule] Removing closing fenced block line at bottom.`));
             lines.pop();
         }
-        // Remove noise lines after closing fence (now bottom)
         while (lines.length && (lines[lines.length - 1].trim() === '' || isTopOrBottomNoise(lines[lines.length - 1]))) {
             console.log(chalk.red(`[cleanupModule] Removing noise from bottom after fenced block:`), chalk.yellow(`"${lines[lines.length - 1].trim()}"`));
             lines.pop();
         }
-        // --- FINAL CLEANUP: REMOVE ANY LINGERING TRIPLE TICK LINES ANYWHERE ---
+        // --- REMOVE ANY LINGERING TRIPLE TICK LINES ANYWHERE ---
         lines = lines.filter(line => {
             const trimmed = line.trim();
             if (/^```(?:\w+)?$/.test(trimmed)) {
@@ -85,6 +77,33 @@ export const cleanupModule = {
             }
             return true;
         });
-        return { content: lines.join('\n').trim() };
+        // --- FINAL CLEANUP: KEEP ONLY JSON LINES INSIDE BRACES ---
+        let jsonLines = [];
+        let braceDepth = 0;
+        let insideBraces = false;
+        for (let line of lines) {
+            const trimmed = line.trim();
+            // Detect start of JSON object/array
+            if (!insideBraces && (trimmed.startsWith('{') || trimmed.startsWith('['))) {
+                insideBraces = true;
+            }
+            if (insideBraces) {
+                // Track nested braces/brackets
+                for (const char of trimmed) {
+                    if (char === '{' || char === '[')
+                        braceDepth++;
+                    if (char === '}' || char === ']')
+                        braceDepth--;
+                }
+                // Skip lines that are clearly non-JSON inside braces
+                if (!trimmed.startsWith('//') && !/^\/\*/.test(trimmed) && trimmed !== '') {
+                    jsonLines.push(line);
+                }
+                // Stop collecting after outermost brace closed
+                if (braceDepth === 0)
+                    break;
+            }
+        }
+        return { content: jsonLines.join('\n').trim() };
     }
 };

package/dist/pipeline/modules/commentModule.js

@@ -53,7 +53,7 @@ Rules:
 ${input.content}
 
 `.trim();
-        const response = await generate({ content: prompt }, model);
+        const response = await generate({ content: prompt });
         const contentToReturn = (response.content && response.content !== 'NO UPDATE') ? response.content : input.content;
         return {
             content: contentToReturn,

package/dist/pipeline/modules/commitSuggesterModule.js

@@ -22,7 +22,7 @@ Format your response exactly as:
 Here is the diff:
 ${content}
 `.trim();
-        const response = await generate({ content: prompt }, model);
+        const response = await generate({ content: prompt });
         const lines = response.content
             .split('\n')
             .map(line => line.trim())

package/dist/pipeline/modules/generateTestsModule.js

@@ -45,7 +45,7 @@ describe('moduleUnderTest', () => {
 ${content}
 --- END MODULE CODE ---
 `.trim();
-        const response = await generate({ content: prompt }, model);
+        const response = await generate({ content: prompt });
         if (!response)
             throw new Error('⚠️ No test code returned from model');
         return {

package/dist/pipeline/modules/kgModule.js

@@ -0,0 +1,55 @@
+import { Config } from '../../config.js';
+import { generate } from '../../lib/generate.js';
+import path from 'path';
+import { cleanupModule } from './cleanupModule.js';
+export const kgModule = {
+    name: 'knowledge-graph',
+    description: 'Generates a knowledge graph of entities, tags, and relationships from file content.',
+    run: async (input, content) => {
+        const model = Config.getModel();
+        const ext = input.filepath ? path.extname(input.filepath).toLowerCase() : '';
+        const filename = input.filepath ? path.basename(input.filepath) : '';
+        const prompt = `
+You are an assistant specialized in building knowledge graphs from code or text.
+
+Your task is to extract structured information from the file content below.
+
+File: ${filename}
+Extension: ${ext}
+
+📋 Instructions:
+- Identify all entities (functions, classes, modules, or main concepts)
+- For each entity, generate tags describing its characteristics, purpose, or category
+- Identify relationships between entities (e.g., "uses", "extends", "calls")
+- Return output in JSON format with the following structure:
+
+{
+  "entities": [
+    { "name": "EntityName", "type": "class|function|module|concept", "tags": ["tag1", "tag2"] }
+  ],
+  "edges": [
+    { "from": "EntityName1", "to": "EntityName2", "type": "relationship_type" }
+  ]
+}
+
+Do NOT include raw content from the file. Only provide the structured JSON output.
+
+--- FILE CONTENT START ---
+${content}
+--- FILE CONTENT END ---
+    `.trim();
+        const response = await generate({ content: prompt, filepath: input.filepath });
+        try {
+            // Clean the model output first
+            const cleaned = await cleanupModule.run({ content: response.content });
+            console.log("Cleaned knowledge graph data: ", cleaned);
+            const jsonString = cleaned.content;
+            const parsed = JSON.parse(jsonString);
+            return parsed;
+        }
+        catch (err) {
+            console.warn('⚠️ Failed to parse KG JSON:', err);
+            return { entities: [], edges: [] }; // fallback
+        }
+    }
+};

package/dist/pipeline/modules/refactorModule.js

@@ -19,7 +19,7 @@ Refactor the following code:
 ${input.content}
 --- CODE END ---
     `.trim();
-        const response = await generate({ content: prompt }, model);
+        const response = await generate({ content: prompt });
         if (!response) {
             throw new Error('❌ Model returned empty response for refactoring.');
         }

package/dist/pipeline/modules/repairTestsModule.js

@@ -26,7 +26,7 @@ Instructions:
 
 Output the repaired test file:
 `.trim();
-        const response = await generate({ content: prompt }, model);
+        const response = await generate({ content: prompt });
         if (!response)
             throw new Error("⚠️ No repaired test code returned from model");
         return {

package/dist/pipeline/modules/reviewModule.js

@@ -20,7 +20,7 @@ Format your response exactly as:
 Changes:
 ${content}
 `.trim();
-        const response = await generate({ content: prompt, filepath }, model);
+        const response = await generate({ content: prompt, filepath });
         // Parse response: only keep numbered lines
         const lines = response.content
             .split('\n')

package/dist/pipeline/modules/summaryModule.js

@@ -27,7 +27,7 @@ Extension: ${ext}
 ${content}
 --- FILE CONTENT END ---
     `.trim();
-        const response = await generate({ content: prompt, filepath }, model);
+        const response = await generate({ content: prompt, filepath });
         if (response.content) {
             response.summary = response.content;
             console.log('\n📝 Summary:\n');

package/dist/scripts/dbcheck.js

@@ -224,3 +224,101 @@ const functionRows = db.prepare(`
   LIMIT 50
 `).all();
 console.table(functionRows);
+// === Class Table Stats ===
+console.log('\n📊 Stats for Table: classes');
+console.log('-------------------------------------------');
+try {
+    const classCount = db.prepare(`SELECT COUNT(*) AS count FROM classes`).get().count;
+    const distinctClassFiles = db.prepare(`SELECT COUNT(DISTINCT file_id) AS count FROM classes`).get().count;
+    console.log(`🏷 Total classes:           ${classCount}`);
+    console.log(`📂 Distinct files:          ${distinctClassFiles}`);
+}
+catch (err) {
+    console.error('❌ Error accessing classes table:', err.message);
+}
+// === Example Classes ===
+console.log('\n🧪 Example extracted classes:');
+try {
+    const sampleClasses = db.prepare(`
+    SELECT id, name, file_id, start_line, end_line, substr(content, 1, 100) || '...' AS short_body
+    FROM classes
+    ORDER BY id DESC
+    LIMIT 5
+  `).all();
+    sampleClasses.forEach(cls => {
+        console.log(`🏷 ID: ${cls.id}`);
+        console.log(`   Name: ${cls.name}`);
+        console.log(`   File: ${cls.file_id}`);
+        console.log(`   Lines: ${cls.start_line}-${cls.end_line}`);
+        console.log(`   Body: ${cls.short_body}\n`);
+    });
+}
+catch (err) {
+    console.error('❌ Error printing class examples:', err.message);
+}
+// === Edge Table Stats ===
+console.log('\n📊 Stats for Table: edges');
+console.log('-------------------------------------------');
+try {
+    const edgeCount = db.prepare(`SELECT COUNT(*) AS count FROM edges`).get().count;
+    const distinctRelations = db.prepare(`SELECT COUNT(DISTINCT relation) AS count FROM edges`).get().count;
+    console.log(`🔗 Total edges:             ${edgeCount}`);
+    console.log(`🧩 Distinct relations:      ${distinctRelations}`);
+}
+catch (err) {
+    console.error('❌ Error accessing edges table:', err.message);
+}
+// === Example Edges ===
+console.log('\n🧪 Example edges:');
+try {
+    const sampleEdges = db.prepare(`
+    SELECT id, source_id, target_id, relation
+    FROM edges
+    ORDER BY id DESC
+    LIMIT 10
+  `).all();
+    sampleEdges.forEach(e => {
+        console.log(`🔗 Edge ${e.id}: ${e.source_id} -[${e.relation}]-> ${e.target_id}`);
+    });
+}
+catch (err) {
+    console.error('❌ Error printing edge examples:', err.message);
+}
+// === Tags Master Stats ===
+console.log('\n📊 Stats for Table: tags_master');
+console.log('-------------------------------------------');
+try {
+    const tagCount = db.prepare(`SELECT COUNT(*) AS count FROM tags_master`).get().count;
+    console.log(`🏷 Total tags: ${tagCount}`);
+    const sampleTags = db.prepare(`
+    SELECT id, name
+    FROM tags_master
+    ORDER BY id DESC
+    LIMIT 5
+  `).all();
+    sampleTags.forEach(tag => {
+        console.log(`🏷 Tag ${tag.id}: ${tag.name}`);
+    });
+}
+catch (err) {
+    console.error('❌ Error accessing tags_master table:', err.message);
+}
+// === Entity Tags Stats ===
+console.log('\n📊 Stats for Table: entity_tags');
+console.log('-------------------------------------------');
+try {
+    const entityTagCount = db.prepare(`SELECT COUNT(*) AS count FROM entity_tags`).get().count;
+    console.log(`🔗 Total entity-tags: ${entityTagCount}`);
+    const sampleEntityTags = db.prepare(`
+    SELECT id, entity_type, entity_id, tag_id
+    FROM entity_tags
+    ORDER BY id DESC
+    LIMIT 10
+  `).all();
+    sampleEntityTags.forEach(et => {
+        console.log(`🔗 EntityTag ${et.id}: ${et.entity_type} ${et.entity_id} -> tag ${et.tag_id}`);
+    });
+}
+catch (err) {
+    console.error('❌ Error accessing entity_tags table:', err.message);
+}

package/dist/utils/buildContextualPrompt.js

@@ -1,74 +1,112 @@
-// File: src/utils/buildContextualPrompt.ts
-import chalk from 'chalk';
-import path from 'path';
-function estimateTokenCount(text) {
-    return Math.round(text.length / 4); // simple heuristic approximation
-}
-export function buildContextualPrompt({ baseInstruction, code, summary, functions, relatedFiles, projectFileTree, }) {
-    const parts = [baseInstruction];
-    if (summary) {
-        parts.push(`📄 File Summary:\n${summary}`);
-    }
-    if (functions?.length) {
-        const formattedFunctions = functions
-            .map(fn => `• ${fn.name}:\n${fn.content}`)
-            .join('\n\n');
-        parts.push(`🔧 Functions:\n${formattedFunctions}`);
+// src/utils/buildContextualPrompt.ts
+import { getDbForRepo } from "../db/client.js";
+import { generateFocusedFileTree } from "./fileTree.js";
+export async function buildContextualPrompt({ topFile, query, kgDepth = 3, }) {
+    const db = getDbForRepo();
+    const log = (...args) => console.log("[buildContextualPrompt]", ...args);
+    const promptSections = [];
+    const seenPaths = new Set();
+    function summarizeForPrompt(summary, maxLines = 5) {
+        if (!summary)
+            return undefined;
+        const lines = summary.split("\n").map(l => l.trim()).filter(Boolean);
+        if (lines.length <= maxLines)
+            return lines.join(" ");
+        return lines.slice(0, maxLines).join(" ") + " …";
     }
-    else {
-        console.log(chalk.yellow(` ⚠️ No functions found in top rated file.`));
+    // --- Step 1: Top file summary ---
+    if (topFile.summary) {
+        promptSections.push(`**Top file:** ${topFile.path}\n${topFile.summary}`);
+        seenPaths.add(topFile.path);
     }
-    if (relatedFiles?.length) {
-        const formattedRelatedFiles = relatedFiles
-            .map(f => {
-            const relatedFunctions = f.functions
-                .map(fn => `  • ${fn.name}:\n    ${fn.content}`)
-                .join('\n\n');
-            return `• ${f.path}: ${f.summary}\n${relatedFunctions}`;
-        })
-            .join('\n\n');
-        parts.push(`📚 Related Files:\n${formattedRelatedFiles}`);
+    // --- Step 2: KG entities/tags for top file ---
+    const topEntitiesStmt = db.prepare(`
+    SELECT et.entity_type, et.entity_id, tm.name AS tag
+    FROM entity_tags et
+    JOIN tags_master tm ON et.tag_id = tm.id
+    WHERE et.entity_id = ?
+  `);
+    const topEntitiesRows = topEntitiesStmt.all(topFile.id);
+    if (topEntitiesRows.length > 0) {
+        const tags = topEntitiesRows.map(r => `- **${r.entity_type}**: ${r.tag}`);
+        promptSections.push(`**Knowledge Graph context for ${topFile.path}:**\n${tags.join("\n")}`);
     }
-    if (projectFileTree) {
-        parts.push(`📁 Project File Structure:\n\`\`\`\n${projectFileTree.trim()}\n\`\`\``);
+    // --- Step 3: Recursive KG traversal ---
+    const kgRelatedStmt = db.prepare(`
+    SELECT DISTINCT f.id, f.path, f.summary
+    FROM edges e
+    JOIN files f ON e.target_id = f.id
+    WHERE e.source_type = 'file'
+      AND e.target_type = 'file'
+      AND e.source_id = ?
+  `);
+    function getRelatedKGFiles(fileId, visited = new Set()) {
+        if (visited.has(fileId)) {
+            log(`🔹 Already visited fileId ${fileId}, skipping`);
+            return [];
+        }
+        visited.add(fileId);
+        const rows = kgRelatedStmt.all(fileId);
+        if (rows.length === 0) {
+            log(`⚠️ No edges found for fileId ${fileId}`);
+        }
+        else {
+            log(`🔹 Found ${rows.length} related files for fileId ${fileId}:`, rows.map(r => r.path));
+        }
+        let results = [];
+        for (const row of rows) {
+            results.push(row);
+            results.push(...getRelatedKGFiles(row.id, visited));
+        }
+        return results;
     }
-    parts.push(`\n--- CODE START ---\n${code}\n--- CODE END ---`);
-    const prompt = parts.join('\n\n');
-    const tokenEstimate = estimateTokenCount(prompt);
-    // 🔵 Colorized diagnostic output
-    // 🔵 Colorized diagnostic output
-    const header = chalk.bgBlue.white.bold(' [SCAI] Prompt Overview ');
-    const labelColor = chalk.cyan;
-    const contentColor = chalk.gray;
-    console.log('\n' + header);
-    console.log(labelColor('🔢 Token Estimate:'), contentColor(`${tokenEstimate.toLocaleString()} tokens`));
-    // 📄 Summary
-    if (summary) {
-        console.log(labelColor('📄 Summary:'), contentColor(`${estimateTokenCount(summary).toLocaleString()} tokens`));
+    function buildFileTree(file, depth, visited = new Set()) {
+        log(`buildFileTree - file=${file.path}, depth=${depth}`);
+        if (depth === 0 || visited.has(file.id)) {
+            return { id: file.id.toString(), path: file.path, summary: summarizeForPrompt(file.summary) };
+        }
+        visited.add(file.id);
+        const relatedFiles = getRelatedKGFiles(file.id, visited)
+            .map(f => ({ id: f.id, path: f.path, summary: f.summary }))
+            .slice(0, 5); // limit max 5 children per node
+        log(`File ${file.path} has ${relatedFiles.length} related files`);
+        const relatedNodes = relatedFiles.map(f => buildFileTree(f, depth - 1, visited));
+        return {
+            id: file.id.toString(),
+            path: file.path,
+            summary: summarizeForPrompt(file.summary),
+            related: relatedNodes.length ? relatedNodes : undefined,
+        };
     }
-    // 🔧 Functions
-    if (functions?.length) {
-        const fnCount = functions.length;
-        const fnTokens = functions.reduce((sum, f) => sum + estimateTokenCount(f.content), 0);
-        console.log(labelColor(`🔧 Functions (${fnCount}):`), contentColor(`${fnTokens.toLocaleString()} tokens`));
+    const kgTree = buildFileTree({ id: topFile.id, path: topFile.path, summary: topFile.summary }, kgDepth);
+    const kgJson = JSON.stringify(kgTree, null, 2);
+    promptSections.push(`**KG-Related Files (JSON tree, depth ${kgDepth}):**\n\`\`\`json\n${kgJson}\n\`\`\``);
+    // --- Step 4: File tree (shallow, depth 2) ---
+    let fileTree = "";
+    try {
+        fileTree = generateFocusedFileTree(topFile.path, 2);
+        if (fileTree) {
+            promptSections.push(`**Focused File Tree (depth 2):**\n\`\`\`\n${fileTree}\n\`\`\``);
+        }
     }
-    // 📚 Related Files
-    if (relatedFiles?.length) {
-        const relCount = relatedFiles.length;
-        const relTokens = relatedFiles.reduce((sum, f) => sum + estimateTokenCount(f.summary), 0);
-        console.log(labelColor(`📚 Related Files (${relCount}):`), contentColor(`${relTokens.toLocaleString()} tokens`));
-        // Optional: Show top 3 file names
-        const fileList = relatedFiles.slice(0, 3).map(f => `- ${path.basename(f.path)}`).join('\n');
-        if (fileList)
-            console.log(contentColor(fileList + (relCount > 3 ? `\n  ...+${relCount - 3} more` : '')));
+    catch (e) {
+        console.warn("⚠️ Could not generate file tree:", e);
     }
-    // 📁 File Tree
-    if (projectFileTree) {
-        console.log(labelColor('📁 File Tree:'), contentColor(`${estimateTokenCount(projectFileTree).toLocaleString()} tokens`));
+    // --- Step 5: Code snippet ---
+    const MAX_LINES = 50;
+    if (topFile.code) {
+        const lines = topFile.code.split("\n").slice(0, MAX_LINES);
+        let snippet = lines.join("\n");
+        if (topFile.code.split("\n").length > MAX_LINES) {
+            snippet += "\n... [truncated]";
+        }
+        promptSections.push(`**Code Context (first ${MAX_LINES} lines):**\n\`\`\`\n${snippet}\n\`\`\``);
     }
-    // 📦 Code Section
-    console.log(labelColor('📦 Code:'), contentColor(`${estimateTokenCount(prompt).toLocaleString()} tokens`));
-    // 📌 Key
-    console.log(labelColor('🔍 Key:'), contentColor('[buildContextualPrompt]\n'));
-    return prompt;
+    // --- Step 6: User query ---
+    promptSections.push(`**Query:** ${query}`);
+    // --- Step 7: Combine prompt ---
+    const promptText = promptSections.join("\n\n---\n\n");
+    log("✅ Contextual prompt built for:", topFile.path);
+    log("📄 Prompt preview:\n", promptText + "\n");
+    return promptText;
 }

package/dist/utils/log.js

@@ -1,5 +1,5 @@
+import { LOG_PATH } from "../constants.js";
 import fs from 'fs';
-import { LOG_PATH } from '../constants.js';
 export function log(...args) {
     const timestamp = new Date().toISOString();
     const message = args.map(arg => typeof arg === 'string' ? arg : JSON.stringify(arg, null, 2)).join(' ');

package/dist/utils/sanitizeQuery.js

@@ -2,18 +2,26 @@
 import { STOP_WORDS } from '../fileRules/stopWords.js';
 export function sanitizeQueryForFts(input) {
     input = input.trim().toLowerCase();
-    // If it's a single filename-like string (includes dots or slashes), quote it
+    // If the whole input looks like a filename/path, quote it
     if (/^[\w\-./]+$/.test(input) && !/\s/.test(input)) {
-        // Escape quotes and wrap with double-quotes for FTS safety
         return `"${input.replace(/"/g, '""')}"*`;
     }
-    // Otherwise, treat it as a natural language prompt
     const tokens = input
         .split(/\s+/)
         .map(token => token.toLowerCase())
+        .map(token => {
+        // If the token looks like a filename/path, keep it quoted
+        if (/[\w]+\.[a-z0-9]+$/.test(token)) {
+            return `"${token.replace(/"/g, '""')}"`;
+        }
+        // Otherwise, clean it like normal
+        return token
+            .replace(/[^a-z0-9_*"]/gi, '') // remove all invalid FTS5 chars
+            .replace(/'/g, "''");
+    })
         .filter(token => token.length > 2 &&
-        !STOP_WORDS.has(token) &&
-        /^[a-z0-9]+$/.test(token))
-        .map(token => token.replace(/[?*\\"]/g, '').replace(/'/g, "''") + '*');
+        !STOP_WORDS.has(token.replace(/[*"]/g, '')) // check unquoted
+    )
+        .map(token => (token.startsWith('"') ? token : token + '*'));
     return tokens.length > 0 ? tokens.join(' OR ') : '*';
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.109",
+  "version": "0.1.111",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"
@@ -34,7 +34,7 @@
     "workflow"
   ],
   "scripts": {
-    "build": "rm -rfd dist && tsc && git add .",
+    "build": "rm -rfd dist && tsc && chmod +x dist/index.js && git add .",
     "start": "node dist/index.js"
   },
   "dependencies": {