scai 0.1.115 → 0.1.117

package/dist/scripts/dbcheck.js

@@ -9,7 +9,6 @@ if (!repoKey) {
     console.error("❌ No active repo found. Use `scai set-index` to set one.");
     process.exit(1);
 }
-// Get the basename (repo name) from the full repoKey (which is the path)
 const repoName = path.basename(repoKey);
 const scaiRepoRoot = path.join(os.homedir(), ".scai", "repos", repoName);
 const dbPath = path.join(scaiRepoRoot, "db.sqlite");
@@ -18,307 +17,253 @@ if (!fs.existsSync(dbPath)) {
     process.exit(1);
 }
 const db = new Database(dbPath);
-// === Basic Stats ===
-const stats = {
-    total: db.prepare('SELECT COUNT(*) as count FROM files').get().count,
-    withSummary: db.prepare(`SELECT COUNT(*) as count FROM files WHERE summary IS NOT NULL AND summary != ''`).get().count,
-    withoutSummary: db.prepare(`SELECT COUNT(*) as count FROM files WHERE summary IS NULL OR summary = ''`).get().count,
-    withEmbedding: db.prepare(`SELECT COUNT(*) as count FROM files WHERE embedding IS NOT NULL AND embedding != ''`).get().count,
-    withoutEmbedding: db.prepare(`SELECT COUNT(*) as count FROM files WHERE embedding IS NULL OR embedding = ''`).get().count,
-    withProcessingStatusExtracted: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status = 'extracted'`).get().count,
-    withoutProcessingStatusExtracted: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status IS NULL OR processing_status != 'extracted'`).get().count,
-    withProcessingStatusSkipped: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status = 'skipped'`).get().count,
-    withProcessingStatusFailed: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status = 'failed'`).get().count,
-    withProcessingStatusUnprocessed: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status = 'unprocessed'`).get().count,
-};
-console.log("📊 SQLite Stats for Table: files");
-console.log("-------------------------------------------");
-console.log(`🔢 Total rows:               ${stats.total}`);
-console.log(`✅ With summary:             ${stats.withSummary}`);
-console.log(`❌ Without summary:          ${stats.withoutSummary}`);
-console.log(`✅ With embedding:           ${stats.withEmbedding}`);
-console.log(`❌ Without embedding:        ${stats.withoutEmbedding}`);
-console.log(`✅ With processing_status = 'extracted': ${stats.withProcessingStatusExtracted}`);
-console.log(`❌ Without processing_status = 'extracted': ${stats.withoutProcessingStatusExtracted}`);
-console.log(`✅ With processing_status = 'skipped':     ${stats.withProcessingStatusSkipped}`);
-console.log(`✅ With processing_status = 'failed':      ${stats.withProcessingStatusFailed}`);
-console.log(`✅ With processing_status = 'unprocessed': ${stats.withProcessingStatusUnprocessed}`);
-// === Example Summaries ===
-console.log("\n🧾 Example summaries and embeddings:\n--------------------------");
-console.log("Example summaries (first 50 characters):");
-const summaries = db.prepare(`
-  SELECT id, substr(summary, 1, 50) || '...' AS short_summary
+// Utility to log simple table stats
+function tableStats(table, column) {
+    const total = db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get().count;
+    let extra = "";
+    if (column) {
+        const nonNull = db.prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${column} IS NOT NULL AND ${column} != ''`).get().count;
+        extra = ` | Non-null ${column}: ${nonNull}`;
+    }
+    console.log(`📊 ${table}: ${total}${extra}`);
+}
+// === Files Table ===
+console.log("\n🗂 Table: files");
+tableStats("files", "summary");
+tableStats("files", "embedding");
+const processingStatuses = ["extracted", "skipped", "failed", "unprocessed"];
+for (const status of processingStatuses) {
+    const count = db.prepare(`SELECT COUNT(*) AS count FROM files WHERE processing_status = ?`).get(status).count;
+    console.log(`   Status '${status}': ${count}`);
+}
+// === Example Files ===
+const exampleFiles = db.prepare(`
+  SELECT id, filename, substr(summary,1,50) AS summary, substr(embedding,1,50) AS embedding
   FROM files
-  WHERE summary IS NOT NULL AND summary != ''
   LIMIT 5
 `).all();
-summaries.forEach(row => {
-    console.log(`${row.id.toString().padEnd(4)}  ${row.short_summary}`);
+console.log("\n🧾 Example Files:");
+exampleFiles.forEach(f => {
+    console.log(`   [${f.id}] ${f.filename} | summary: "${f.summary}" | embedding: "${f.embedding}"`);
 });
-console.log("\nExample embeddings (first 50 characters):");
-const embeddings = db.prepare(`
-  SELECT id, substr(embedding, 1, 50) || '...' AS short_embedding
-  FROM files
-  WHERE embedding IS NOT NULL AND embedding != ''
+// === FTS5 Check ===
+const ftsExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='files_fts'`).get();
+console.log("\n🔍 FTS5 Table:", ftsExists ? "✅ exists" : "❌ missing");
+// === Functions Table ===
+console.log("\n🧑‍💻 Table: functions");
+tableStats("functions");
+const sampleFuncs = db.prepare(`
+  SELECT id, name, file_id, substr(content,1,50) AS preview
+  FROM functions
+  ORDER BY RANDOM()
   LIMIT 5
 `).all();
-embeddings.forEach(row => {
-    console.log(`${row.id.toString().padEnd(4)}  ${row.short_embedding}`);
+sampleFuncs.forEach(f => {
+    console.log(`   [${f.id}] ${f.name} (file ${f.file_id}): ${f.preview}`);
 });
-// === FTS5 Check ===
-console.log("\n🔍 FTS5 Check");
-console.log("--------------------------");
-try {
-    const ftsExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='files_fts'`).get();
-    if (!ftsExists) {
-        console.log("❌ No FTS5 table `files_fts` found in database.");
-    }
-    else {
-        const ftsRowCount = db.prepare(`SELECT COUNT(*) AS count FROM files_fts`).get().count;
-        console.log(`✅ files_fts table exists. Rows: ${ftsRowCount}`);
-    }
-}
-catch (err) {
-    console.error("❌ Error while accessing files_fts:", err.message);
-}
-// === Rebuild FTS Index ===
-console.log("\n🔧 Rebuilding FTS5 index...");
-try {
-    db.prepare(`INSERT INTO files_fts(files_fts) VALUES ('rebuild')`).run();
-    console.log(`✅ Rebuild completed.`);
-}
-catch (err) {
-    console.error("❌ FTS5 rebuild failed:", err.message);
-}
-// === FTS Search Test ===
-console.log('\n🔍 Test MATCH query for "minimap":');
-try {
-    const minimapMatches = db.prepare(`
-    SELECT f.id, f.path 
-    FROM files f
-    JOIN files_fts fts ON f.id = fts.rowid
-    WHERE fts.files_fts MATCH 'minimap'
-    LIMIT 10
-  `).all();
-    if (minimapMatches.length === 0) {
-        console.warn('⚠️ No matches for "minimap" in FTS index');
-    }
-    else {
-        minimapMatches.forEach(row => {
-            console.log(`📄 ${row.path}`);
-        });
-    }
-}
-catch (err) {
-    console.error('❌ Error running MATCH query:', err.message);
-}
-// === Direct LIKE Fallback ===
-console.log('\n🔍 Direct LIKE query on path for "minimap":');
-const likeMatches = db.prepare(`
-  SELECT id, path 
-  FROM files 
-  WHERE path LIKE '%minimap%' 
-  LIMIT 10
+// === Graph Classes Table ===
+console.log("\n🏷 Table: graph_classes");
+tableStats("graph_classes");
+const sampleClasses = db.prepare(`
+  SELECT id, name, file_id, substr(content,1,50) AS preview
+  FROM graph_classes
+  ORDER BY RANDOM()
+  LIMIT 5
 `).all();
-if (likeMatches.length === 0) {
-    console.warn('⚠️ No file paths contain "minimap"');
-}
-else {
-    likeMatches.forEach(row => {
-        console.log(`📄 ${row.path}`);
-    });
-}
-// === Function Table Stats ===
-console.log('\n📊 Stats for Table: functions');
-console.log('-------------------------------------------');
-try {
-    const funcCount = db.prepare(`SELECT COUNT(*) AS count FROM functions`).get().count;
-    const distinctFiles = db.prepare(`SELECT COUNT(DISTINCT file_id) AS count FROM functions`).get().count;
-    console.log(`🔢 Total functions:         ${funcCount}`);
-    console.log(`📂 Distinct files:          ${distinctFiles}`);
-}
-catch (err) {
-    console.error('❌ Error accessing functions table:', err.message);
-}
-// === Example Functions ===
-console.log('\n🧪 Example extracted functions:');
-try {
-    const sampleFunctions = db.prepare(`
-    SELECT id, name, start_line, end_line, substr(content, 1, 100) || '...' AS short_body
-    FROM functions
-    ORDER BY id DESC
-    LIMIT 5
-  `).all();
-    sampleFunctions.forEach(fn => {
-        console.log(`🔹 ID: ${fn.id}`);
-        console.log(`   Name: ${fn.name}`);
-        console.log(`   Lines: ${fn.start_line}-${fn.end_line}`);
-        console.log(`   Body: ${fn.short_body}\n`);
-    });
-}
-catch (err) {
-    console.error('❌ Error printing function examples:', err.message);
-}
-// === Function Calls Table Stats ===
-console.log('\n📊 Stats for Table: function_calls');
-console.log('-------------------------------------------');
-try {
-    const callCount = db.prepare(`SELECT COUNT(*) AS count FROM function_calls`).get().count;
-    const topCallers = db.prepare(`
-    SELECT caller_id, COUNT(*) AS num_calls 
-    FROM function_calls 
-    GROUP BY caller_id 
-    ORDER BY num_calls DESC 
-    LIMIT 5
-  `).all();
-    console.log(`🔁 Total function calls:    ${callCount}`);
-    console.log('📞 Top callers:');
-    topCallers.forEach(row => {
-        console.log(`   - Caller ${row.caller_id} made ${row.num_calls} calls`);
-    });
-}
-catch (err) {
-    console.error('❌ Error accessing function_calls table:', err.message);
-}
-// === Random Summary Samples ===
-console.log('\n🧾 Random Summaries (ID + Preview):');
-console.log('-------------------------------------------');
-const randomSummaries = db.prepare(`
-  SELECT id, filename, substr(summary, 1, 1000) || '...' AS preview
-  FROM files
-  WHERE summary IS NOT NULL AND summary != ''
+sampleClasses.forEach(c => {
+    console.log(`   [${c.id}] ${c.name} (file ${c.file_id}): ${c.preview}`);
+});
+// === Graph Edges Table ===
+console.log("\n🔗 Table: graph_edges");
+tableStats("graph_edges", "relation");
+// Dynamically get column names
+const columns = db.prepare(`PRAGMA table_info(graph_edges)`).all();
+const colNames = columns.map(c => c.name);
+// Determine source/target ID columns
+const sourceCol = colNames.includes("source_id") ? "source_id" : "source_unique_id";
+const targetCol = colNames.includes("target_id") ? "target_id" : "target_unique_id";
+const sampleEdges = db.prepare(`
+  SELECT id, source_type, ${sourceCol} AS source_id,
+         target_type, ${targetCol} AS target_id,
+         relation
+  FROM graph_edges
   ORDER BY RANDOM()
   LIMIT 5
 `).all();
-randomSummaries.forEach(row => {
-    console.log(`📄 [${row.id}] ${row.filename}: ${row.preview}`);
+sampleEdges.forEach(e => {
+    console.log(`   [${e.id}] ${e.source_type}(${e.source_id}) -[${e.relation}]-> ${e.target_type}(${e.target_id})`);
 });
-// === Random Functions Samples ===
-console.log('\n🧑‍💻 20 Random Functions (ID, name, body):');
-console.log('-------------------------------------------');
-const randomFunctions = db.prepare(`
-  SELECT id, name, file_id, substr(content, 1, 100) || '...' AS preview
-  FROM functions
+// === Tags Tables ===
+console.log("\n🏷 Table: graph_tags_master");
+tableStats("graph_tags_master");
+const sampleTags = db.prepare(`
+  SELECT id, name
+  FROM graph_tags_master
   ORDER BY RANDOM()
   LIMIT 5
 `).all();
-randomFunctions.forEach(row => {
-    console.log(`🔹 [${row.id}] ${row.name} (file_id: ${row.file_id})`);
-    console.log(`   ${row.preview}\n`);
+sampleTags.forEach(t => {
+    console.log(`   [${t.id}] ${t.name}`);
 });
-// === Column View of Files ===
-console.log('\n📊 Table View: Files');
-console.log('-------------------------------------------');
-const fileRows = db.prepare(`
-  SELECT id, filename, type, processing_status, functions_extracted_at, length(summary) AS summary_len
-  FROM files
-  LIMIT 50
+console.log("\n🔖 Table: graph_entity_tags");
+tableStats("graph_entity_tags");
+const sampleEntityTags = db.prepare(`
+  SELECT id, entity_type, entity_unique_id AS entity_id, tag_id
+  FROM graph_entity_tags
+  ORDER BY RANDOM()
+  LIMIT 5
 `).all();
-console.table(fileRows);
-// === Column View of Functions ===
-console.log('\n📊 Table View: Functions');
-console.log('-------------------------------------------');
-const functionRows = db.prepare(`
-  SELECT id, file_id, name, start_line, end_line, length(content) AS length
-  FROM functions
-  LIMIT 50
+sampleEntityTags.forEach(et => {
+    console.log(`   [${et.id}] ${et.entity_type}(${et.entity_id}) -> tag ${et.tag_id}`);
+});
+// === Function → Edges Check ===
+console.log("\n🕸 Function Calls / CalledBy consistency");
+// 1. Check for edges pointing to missing functions
+const missingFuncs = db.prepare(`
+  SELECT e.id, e.source_unique_id, e.target_unique_id
+  FROM graph_edges e
+  WHERE e.source_type = 'function'
+    AND NOT EXISTS (SELECT 1 FROM functions f WHERE f.unique_id = e.source_unique_id)
+  UNION
+  SELECT e.id, e.source_unique_id, e.target_unique_id
+  FROM graph_edges e
+  WHERE e.target_type = 'function'
+    AND NOT EXISTS (SELECT 1 FROM functions f WHERE f.unique_id = e.target_unique_id)
+  LIMIT 10
 `).all();
-console.table(functionRows);
-// === Class Table Stats ===
-console.log('\n📊 Stats for Table: classes');
-console.log('-------------------------------------------');
-try {
-    const classCount = db.prepare(`SELECT COUNT(*) AS count FROM classes`).get().count;
-    const distinctClassFiles = db.prepare(`SELECT COUNT(DISTINCT file_id) AS count FROM classes`).get().count;
-    console.log(`🏷 Total classes:           ${classCount}`);
-    console.log(`📂 Distinct files:          ${distinctClassFiles}`);
-}
-catch (err) {
-    console.error('❌ Error accessing classes table:', err.message);
+if (missingFuncs.length === 0) {
+    console.log("✅ All edges reference valid functions.");
 }
-// === Example Classes ===
-console.log('\n🧪 Example extracted classes:');
-try {
-    const sampleClasses = db.prepare(`
-    SELECT id, name, file_id, start_line, end_line, substr(content, 1, 100) || '...' AS short_body
-    FROM classes
-    ORDER BY id DESC
-    LIMIT 5
-  `).all();
-    sampleClasses.forEach(cls => {
-        console.log(`🏷 ID: ${cls.id}`);
-        console.log(`   Name: ${cls.name}`);
-        console.log(`   File: ${cls.file_id}`);
-        console.log(`   Lines: ${cls.start_line}-${cls.end_line}`);
-        console.log(`   Body: ${cls.short_body}\n`);
+else {
+    console.log("❌ Found edges pointing to missing functions:");
+    missingFuncs.forEach(e => {
+        console.log(`   Edge ${e.id}: ${e.source_unique_id} -> ${e.target_unique_id}`);
     });
 }
-catch (err) {
-    console.error('❌ Error printing class examples:', err.message);
+// 2. Functions with outgoing calls
+const funcWithCalls = db.prepare(`
+  SELECT f.id, f.name, COUNT(e.id) AS callCount
+  FROM functions f
+  JOIN graph_edges e
+    ON e.source_type = 'function'
+   AND e.source_unique_id = f.unique_id
+  GROUP BY f.id
+  HAVING callCount > 0
+  ORDER BY callCount DESC
+  LIMIT 5
+`).all();
+funcWithCalls.forEach(f => {
+    console.log(`   🔹 Function [${f.id}] ${f.name} has ${f.callCount} outgoing calls`);
+});
+// 3. Functions with incoming calls
+const funcWithCalledBy = db.prepare(`
+  SELECT f.id, f.name, COUNT(e.id) AS calledByCount
+  FROM functions f
+  JOIN graph_edges e
+    ON e.target_type = 'function'
+   AND e.target_unique_id = f.unique_id
+  GROUP BY f.id
+  HAVING calledByCount > 0
+  ORDER BY calledByCount DESC
+  LIMIT 5
+`).all();
+funcWithCalledBy.forEach(f => {
+    console.log(`   🔸 Function [${f.id}] ${f.name} is called by ${f.calledByCount} functions`);
+});
+// 4. Check for duplicate edges (same source→target)
+const duplicateEdges = db.prepare(`
+  SELECT source_unique_id, target_unique_id, COUNT(*) as dupCount
+  FROM graph_edges
+  WHERE source_type = 'function' AND target_type = 'function'
+  GROUP BY source_unique_id, target_unique_id
+  HAVING dupCount > 1
+  LIMIT 5
+`).all();
+if (duplicateEdges.length > 0) {
+    console.log("⚠️ Duplicate function call edges found:");
+    duplicateEdges.forEach(d => {
+        console.log(`   ${d.source_unique_id} -> ${d.target_unique_id} (x${d.dupCount})`);
+    });
 }
-// === Edge Table Stats ===
-console.log('\n📊 Stats for Table: edges');
-console.log('-------------------------------------------');
-try {
-    const edgeCount = db.prepare(`SELECT COUNT(*) AS count FROM edges`).get().count;
-    const distinctRelations = db.prepare(`SELECT COUNT(DISTINCT relation) AS count FROM edges`).get().count;
-    console.log(`🔗 Total edges:             ${edgeCount}`);
-    console.log(`🧩 Distinct relations:      ${distinctRelations}`);
+else {
+    console.log("✅ No duplicate function call edges.");
 }
-catch (err) {
-    console.error('❌ Error accessing edges table:', err.message);
+// === File-specific check (AskCmd.ts) ===
+const targetPath = "/Users/rzs/dev/repos/scai/cli/src/commands/AskCmd.ts";
+// --- MUST KEEP: find the file row ---
+const targetFile = db.prepare(`
+  SELECT id, path, filename, processing_status
+  FROM files
+  WHERE path = ?
+`).get(targetPath);
+if (!targetFile) {
+    console.log(`❌ File not found in DB: ${targetPath}`);
 }
-// === Example Edges ===
-console.log('\n🧪 Example edges:');
-try {
-    const sampleEdges = db.prepare(`
-    SELECT id, source_id, target_id, relation
-    FROM edges
-    ORDER BY id DESC
-    LIMIT 10
-  `).all();
-    sampleEdges.forEach(e => {
-        console.log(`🔗 Edge ${e.id}: ${e.source_id} -[${e.relation}]-> ${e.target_id}`);
+else {
+    console.log(`\n📂 File found: [${targetFile.id}] ${targetFile.filename} (${targetFile.processing_status})`);
+    // --- MUST KEEP: list functions in this file ---
+    const funcs = db.prepare(`
+    SELECT id, name, unique_id, start_line, end_line
+    FROM functions
+    WHERE file_id = ?
+    ORDER BY start_line
+  `).all(targetFile.id);
+    console.log(`   Functions (${funcs.length}):`);
+    funcs.forEach((f) => {
+        console.log(`     [${f.id}] ${f.name} (${f.unique_id}) lines ${f.start_line}-${f.end_line}`);
     });
-}
-catch (err) {
-    console.error('❌ Error printing edge examples:', err.message);
-}
-// === Tags Master Stats ===
-console.log('\n📊 Stats for Table: tags_master');
-console.log('-------------------------------------------');
-try {
-    const tagCount = db.prepare(`SELECT COUNT(*) AS count FROM tags_master`).get().count;
-    console.log(`🏷 Total tags: ${tagCount}`);
-    const sampleTags = db.prepare(`
-    SELECT id, name
-    FROM tags_master
-    ORDER BY id DESC
-    LIMIT 5
-  `).all();
-    sampleTags.forEach(tag => {
-        console.log(`🏷 Tag ${tag.id}: ${tag.name}`);
+    // --- OPTIONAL: outgoing calls from this file’s functions ---
+    const outgoing = db.prepare(`
+    SELECT f.name AS source_name, e.target_unique_id, e.relation
+    FROM functions f
+    JOIN graph_edges e
+      ON e.source_unique_id = f.unique_id
+     AND e.source_type = 'function'
+    WHERE f.file_id = ?
+  `).all(targetFile.id);
+    console.log(`   Outgoing calls (${outgoing.length}):`);
+    outgoing.forEach((o) => {
+        console.log(`     ${o.source_name} -[${o.relation}]-> ${o.target_unique_id}`);
     });
-}
-catch (err) {
-    console.error('❌ Error accessing tags_master table:', err.message);
-}
-// === Entity Tags Stats ===
-console.log('\n📊 Stats for Table: entity_tags');
-console.log('-------------------------------------------');
-try {
-    const entityTagCount = db.prepare(`SELECT COUNT(*) AS count FROM entity_tags`).get().count;
-    console.log(`🔗 Total entity-tags: ${entityTagCount}`);
-    const sampleEntityTags = db.prepare(`
-    SELECT id, entity_type, entity_id, tag_id
-    FROM entity_tags
-    ORDER BY id DESC
-    LIMIT 10
-  `).all();
-    sampleEntityTags.forEach(et => {
-        console.log(`🔗 EntityTag ${et.id}: ${et.entity_type} ${et.entity_id} -> tag ${et.tag_id}`);
+    // --- OPTIONAL: incoming calls to this file’s functions ---
+    const incoming = db.prepare(`
+    SELECT f.name AS target_name, e.source_unique_id, e.relation
+    FROM functions f
+    JOIN graph_edges e
+      ON e.target_unique_id = f.unique_id
+     AND e.target_type = 'function'
+    WHERE f.file_id = ?
+  `).all(targetFile.id);
+    console.log(`   Incoming calls (${incoming.length}):`);
+    incoming.forEach((i) => {
+        console.log(`     ${i.source_unique_id} -[${i.relation}]-> ${i.target_name}`);
     });
+    // --- NICE TO HAVE: dangling edges check ---
+    const dangling = db.prepare(`
+    SELECT e.id, e.source_unique_id, e.target_unique_id, e.relation
+    FROM graph_edges e
+    WHERE e.source_unique_id IN (SELECT unique_id FROM functions WHERE file_id = ?)
+       OR e.target_unique_id IN (SELECT unique_id FROM functions WHERE file_id = ?)
+  `).all(targetFile.id, targetFile.id);
+    console.log(`   Edges referencing this file's functions (dangling check): ${dangling.length}`);
+    dangling.forEach((d) => {
+        console.log(`     Edge ${d.id}: ${d.source_unique_id} -[${d.relation}]-> ${d.target_unique_id}`);
+    });
+    // --- OPTIONAL: consistency check summary ---
+    if (funcs.length > 0 && outgoing.length === 0 && incoming.length === 0 && dangling.length === 0) {
+        console.log("⚠️  This file has functions but no graph edges. Possible causes:");
+        console.log("     - Edges were never extracted for this file, OR");
+        console.log("     - unique_id mismatch between functions and edges.");
+    }
+    else {
+        console.log("✅ Edges and functions are consistent for this file.");
+    }
+    const funcsDebug = db.prepare(`
+  SELECT id, name, unique_id
+  FROM functions
+  WHERE file_id = ?
+`).all(targetFile.id);
+    console.log("Funcsdebug:\n");
+    console.log(funcsDebug);
 }
-catch (err) {
-    console.error('❌ Error accessing entity_tags table:', err.message);
-}
+console.log("\n✅ DB check completed.\n");