scai 0.1.57 → 0.1.59

package/dist/db/functionExtractors/extractFromTs.js

@@ -0,0 +1,72 @@
+import { Project, SyntaxKind } from 'ts-morph';
+import path from 'path';
+import { generateEmbedding } from '../../lib/generateEmbedding.js';
+import { log } from '../../utils/log.js';
+import { getDbForRepo } from '../client.js';
+import { markFileAsSkippedTemplate, markFileAsExtractedTemplate, markFileAsFailedTemplate } from '../sqlTemplates.js';
+export async function extractFromTS(filePath, content, fileId) {
+    const db = getDbForRepo();
+    try {
+        const project = new Project({ useInMemoryFileSystem: true });
+        const sourceFile = project.createSourceFile(filePath, content);
+        const functions = [];
+        const allFuncs = [
+            ...sourceFile.getDescendantsOfKind(SyntaxKind.FunctionDeclaration),
+            ...sourceFile.getDescendantsOfKind(SyntaxKind.FunctionExpression),
+            ...sourceFile.getDescendantsOfKind(SyntaxKind.ArrowFunction),
+        ];
+        for (const fn of allFuncs) {
+            const name = fn.getSymbol()?.getName() ?? `${path.basename(filePath)}:<anon>`;
+            const start = fn.getStartLineNumber();
+            const end = fn.getEndLineNumber();
+            const code = fn.getText();
+            functions.push({ name, start_line: start, end_line: end, content: code });
+        }
+        if (functions.length === 0) {
+            log(`⚠️ No functions found in TS file: ${filePath}`);
+            db.prepare(markFileAsSkippedTemplate).run({ id: fileId });
+            return false;
+        }
+        log(`🔍 Found ${functions.length} TypeScript functions in ${filePath}`);
+        for (const fn of functions) {
+            const embedding = await generateEmbedding(fn.content);
+            const result = db.prepare(`
+        INSERT INTO functions (
+          file_id, name, start_line, end_line, content, embedding, lang
+        ) VALUES (
+          @file_id, @name, @start_line, @end_line, @content, @embedding, @lang
+        )
+      `).run({
+                file_id: fileId,
+                name: fn.name,
+                start_line: fn.start_line,
+                end_line: fn.end_line,
+                content: fn.content,
+                embedding: JSON.stringify(embedding),
+                lang: 'ts'
+            });
+            const callerId = result.lastInsertRowid;
+            // Simplified call detection (no walking for now)
+            const callMatches = fn.content.matchAll(/(\w+)\s*\(/g);
+            for (const match of callMatches) {
+                db.prepare(`
+          INSERT INTO function_calls (caller_id, callee_name)
+          VALUES (@caller_id, @callee_name)
+        `).run({
+                    caller_id: callerId,
+                    callee_name: match[1],
+                });
+            }
+            log(`📌 Indexed TS function: ${fn.name}`);
+        }
+        db.prepare(markFileAsExtractedTemplate).run({ id: fileId });
+        log(`✅ Marked TS functions as extracted for ${filePath}`);
+        return true;
+    }
+    catch (err) {
+        log(`❌ Failed to extract from TS file: ${filePath}`);
+        log(`   ↳ ${String(err.message)}`);
+        db.prepare(markFileAsFailedTemplate).run({ id: fileId });
+        return false;
+    }
+}

package/dist/db/functionExtractors/index.js

@@ -5,6 +5,7 @@ import { extractFromJS } from './extractFromJs.js';
 import { extractFromXML } from './extractFromXML.js';
 import { getDbForRepo } from '../client.js';
 import { markFileAsFailedTemplate, markFileAsSkippedByPath } from '../sqlTemplates.js';
+import { extractFromTS } from './extractFromTs.js';
 /**
  * Detects file type and delegates to the appropriate extractor.
  */
@@ -12,10 +13,14 @@ export async function extractFunctionsFromFile(filePath, content, fileId) {
     const type = detectFileType(filePath).trim().toLowerCase();
     const db = getDbForRepo();
     try {
-        if (type === 'js' || type === 'ts' || type === 'javascript' || type === 'typescript') {
+        if (type === 'js' || type === 'javascript') {
             log(`✅ Attempting to extract JS functions from ${filePath}`);
             return await extractFromJS(filePath, content, fileId);
         }
+        if (type === 'ts' || type === 'typescript') {
+            log(`📘 Extracting TS functions from ${filePath}`);
+            return await extractFromTS(filePath, content, fileId);
+        }
         if (type === 'java') {
             log(`❌ Nothing extracted for ${filePath} due to missing implementation`);
             await extractFromJava(filePath, content, fileId);

package/dist/index.js

@@ -26,6 +26,9 @@ import { promptForToken } from "./github/token.js";
 import { validateGitHubTokenAgainstRepo } from "./github/githubAuthCheck.js";
 import { checkGit } from "./commands/GitCmd.js";
 import { runSwitchCommand, runInteractiveSwitch } from "./commands/SwitchCmd.js";
+import { execSync } from "child_process";
+import path from "path";
+import fs from "fs";
 // 🎛️ CLI Setup
 const cmd = new Command('scai')
     .version(version)
@@ -174,6 +177,32 @@ index
         runInteractiveSwitch();
     }
 });
+// This will help resolve the current directory in an ES Module
+cmd
+    .command('check-db')
+    .description('Run the dbcheck script to check the database status')
+    .action(() => {
+    try {
+        const cfg = Config.getRaw();
+        const indexDir = cfg.activeRepo;
+        console.log('RepoKey: ', indexDir);
+        if (!indexDir) {
+            console.error('❌ No indexDir found for the active repo.');
+            process.exit(1);
+        }
+        // Prefer dist version if it exists (built), otherwise use src
+        let scriptPath = path.resolve(indexDir, 'dist/scripts/dbcheck.js');
+        if (!fs.existsSync(scriptPath)) {
+            scriptPath = path.resolve(indexDir, 'src/scripts/dbcheck.ts');
+        }
+        console.log(`🚀 Running database check script: ${scriptPath}`);
+        execSync(`node ${scriptPath}`, { stdio: 'inherit' });
+        console.log('✅ Database check completed!');
+    }
+    catch (err) {
+        console.error('❌ Error running dbcheck script:', err instanceof Error ? err.message : err);
+    }
+});
 cmd
     .command('backup')
     .description('Backup the current .scai folder')

package/dist/scripts/dbcheck.js

@@ -0,0 +1,226 @@
+import Database from "better-sqlite3";
+import path from "path";
+import os from "os";
+import { Config } from "../config.js";
+import fs from "fs";
+const cfg = Config.getRaw();
+const repoKey = cfg.activeRepo;
+if (!repoKey) {
+    console.error("❌ No active repo found. Use `scai set-index` to set one.");
+    process.exit(1);
+}
+// Get the basename (repo name) from the full repoKey (which is the path)
+const repoName = path.basename(repoKey);
+const scaiRepoRoot = path.join(os.homedir(), ".scai", "repos", repoName);
+const dbPath = path.join(scaiRepoRoot, "db.sqlite");
+if (!fs.existsSync(dbPath)) {
+    console.error(`❌ No database found at ${dbPath}`);
+    process.exit(1);
+}
+const db = new Database(dbPath);
+// === Basic Stats ===
+const stats = {
+    total: db.prepare('SELECT COUNT(*) as count FROM files').get().count,
+    withSummary: db.prepare(`SELECT COUNT(*) as count FROM files WHERE summary IS NOT NULL AND summary != ''`).get().count,
+    withoutSummary: db.prepare(`SELECT COUNT(*) as count FROM files WHERE summary IS NULL OR summary = ''`).get().count,
+    withEmbedding: db.prepare(`SELECT COUNT(*) as count FROM files WHERE embedding IS NOT NULL AND embedding != ''`).get().count,
+    withoutEmbedding: db.prepare(`SELECT COUNT(*) as count FROM files WHERE embedding IS NULL OR embedding = ''`).get().count,
+    withProcessingStatusExtracted: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status = 'extracted'`).get().count,
+    withoutProcessingStatusExtracted: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status IS NULL OR processing_status != 'extracted'`).get().count,
+    withProcessingStatusSkipped: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status = 'skipped'`).get().count,
+    withProcessingStatusFailed: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status = 'failed'`).get().count,
+    withProcessingStatusUnprocessed: db.prepare(`SELECT COUNT(*) as count FROM files WHERE processing_status = 'unprocessed'`).get().count,
+};
+console.log("📊 SQLite Stats for Table: files");
+console.log("-------------------------------------------");
+console.log(`🔢 Total rows:               ${stats.total}`);
+console.log(`✅ With summary:             ${stats.withSummary}`);
+console.log(`❌ Without summary:          ${stats.withoutSummary}`);
+console.log(`✅ With embedding:           ${stats.withEmbedding}`);
+console.log(`❌ Without embedding:        ${stats.withoutEmbedding}`);
+console.log(`✅ With processing_status = 'extracted': ${stats.withProcessingStatusExtracted}`);
+console.log(`❌ Without processing_status = 'extracted': ${stats.withoutProcessingStatusExtracted}`);
+console.log(`✅ With processing_status = 'skipped':     ${stats.withProcessingStatusSkipped}`);
+console.log(`✅ With processing_status = 'failed':      ${stats.withProcessingStatusFailed}`);
+console.log(`✅ With processing_status = 'unprocessed': ${stats.withProcessingStatusUnprocessed}`);
+// === Example Summaries ===
+console.log("\n🧾 Example summaries and embeddings:\n--------------------------");
+console.log("Example summaries (first 50 characters):");
+const summaries = db.prepare(`
+  SELECT id, substr(summary, 1, 50) || '...' AS short_summary
+  FROM files
+  WHERE summary IS NOT NULL AND summary != ''
+  LIMIT 10
+`).all();
+summaries.forEach(row => {
+    console.log(`${row.id.toString().padEnd(4)}  ${row.short_summary}`);
+});
+console.log("\nExample embeddings (first 50 characters):");
+const embeddings = db.prepare(`
+  SELECT id, substr(embedding, 1, 50) || '...' AS short_embedding
+  FROM files
+  WHERE embedding IS NOT NULL AND embedding != ''
+  LIMIT 10
+`).all();
+embeddings.forEach(row => {
+    console.log(`${row.id.toString().padEnd(4)}  ${row.short_embedding}`);
+});
+// === FTS5 Check ===
+console.log("\n🔍 FTS5 Check");
+console.log("--------------------------");
+try {
+    const ftsExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='files_fts'`).get();
+    if (!ftsExists) {
+        console.log("❌ No FTS5 table `files_fts` found in database.");
+    }
+    else {
+        const ftsRowCount = db.prepare(`SELECT COUNT(*) AS count FROM files_fts`).get().count;
+        console.log(`✅ files_fts table exists. Rows: ${ftsRowCount}`);
+    }
+}
+catch (err) {
+    console.error("❌ Error while accessing files_fts:", err.message);
+}
+// === Rebuild FTS Index ===
+console.log("\n🔧 Rebuilding FTS5 index...");
+try {
+    db.prepare(`INSERT INTO files_fts(files_fts) VALUES ('rebuild')`).run();
+    console.log(`✅ Rebuild completed.`);
+}
+catch (err) {
+    console.error("❌ FTS5 rebuild failed:", err.message);
+}
+// === FTS Search Test ===
+console.log('\n🔍 Test MATCH query for "minimap":');
+try {
+    const minimapMatches = db.prepare(`
+    SELECT f.id, f.path 
+    FROM files f
+    JOIN files_fts fts ON f.id = fts.rowid
+    WHERE fts.files_fts MATCH 'minimap'
+    LIMIT 10
+  `).all();
+    if (minimapMatches.length === 0) {
+        console.warn('⚠️ No matches for "minimap" in FTS index');
+    }
+    else {
+        minimapMatches.forEach(row => {
+            console.log(`📄 ${row.path}`);
+        });
+    }
+}
+catch (err) {
+    console.error('❌ Error running MATCH query:', err.message);
+}
+// === Direct LIKE Fallback ===
+console.log('\n🔍 Direct LIKE query on path for "minimap":');
+const likeMatches = db.prepare(`
+  SELECT id, path 
+  FROM files 
+  WHERE path LIKE '%minimap%' 
+  LIMIT 10
+`).all();
+if (likeMatches.length === 0) {
+    console.warn('⚠️ No file paths contain "minimap"');
+}
+else {
+    likeMatches.forEach(row => {
+        console.log(`📄 ${row.path}`);
+    });
+}
+// === Function Table Stats ===
+console.log('\n📊 Stats for Table: functions');
+console.log('-------------------------------------------');
+try {
+    const funcCount = db.prepare(`SELECT COUNT(*) AS count FROM functions`).get().count;
+    const distinctFiles = db.prepare(`SELECT COUNT(DISTINCT file_id) AS count FROM functions`).get().count;
+    console.log(`🔢 Total functions:         ${funcCount}`);
+    console.log(`📂 Distinct files:          ${distinctFiles}`);
+}
+catch (err) {
+    console.error('❌ Error accessing functions table:', err.message);
+}
+// === Example Functions ===
+console.log('\n🧪 Example extracted functions:');
+try {
+    const sampleFunctions = db.prepare(`
+    SELECT id, name, start_line, end_line, substr(content, 1, 100) || '...' AS short_body
+    FROM functions
+    ORDER BY id DESC
+    LIMIT 5
+  `).all();
+    sampleFunctions.forEach(fn => {
+        console.log(`🔹 ID: ${fn.id}`);
+        console.log(`   Name: ${fn.name}`);
+        console.log(`   Lines: ${fn.start_line}-${fn.end_line}`);
+        console.log(`   Body: ${fn.short_body}\n`);
+    });
+}
+catch (err) {
+    console.error('❌ Error printing function examples:', err.message);
+}
+// === Function Calls Table Stats ===
+console.log('\n📊 Stats for Table: function_calls');
+console.log('-------------------------------------------');
+try {
+    const callCount = db.prepare(`SELECT COUNT(*) AS count FROM function_calls`).get().count;
+    const topCallers = db.prepare(`
+    SELECT caller_id, COUNT(*) AS num_calls 
+    FROM function_calls 
+    GROUP BY caller_id 
+    ORDER BY num_calls DESC 
+    LIMIT 5
+  `).all();
+    console.log(`🔁 Total function calls:    ${callCount}`);
+    console.log('📞 Top callers:');
+    topCallers.forEach(row => {
+        console.log(`   - Caller ${row.caller_id} made ${row.num_calls} calls`);
+    });
+}
+catch (err) {
+    console.error('❌ Error accessing function_calls table:', err.message);
+}
+// === Random Summary Samples ===
+console.log('\n🧾 10 Random Summaries (ID + Preview):');
+console.log('-------------------------------------------');
+const randomSummaries = db.prepare(`
+  SELECT id, filename, substr(summary, 1, 1000) || '...' AS preview
+  FROM files
+  WHERE summary IS NOT NULL AND summary != ''
+  ORDER BY RANDOM()
+  LIMIT 10
+`).all();
+randomSummaries.forEach(row => {
+    console.log(`📄 [${row.id}] ${row.filename}: ${row.preview}`);
+});
+// === Random Functions Samples ===
+console.log('\n🧑‍💻 20 Random Functions (ID, name, body):');
+console.log('-------------------------------------------');
+const randomFunctions = db.prepare(`
+  SELECT id, name, file_id, substr(content, 1, 100) || '...' AS preview
+  FROM functions
+  ORDER BY RANDOM()
+  LIMIT 20
+`).all();
+randomFunctions.forEach(row => {
+    console.log(`🔹 [${row.id}] ${row.name} (file_id: ${row.file_id})`);
+    console.log(`   ${row.preview}\n`);
+});
+// === Column View of 100 Files ===
+console.log('\n📊 Table View: First 100 Files');
+console.log('-------------------------------------------');
+const fileRows = db.prepare(`
+  SELECT id, filename, type, processing_status, functions_extracted_at, length(summary) AS summary_len
+  FROM files
+  LIMIT 1000
+`).all();
+console.table(fileRows);
+// === Column View of 100 Functions ===
+console.log('\n📊 Table View: First 100 Functions');
+console.log('-------------------------------------------');
+const functionRows = db.prepare(`
+  SELECT id, file_id, name, start_line, end_line, length(content) AS length
+  FROM functions
+  LIMIT 1000
+`).all();
+console.table(functionRows);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.57",
+  "version": "0.1.59",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"
@@ -44,15 +44,17 @@
     "commander": "^11.0.0",
     "fast-glob": "^3.3.3",
     "proper-lockfile": "^4.1.2",
-    "string-similarity-js": "^2.1.4"
+    "string-similarity-js": "^2.1.4",
+    "ts-morph": "^26.0.0"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",
     "@types/jest": "^30.0.0",
-    "@types/node": "^24.0.13",
+    "@types/node": "^24.1.0",
     "@types/proper-lockfile": "^4.1.4",
     "jest": "^30.0.2",
     "ts-jest": "^29.4.0",
+    "ts-node": "^10.9.2",
     "typescript": "^5.8.3"
   },
   "files": [