semantic-code-mcp 2.0.0

package/lib/project-detector.js

@@ -0,0 +1,75 @@
+import fs from "fs/promises";
+import path from "path";
+import { FILE_TYPE_MAP, IGNORE_PATTERNS } from "./ignore-patterns.js";
+
+export class ProjectDetector {
+  constructor(searchDirectory) {
+    this.searchDirectory = searchDirectory;
+    this.detectedTypes = new Set();
+  }
+
+  async detectProjectTypes() {
+    const markerFiles = Object.keys(FILE_TYPE_MAP);
+    
+    for (const marker of markerFiles) {
+      // Handle wildcard patterns like *.csproj
+      if (marker.includes('*')) {
+        await this.detectWithWildcard(marker);
+      } else {
+        await this.detectExactFile(marker);
+      }
+    }
+
+    return Array.from(this.detectedTypes);
+  }
+
+  async detectExactFile(markerFile) {
+    const markerPath = path.join(this.searchDirectory, markerFile);
+    try {
+      await fs.access(markerPath);
+      const projectType = FILE_TYPE_MAP[markerFile];
+      this.detectedTypes.add(projectType);
+      console.error(`[Detector] Detected ${projectType} project (${markerFile})`);
+    } catch {
+      // File doesn't exist, continue
+    }
+  }
+
+  async detectWithWildcard(pattern) {
+    try {
+      const files = await fs.readdir(this.searchDirectory);
+      const regex = new RegExp('^' + pattern.replace('*', '.*') + '$');
+      
+      for (const file of files) {
+        if (regex.test(file)) {
+          const projectType = FILE_TYPE_MAP[pattern];
+          this.detectedTypes.add(projectType);
+          console.error(`[Detector] Detected ${projectType} project (${file})`);
+          break;
+        }
+      }
+    } catch {
+      // Directory read failed, continue
+    }
+  }
+
+  getSmartIgnorePatterns() {
+    const patterns = [...IGNORE_PATTERNS.common];
+    
+    for (const type of this.detectedTypes) {
+      if (IGNORE_PATTERNS[type]) {
+        patterns.push(...IGNORE_PATTERNS[type]);
+      }
+    }
+
+    // Remove duplicates
+    return [...new Set(patterns)];
+  }
+
+  getSummary() {
+    return {
+      detectedTypes: Array.from(this.detectedTypes),
+      patternCount: this.getSmartIgnorePatterns().length
+    };
+  }
+}

package/lib/resource-throttle.js

@@ -0,0 +1,85 @@
+import os from 'os';
+
+/**
+ * Resource throttling utility to prevent CPU/memory exhaustion during indexing
+ * Ensures the MCP server doesn't freeze the user's laptop
+ */
+export class ResourceThrottle {
+  constructor(config) {
+    // Max CPU usage as percentage (default 50%)
+    this.maxCpuPercent = config.maxCpuPercent || 50;
+
+    // Delay between batches in milliseconds (reduced from 100ms for better performance)
+    this.batchDelay = config.batchDelay ?? 10;
+
+    // Max worker threads (override auto-detection)
+    const cpuCount = os.cpus().length;
+    if (config.maxWorkers === 'auto' || config.maxWorkers === undefined) {
+      // Use 50% of cores by default (balanced performance/responsiveness)
+      this.maxWorkers = Math.max(1, Math.floor(cpuCount * 0.5));
+    } else {
+      // Validate and parse the value
+      const parsed = typeof config.maxWorkers === 'number'
+        ? config.maxWorkers
+        : parseInt(config.maxWorkers, 10);
+
+      if (isNaN(parsed) || parsed < 1) {
+        console.error(`[Throttle] Invalid maxWorkers: ${config.maxWorkers}, using auto`);
+        this.maxWorkers = Math.max(1, Math.floor(cpuCount * 0.5));
+      } else {
+        this.maxWorkers = Math.max(1, parsed);
+      }
+    }
+
+    console.error(`[Throttle] CPU limit: ${this.maxCpuPercent}%, Batch delay: ${this.batchDelay}ms, Max workers: ${this.maxWorkers}`);
+  }
+
+  /**
+   * Execute work with delay to throttle CPU usage
+   */
+  async throttledBatch(work, signal = null) {
+    // Execute the work
+    if (work) {
+      await work();
+    }
+
+    // Apply delay if not aborted
+    if (!signal?.aborted && this.batchDelay > 0) {
+      await this.sleep(this.batchDelay);
+    }
+  }
+
+  /**
+   * Sleep utility
+   */
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Calculate optimal worker count based on CPU limit
+   */
+  getWorkerCount(requestedWorkers) {
+    if (requestedWorkers === 'auto') {
+      return this.maxWorkers;
+    }
+    return Math.min(requestedWorkers, this.maxWorkers);
+  }
+
+  /**
+   * Check if we should pause due to high CPU usage
+   * This is a simple implementation - could be enhanced with actual CPU monitoring
+   */
+  async checkCpuUsage() {
+    // Future enhancement: monitor actual CPU usage and pause if needed
+    // For now, we rely on worker limits and batch delays
+    return true;
+  }
+}
+
+/**
+ * Sleep utility function
+ */
+export function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

package/lib/sqlite-cache.js

@@ -0,0 +1,468 @@
+import Database from 'better-sqlite3';
+import fs from 'fs/promises';
+import path from 'path';
+import { cosineSimilarity } from './utils.js';
+
+/**
+ * SQLite-based embeddings cache for fast, efficient storage
+ * Replaces JSON-based cache for better performance on large codebases
+ */
+export class SQLiteCache {
+  constructor(config) {
+    this.config = config;
+    this.db = null;
+    this.isSaving = false;
+    this.dbPath = path.join(config.cacheDirectory, 'embeddings.db');
+    
+    // Track indexing status for progressive indexing
+    this.indexingStatus = {
+      inProgress: false,
+      totalFiles: 0,
+      processedFiles: 0,
+      percentage: 0
+    };
+  }
+
+  /**
+   * Initialize SQLite database and create schema
+   */
+  async load() {
+    if (!this.config.enableCache) return;
+    
+    try {
+      // Ensure cache directory exists
+      await fs.mkdir(this.config.cacheDirectory, { recursive: true });
+      
+      // Check if we need to migrate from JSON
+      const jsonCacheExists = await this.checkJSONCache();
+      
+      // Open SQLite database
+      this.db = new Database(this.dbPath);
+      
+      // Enable performance optimizations
+      this.db.pragma('journal_mode = WAL'); // Write-Ahead Logging for better concurrency
+      this.db.pragma('synchronous = NORMAL'); // Faster writes, still safe
+      this.db.pragma('cache_size = 10000'); // 10MB cache
+      this.db.pragma('temp_store = MEMORY'); // Temp tables in memory
+      
+      // Create schema if not exists
+      this.createSchema();
+      
+      // Migrate from JSON if needed
+      if (jsonCacheExists && this.getVectorCount() === 0) {
+        console.error('[Cache] Migrating from JSON to SQLite...');
+        await this.migrateFromJSON();
+      }
+      
+      const count = this.getVectorCount();
+      const fileCount = this.getFileCount();
+      console.error(`[Cache] Loaded SQLite cache: ${count} embeddings from ${fileCount} files`);
+    } catch (error) {
+      console.error('[Cache] Failed to initialize SQLite cache:', error.message);
+      throw error;
+    }
+  }
+
+  /**
+   * Create database schema
+   */
+  createSchema() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS embeddings (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        file TEXT NOT NULL,
+        start_line INTEGER NOT NULL,
+        end_line INTEGER NOT NULL,
+        content TEXT NOT NULL,
+        vector BLOB NOT NULL,
+        indexed_at INTEGER NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS file_hashes (
+        file TEXT PRIMARY KEY,
+        hash TEXT NOT NULL,
+        mtime REAL,
+        indexed_at INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_file ON embeddings(file);
+      CREATE INDEX IF NOT EXISTS idx_indexed_at ON embeddings(indexed_at);
+    `);
+
+    // Migration: Add mtime column if it doesn't exist (for existing databases)
+    try {
+      this.db.exec('ALTER TABLE file_hashes ADD COLUMN mtime REAL');
+    } catch (e) {
+      // Column already exists, ignore
+    }
+  }
+
+  /**
+   * Check if JSON cache exists
+   */
+  async checkJSONCache() {
+    try {
+      const jsonPath = path.join(this.config.cacheDirectory, 'embeddings.json');
+      await fs.access(jsonPath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Migrate from JSON cache to SQLite
+   */
+  async migrateFromJSON() {
+    try {
+      const jsonCachePath = path.join(this.config.cacheDirectory, 'embeddings.json');
+      const jsonHashPath = path.join(this.config.cacheDirectory, 'file-hashes.json');
+      
+      const [cacheData, hashData] = await Promise.all([
+        fs.readFile(jsonCachePath, 'utf-8').catch(() => null),
+        fs.readFile(jsonHashPath, 'utf-8').catch(() => null)
+      ]);
+
+      if (!cacheData || !hashData) {
+        console.error('[Cache] No JSON cache found to migrate');
+        return;
+      }
+
+      const vectorStore = JSON.parse(cacheData);
+      const fileHashes = new Map(Object.entries(JSON.parse(hashData)));
+
+      console.error(`[Cache] Migrating ${vectorStore.length} embeddings...`);
+
+      // Use transaction for fast batch insert
+      const insertVector = this.db.prepare(`
+        INSERT INTO embeddings (file, start_line, end_line, content, vector, indexed_at)
+        VALUES (?, ?, ?, ?, ?, ?)
+      `);
+
+      const insertHash = this.db.prepare(`
+        INSERT OR REPLACE INTO file_hashes (file, hash, indexed_at)
+        VALUES (?, ?, ?)
+      `);
+
+      const transaction = this.db.transaction(() => {
+        const now = Date.now();
+        
+        for (const chunk of vectorStore) {
+          const vectorBuffer = this.vectorToBuffer(chunk.vector);
+          insertVector.run(
+            chunk.file,
+            chunk.startLine,
+            chunk.endLine,
+            chunk.content,
+            vectorBuffer,
+            now
+          );
+        }
+
+        for (const [file, hash] of fileHashes) {
+          insertHash.run(file, hash, now);
+        }
+      });
+
+      transaction();
+
+      console.error('[Cache] Migration complete! Backing up JSON files...');
+      
+      // Backup old JSON files
+      await fs.rename(jsonCachePath, jsonCachePath + '.backup');
+      await fs.rename(jsonHashPath, jsonHashPath + '.backup');
+      
+      console.error('[Cache] JSON cache backed up (you can delete .backup files if everything works)');
+    } catch (error) {
+      console.error('[Cache] Migration failed:', error.message);
+      throw error;
+    }
+  }
+
+  /**
+   * Convert Float32Array/Array to Buffer for SQLite storage
+   */
+  vectorToBuffer(vector) {
+    const float32 = new Float32Array(vector);
+    return Buffer.from(float32.buffer);
+  }
+
+  /**
+   * Convert Buffer back to Array for compatibility
+   */
+  bufferToVector(buffer) {
+    const float32 = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / 4);
+    return Array.from(float32);
+  }
+
+  /**
+   * Get all vectors from store (lazy loaded)
+   */
+  getVectorStore() {
+    if (!this.db) return [];
+    
+    const stmt = this.db.prepare(`
+      SELECT file, start_line, end_line, content, vector
+      FROM embeddings
+      ORDER BY file, start_line
+    `);
+
+    const rows = stmt.all();
+    return rows.map(row => ({
+      file: row.file,
+      startLine: row.start_line,
+      endLine: row.end_line,
+      content: row.content,
+      vector: this.bufferToVector(row.vector)
+    }));
+  }
+
+  /**
+   * Vector search compatibility API used by ANN-capable search path.
+   * SQLite fallback performs in-process cosine scoring.
+   */
+  searchByVector(queryVector, topK = 10) {
+    const normalizedTopK = Number.isInteger(topK) && topK > 0 ? topK : 10;
+    const vectorStore = this.getVectorStore();
+
+    return vectorStore
+      .map((chunk) => ({
+        ...chunk,
+        score: cosineSimilarity(queryVector, chunk.vector)
+      }))
+      .sort((a, b) => b.score - a.score)
+      .slice(0, normalizedTopK);
+  }
+
+  /**
+   * Get vector count
+   */
+  getVectorCount() {
+    if (!this.db) return 0;
+    const result = this.db.prepare('SELECT COUNT(*) as count FROM embeddings').get();
+    return result.count;
+  }
+
+  /**
+   * Get unique file count
+   */
+  getFileCount() {
+    if (!this.db) return 0;
+    const result = this.db.prepare('SELECT COUNT(DISTINCT file) as count FROM embeddings').get();
+    return result.count;
+  }
+
+  getStats() {
+    return {
+      totalChunks: this.getVectorCount(),
+      totalFiles: this.getFileCount()
+    };
+  }
+
+  /**
+   * Add chunk to store with batch optimization
+   */
+  addToStore(chunk) {
+    if (!this.db) return;
+    
+    const vectorBuffer = this.vectorToBuffer(chunk.vector);
+    const stmt = this.db.prepare(`
+      INSERT INTO embeddings (file, start_line, end_line, content, vector, indexed_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `);
+
+    stmt.run(
+      chunk.file,
+      chunk.startLine,
+      chunk.endLine,
+      chunk.content,
+      vectorBuffer,
+      Date.now()
+    );
+  }
+
+  /**
+   * Add multiple chunks in a transaction (much faster)
+   */
+  addBatchToStore(chunks) {
+    if (!this.db || chunks.length === 0) return;
+    
+    const stmt = this.db.prepare(`
+      INSERT INTO embeddings (file, start_line, end_line, content, vector, indexed_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `);
+
+    const transaction = this.db.transaction(() => {
+      const now = Date.now();
+      for (const chunk of chunks) {
+        const vectorBuffer = this.vectorToBuffer(chunk.vector);
+        stmt.run(
+          chunk.file,
+          chunk.startLine,
+          chunk.endLine,
+          chunk.content,
+          vectorBuffer,
+          now
+        );
+      }
+    });
+
+    transaction();
+  }
+
+  /**
+   * Remove all chunks for a specific file
+   */
+  removeFileFromStore(file) {
+    if (!this.db) return;
+    
+    const stmt = this.db.prepare('DELETE FROM embeddings WHERE file = ?');
+    stmt.run(file);
+  }
+
+  /**
+   * Get file hash
+   */
+  getFileHash(file) {
+    if (!this.db) return null;
+
+    const stmt = this.db.prepare('SELECT hash FROM file_hashes WHERE file = ?');
+    const row = stmt.get(file);
+    return row ? row.hash : null;
+  }
+
+  /**
+   * Get file mtime (modification time) for fast change detection
+   */
+  getFileMtime(file) {
+    if (!this.db) return null;
+
+    const stmt = this.db.prepare('SELECT mtime FROM file_hashes WHERE file = ?');
+    const row = stmt.get(file);
+    return row ? row.mtime : null;
+  }
+
+  /**
+   * Set file hash with optional mtime
+   */
+  setFileHash(file, hash, mtime = null) {
+    if (!this.db) return;
+
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO file_hashes (file, hash, mtime, indexed_at)
+      VALUES (?, ?, ?, ?)
+    `);
+    stmt.run(file, hash, mtime, Date.now());
+  }
+
+  /**
+   * Delete file hash
+   */
+  deleteFileHash(file) {
+    if (!this.db) return;
+    
+    const stmt = this.db.prepare('DELETE FROM file_hashes WHERE file = ?');
+    stmt.run(file);
+  }
+
+  /**
+   * Get all file hashes as Map
+   */
+  getAllFileHashes() {
+    if (!this.db) return new Map();
+    
+    const stmt = this.db.prepare('SELECT file, hash FROM file_hashes');
+    const rows = stmt.all();
+    return new Map(rows.map(row => [row.file, row.hash]));
+  }
+
+  /**
+   * Save (checkpoint WAL for durability)
+   * With SQLite, writes are already persisted, this just checkpoints the WAL
+   */
+  async save() {
+    if (!this.config.enableCache || !this.db) return;
+    
+    this.isSaving = true;
+    
+    try {
+      // Checkpoint WAL to ensure durability
+      this.db.pragma('wal_checkpoint(PASSIVE)');
+    } catch (error) {
+      console.error('[Cache] Failed to checkpoint WAL:', error.message);
+    } finally {
+      this.isSaving = false;
+    }
+  }
+
+  /**
+   * Incremental save during indexing (no-op for SQLite, already persisted)
+   */
+  async saveIncremental() {
+    // SQLite writes are already persisted due to WAL mode
+    // This is a no-op but kept for API compatibility
+    return;
+  }
+
+  /**
+   * Clear all cache data
+   */
+  async clear() {
+    if (!this.config.enableCache) return;
+    
+    try {
+      if (this.db) {
+        this.db.close();
+        this.db = null;
+      }
+      
+      await fs.rm(this.config.cacheDirectory, { recursive: true, force: true });
+      console.error(`[Cache] Cache cleared successfully: ${this.config.cacheDirectory}`);
+    } catch (error) {
+      console.error('[Cache] Failed to clear cache:', error.message);
+      throw error;
+    }
+  }
+
+  /**
+   * Close database connection
+   */
+  close() {
+    if (this.db) {
+      this.db.close();
+      this.db = null;
+    }
+  }
+
+  /**
+   * Clear all file hashes from the database
+   */
+  clearAllFileHashes() {
+    if (!this.db) return;
+    this.db.exec('DELETE FROM file_hashes');
+  }
+
+  /**
+   * Reset vectors + file hashes for full reindex.
+   */
+  async resetForFullReindex() {
+    this.setVectorStore([]);
+    this.clearAllFileHashes();
+  }
+
+  /**
+   * Set vector store (for compatibility with test code)
+   * This is less efficient than batch operations but kept for compatibility
+   */
+  setVectorStore(store) {
+    if (!this.db) return;
+    
+    // Clear existing data
+    this.db.exec('DELETE FROM embeddings');
+    
+    // Insert new data
+    if (store.length > 0) {
+      this.addBatchToStore(store);
+    }
+  }
+}