smart-coding-mcp 1.4.1 → 2.0.0

package/README.md

@@ -5,16 +5,18 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 [![Node.js](https://img.shields.io/badge/Node.js-%3E%3D18-green.svg)](https://nodejs.org/)
 
-An extensible Model Context Protocol (MCP) server that provides intelligent semantic code search for AI assistants. Built with local AI models (RAG), inspired by Cursor's semantic search research.
+An extensible Model Context Protocol (MCP) server that provides intelligent semantic code search for AI assistants. Built with local AI models using Matryoshka Representation Learning (MRL) for flexible embedding dimensions (64-768d), with runtime workspace switching and comprehensive status reporting.
 
 ### Available Tools
 
-| Tool                   | Description                                       | Example                                        |
-| ---------------------- | ------------------------------------------------- | ---------------------------------------------- |
-| `semantic_search`      | Find code by meaning, not just keywords           | `"Where do we validate user input?"`           |
-| `index_codebase`       | Manually trigger reindexing                       | Use after major refactoring or branch switches |
-| `clear_cache`          | Reset the embeddings cache                        | Useful when cache becomes corrupted            |
-| `d_check_last_version` | Get latest version of any package (20 ecosystems) | `"express"`, `"npm:react"`, `"pip:requests"`   |
+| Tool                   | Description                                       | Example                                         |
+| ---------------------- | ------------------------------------------------- | ----------------------------------------------- |
+| `semantic_search`      | Find code by meaning, not just keywords           | `"Where do we validate user input?"`            |
+| `index_codebase`       | Manually trigger reindexing                       | Use after major refactoring or branch switches  |
+| `clear_cache`          | Reset the embeddings cache                        | Useful when cache becomes corrupted             |
+| `d_check_last_version` | Get latest version of any package (20 ecosystems) | `"express"`, `"npm:react"`, `"pip:requests"`    |
+| `e_set_workspace`      | Change project path at runtime                    | Switch to different project without restart     |
+| `f_get_status`         | Get server info: version, index status, config    | Check indexing progress, model info, cache size |
 
 ## What This Does
 
@@ -130,19 +132,22 @@ For clients that support dynamic variables (VS Code, Cursor):
 
 Override configuration settings via environment variables in your MCP config:
 
-| Variable                         | Type    | Default                   | Description                           |
-| -------------------------------- | ------- | ------------------------- | ------------------------------------- |
-| `SMART_CODING_VERBOSE`           | boolean | `false`                   | Enable detailed logging               |
-| `SMART_CODING_BATCH_SIZE`        | number  | `100`                     | Files to process in parallel          |
-| `SMART_CODING_MAX_FILE_SIZE`     | number  | `1048576`                 | Max file size in bytes (1MB)          |
-| `SMART_CODING_CHUNK_SIZE`        | number  | `25`                      | Lines of code per chunk               |
-| `SMART_CODING_MAX_RESULTS`       | number  | `5`                       | Max search results                    |
-| `SMART_CODING_SMART_INDEXING`    | boolean | `true`                    | Enable smart project detection        |
-| `SMART_CODING_WATCH_FILES`       | boolean | `false`                   | Enable file watching for auto-reindex |
-| `SMART_CODING_SEMANTIC_WEIGHT`   | number  | `0.7`                     | Weight for semantic similarity (0-1)  |
-| `SMART_CODING_EXACT_MATCH_BOOST` | number  | `1.5`                     | Boost for exact text matches          |
-| `SMART_CODING_EMBEDDING_MODEL`   | string  | `Xenova/all-MiniLM-L6-v2` | AI embedding model to use             |
-| `SMART_CODING_WORKER_THREADS`    | string  | `auto`                    | Worker threads (`auto` or 1-32)       |
+| Variable                           | Type    | Default                          | Description                                |
+| ---------------------------------- | ------- | -------------------------------- | ------------------------------------------ |
+| `SMART_CODING_VERBOSE`             | boolean | `false`                          | Enable detailed logging                    |
+| `SMART_CODING_BATCH_SIZE`          | number  | `100`                            | Files to process in parallel               |
+| `SMART_CODING_MAX_FILE_SIZE`       | number  | `1048576`                        | Max file size in bytes (1MB)               |
+| `SMART_CODING_CHUNK_SIZE`          | number  | `25`                             | Lines of code per chunk                    |
+| `SMART_CODING_MAX_RESULTS`         | number  | `5`                              | Max search results                         |
+| `SMART_CODING_SMART_INDEXING`      | boolean | `true`                           | Enable smart project detection             |
+| `SMART_CODING_WATCH_FILES`         | boolean | `false`                          | Enable file watching for auto-reindex      |
+| `SMART_CODING_SEMANTIC_WEIGHT`     | number  | `0.7`                            | Weight for semantic similarity (0-1)       |
+| `SMART_CODING_EXACT_MATCH_BOOST`   | number  | `1.5`                            | Boost for exact text matches               |
+| `SMART_CODING_EMBEDDING_MODEL`     | string  | `nomic-ai/nomic-embed-text-v1.5` | AI embedding model to use                  |
+| `SMART_CODING_EMBEDDING_DIMENSION` | number  | `256`                            | MRL dimension (64, 128, 256, 512, 768)     |
+| `SMART_CODING_DEVICE`              | string  | `cpu`                            | Inference device (`cpu`, `webgpu`, `auto`) |
+| `SMART_CODING_CHUNKING_MODE`       | string  | `smart`                          | Code chunking (`smart`, `ast`, `line`)     |
+| `SMART_CODING_WORKER_THREADS`      | string  | `auto`                           | Worker threads (`auto` or 1-32)            |
 
 **Example with environment variables:**
 
@@ -166,16 +171,72 @@ Override configuration settings via environment variables in your MCP config:
 
 ## How It Works
 
-The server indexes your code in four steps:
+```mermaid
+flowchart TB
+    subgraph IDE["IDE / AI Assistant"]
+        Agent["AI Agent<br/>(Claude, GPT, Gemini)"]
+    end
+
+    subgraph MCP["Smart Coding MCP Server"]
+        direction TB
+        Protocol["Model Context Protocol<br/>JSON-RPC over stdio"]
+        Tools["MCP Tools<br/>semantic_search | index_codebase | set_workspace | get_status"]
+
+        subgraph Indexing["Indexing Pipeline"]
+            Discovery["File Discovery<br/>glob patterns + smart ignore"]
+            Chunking["Code Chunking<br/>Smart (regex) / AST (Tree-sitter)"]
+            Embedding["AI Embedding<br/>transformers.js + ONNX Runtime"]
+        end
+
+        subgraph AI["AI Model"]
+            Model["nomic-embed-text-v1.5<br/>Matryoshka Representation Learning"]
+            Dimensions["Flexible Dimensions<br/>64 | 128 | 256 | 512 | 768"]
+            Normalize["Layer Norm → Slice → L2 Normalize"]
+        end
+
+        subgraph Search["Search"]
+            QueryEmbed["Query → Vector"]
+            Cosine["Cosine Similarity"]
+            Hybrid["Hybrid Search<br/>Semantic + Exact Match Boost"]
+        end
+    end
+
+    subgraph Storage["Cache"]
+        Vectors["Vector Store<br/>embeddings.json"]
+        Hashes["File Hashes<br/>Incremental updates"]
+    end
+
+    Agent <-->|"MCP Protocol"| Protocol
+    Protocol --> Tools
+
+    Tools --> Discovery
+    Discovery --> Chunking
+    Chunking --> Embedding
+    Embedding --> Model
+    Model --> Dimensions
+    Dimensions --> Normalize
+    Normalize --> Vectors
+
+    Tools --> QueryEmbed
+    QueryEmbed --> Model
+    Cosine --> Hybrid
+    Vectors --> Cosine
+    Hybrid --> Agent
+```
+
+### Tech Stack
 
-1. **Discovery**: Scans your project for source files
-2. **Chunking**: Breaks code into meaningful pieces (respecting function boundaries)
-3. **Embedding**: Converts each chunk to a vector using a local AI model
-4. **Storage**: Saves embeddings to `.smart-coding-cache/` for fast startup
+| Component     | Technology                            |
+| ------------- | ------------------------------------- |
+| **Protocol**  | Model Context Protocol (JSON-RPC)     |
+| **AI Model**  | nomic-embed-text-v1.5 (MRL)           |
+| **Inference** | transformers.js + ONNX Runtime        |
+| **Chunking**  | Smart regex / Tree-sitter AST         |
+| **Search**    | Cosine similarity + exact match boost |
 
-When you search, your query is converted to the same vector format and compared against all code chunks using cosine similarity. The most relevant matches are returned.
+### Search Flow
 
-![How It Works](how-its-works.png)
+Query → Vector embedding → Cosine similarity → Ranked results
 
 ## Examples
 
@@ -214,11 +275,18 @@ Finds all try/catch blocks and error handling patterns.
 
 ## Technical Details
 
-**Embedding Model**: all-MiniLM-L6-v2 via transformers.js
+**Embedding Model**: nomic-embed-text-v1.5 via transformers.js v3
+
+- Matryoshka Representation Learning (MRL) for flexible dimensions
+- Configurable output: 64, 128, 256, 512, or 768 dimensions
+- Longer context (8192 tokens vs 256 for MiniLM)
+- Better code understanding through specialized training
+- WebGPU support for up to 100x faster inference (when available)
+
+**Legacy Model**: all-MiniLM-L6-v2 (fallback)
 
-- Fast inference (CPU-friendly)
-- Small model size (~100MB)
-- Good accuracy for code search
+- Fast inference, small footprint (~100MB)
+- Fixed 384-dimensional output
 
 **Vector Similarity**: Cosine similarity
 

package/config.json

@@ -60,7 +60,10 @@
   "cacheDirectory": "./.smart-coding-cache",
   "watchFiles": false,
   "verbose": false,
-  "embeddingModel": "Xenova/all-MiniLM-L6-v2",
+  "embeddingModel": "nomic-ai/nomic-embed-text-v1.5",
+  "embeddingDimension": 256,
+  "device": "auto",
+  "chunkingMode": "smart",
   "semanticWeight": 0.7,
   "exactMatchBoost": 1.5,
   "workerThreads": "auto"

package/features/get-status.js

@@ -0,0 +1,132 @@
+/**
+ * Get Status Feature
+ * 
+ * MCP tool to return comprehensive status information about the server.
+ * Useful for agents to understand current state and configuration.
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { createRequire } from 'module';
+
+const require = createRequire(import.meta.url);
+const packageJson = require('../package.json');
+
+/**
+ * Get tool definition for MCP registration
+ */
+export function getToolDefinition(config) {
+  return {
+    name: "f_get_status",
+    description: "Get comprehensive status information about the Smart Coding MCP server. Returns version, workspace path, model configuration, indexing status, and cache information. Useful for understanding the current state of the semantic search system.",
+    inputSchema: {
+      type: "object",
+      properties: {},
+      required: []
+    }
+  };
+}
+
+/**
+ * Status Reporter class
+ */
+export class StatusReporter {
+  constructor(config, cache, indexer, embedder) {
+    this.config = config;
+    this.cache = cache;
+    this.indexer = indexer;
+    this.embedder = embedder;
+    this.startTime = Date.now();
+  }
+
+  /**
+   * Get comprehensive status
+   */
+  async getStatus() {
+    const vectorStore = this.cache?.getVectorStore() || [];
+    
+    // Get unique files from vector store
+    const uniqueFiles = new Set(vectorStore.map(v => v.file));
+    
+    // Get cache size
+    let cacheSizeBytes = 0;
+    try {
+      const cachePath = path.join(this.config.cacheDirectory, 'embeddings.json');
+      const stats = await fs.stat(cachePath);
+      cacheSizeBytes = stats.size;
+    } catch {
+      // Cache file doesn't exist yet
+    }
+
+    // Determine index status
+    let indexStatus = 'empty';
+    if (this.indexer?.isIndexing) {
+      indexStatus = 'indexing';
+    } else if (vectorStore.length > 0) {
+      indexStatus = 'ready';
+    }
+
+    return {
+      version: packageJson.version,
+      uptime: Math.floor((Date.now() - this.startTime) / 1000),
+      
+      workspace: {
+        path: this.config.searchDirectory,
+        cacheDirectory: this.config.cacheDirectory
+      },
+
+      model: {
+        name: this.embedder?.modelName || this.config.embeddingModel,
+        dimension: this.embedder?.dimension || this.config.embeddingDimension,
+        device: this.embedder?.device || this.config.device
+      },
+
+      index: {
+        status: indexStatus,
+        filesIndexed: uniqueFiles.size,
+        chunksCount: vectorStore.length,
+        chunkingMode: this.config.chunkingMode
+      },
+
+      cache: {
+        enabled: this.config.enableCache,
+        path: this.config.cacheDirectory,
+        sizeBytes: cacheSizeBytes,
+        sizeFormatted: formatBytes(cacheSizeBytes)
+      },
+
+      config: {
+        maxResults: this.config.maxResults,
+        chunkSize: this.config.chunkSize,
+        semanticWeight: this.config.semanticWeight,
+        exactMatchBoost: this.config.exactMatchBoost,
+        workerThreads: this.config.workerThreads
+      }
+    };
+  }
+}
+
+/**
+ * Format bytes to human readable
+ */
+function formatBytes(bytes) {
+  if (bytes === 0) return '0 B';
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB', 'GB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+}
+
+/**
+ * Handle MCP tool call
+ */
+export async function handleToolCall(request, instance) {
+  const status = await instance.getStatus();
+
+  return {
+    content: [{
+      type: "text",
+      text: JSON.stringify(status, null, 2)
+    }]
+  };
+}

package/features/index-codebase.js

@@ -25,6 +25,13 @@ export class CodebaseIndexer {
    * Initialize worker thread pool for parallel embedding
    */
   async initializeWorkers() {
+    // Force single-threaded mode for nomic models (transformers.js v3 worker thread issue)
+    const isNomicModel = this.config.embeddingModel?.includes('nomic');
+    if (isNomicModel) {
+      console.error("[Indexer] Single-threaded mode (nomic model - workers disabled for stability)");
+      return;
+    }
+
     const numWorkers = this.config.workerThreads === "auto" 
       ? Math.max(1, os.cpus().length - 1) 
       : (this.config.workerThreads || 1);
@@ -48,6 +55,7 @@ export class CodebaseIndexer {
         const worker = new Worker(workerPath, {
           workerData: { 
             embeddingModel: this.config.embeddingModel,
+            embeddingDimension: this.config.embeddingDimension,
             verbose: this.config.verbose
           }
         });

package/features/set-workspace.js

@@ -0,0 +1,155 @@
+/**
+ * Set Workspace Feature
+ * 
+ * MCP tool to change the project workspace path at runtime.
+ * Useful when agent detects it's in a different directory.
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+
+/**
+ * Get tool definition for MCP registration
+ */
+export function getToolDefinition(config) {
+  return {
+    name: "e_set_workspace",
+    description: "Change the project workspace path at runtime. Use this when you detect the current workspace is incorrect or you need to switch to a different project directory. Creates cache folder automatically and optionally re-indexes the new workspace.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        path: {
+          type: "string",
+          description: "Absolute path to the new workspace directory"
+        },
+        clearCache: {
+          type: "boolean",
+          description: "Whether to clear existing cache before switching (default: false)"
+        },
+        reindex: {
+          type: "boolean", 
+          description: "Whether to trigger re-indexing after switching (default: true)"
+        }
+      },
+      required: ["path"]
+    }
+  };
+}
+
+/**
+ * Workspace Manager class
+ */
+export class WorkspaceManager {
+  constructor(config, cache, indexer) {
+    this.config = config;
+    this.cache = cache;
+    this.indexer = indexer;
+  }
+
+  /**
+   * Set new workspace path
+   */
+  async setWorkspace(newPath, options = {}) {
+    const { clearCache = false, reindex = true } = options;
+
+    // Validate path
+    try {
+      const stats = await fs.stat(newPath);
+      if (!stats.isDirectory()) {
+        return {
+          success: false,
+          error: `Path is not a directory: ${newPath}`
+        };
+      }
+    } catch (err) {
+      return {
+        success: false,
+        error: `Path does not exist: ${newPath}`
+      };
+    }
+
+    const oldPath = this.config.searchDirectory;
+    
+    // Update config
+    this.config.searchDirectory = newPath;
+    
+    // Update cache directory
+    const newCacheDir = path.join(newPath, '.smart-coding-cache');
+    this.config.cacheDirectory = newCacheDir;
+
+    // Ensure cache directory exists
+    try {
+      await fs.mkdir(newCacheDir, { recursive: true });
+    } catch (err) {
+      // Ignore if already exists
+    }
+
+    // Clear cache if requested
+    if (clearCache && this.cache) {
+      this.cache.setVectorStore([]);
+      this.cache.fileHashes = new Map();
+      console.error(`[Workspace] Cache cleared for new workspace`);
+    }
+
+    // Update cache path and reload
+    if (this.cache) {
+      this.cache.config = this.config;
+      await this.cache.load();
+    }
+
+    // Update indexer config
+    if (this.indexer) {
+      this.indexer.config = this.config;
+    }
+
+    console.error(`[Workspace] Changed from ${oldPath} to ${newPath}`);
+
+    // Trigger re-indexing if requested
+    let indexResult = null;
+    if (reindex && this.indexer) {
+      console.error(`[Workspace] Starting re-indexing...`);
+      try {
+        indexResult = await this.indexer.indexAll(clearCache);
+      } catch (err) {
+        console.error(`[Workspace] Re-indexing error: ${err.message}`);
+      }
+    }
+
+    return {
+      success: true,
+      oldPath,
+      newPath,
+      cacheDirectory: newCacheDir,
+      reindexed: reindex,
+      indexResult
+    };
+  }
+}
+
+/**
+ * Handle MCP tool call
+ */
+export async function handleToolCall(request, instance) {
+  const { path: newPath, clearCache, reindex } = request.params.arguments || {};
+
+  if (!newPath) {
+    return {
+      content: [{
+        type: "text",
+        text: JSON.stringify({
+          success: false,
+          error: "Missing required parameter: path"
+        }, null, 2)
+      }]
+    };
+  }
+
+  const result = await instance.setWorkspace(newPath, { clearCache, reindex });
+
+  return {
+    content: [{
+      type: "text",
+      text: JSON.stringify(result, null, 2)
+    }]
+  };
+}

package/index.js

@@ -2,7 +2,6 @@
 import { Server } from "@modelcontextprotocol/sdk/server/index.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
-import { pipeline } from "@xenova/transformers";
 import fs from "fs/promises";
 import { createRequire } from "module";
 
@@ -12,6 +11,7 @@ const packageJson = require("./package.json");
 
 import { loadConfig } from "./lib/config.js";
 import { EmbeddingsCache } from "./lib/cache.js";
+import { createEmbedder } from "./lib/mrl-embedder.js";
 import { CodebaseIndexer } from "./features/index-codebase.js";
 import { HybridSearch } from "./features/hybrid-search.js";
 
@@ -19,6 +19,8 @@ import * as IndexCodebaseFeature from "./features/index-codebase.js";
 import * as HybridSearchFeature from "./features/hybrid-search.js";
 import * as ClearCacheFeature from "./features/clear-cache.js";
 import * as CheckLastVersionFeature from "./features/check-last-version.js";
+import * as SetWorkspaceFeature from "./features/set-workspace.js";
+import * as GetStatusFeature from "./features/get-status.js";
 
 // Parse workspace from command line arguments
 const args = process.argv.slice(2);
@@ -78,6 +80,16 @@ const features = [
     module: CheckLastVersionFeature,
     instance: null,
     handler: CheckLastVersionFeature.handleToolCall
+  },
+  {
+    module: SetWorkspaceFeature,
+    instance: null,
+    handler: SetWorkspaceFeature.handleToolCall
+  },
+  {
+    module: GetStatusFeature,
+    instance: null,
+    handler: GetStatusFeature.handleToolCall
   }
 ];
 
@@ -94,9 +106,10 @@ async function initialize() {
     process.exit(1);
   }
 
-  // Load AI model
+  // Load AI model using MRL embedder factory
   console.error("[Server] Loading AI embedding model (this may take time on first run)...");
-  embedder = await pipeline("feature-extraction", config.embeddingModel);
+  embedder = await createEmbedder(config);
+  console.error(`[Server] Model: ${embedder.modelName} (${embedder.dimension}d, device: ${embedder.device})`);
 
   // Initialize cache
   cache = new EmbeddingsCache(config);
@@ -113,6 +126,12 @@ async function initialize() {
   features[1].instance = indexer;
   features[2].instance = cacheClearer;
   features[3].instance = versionChecker;
+  
+  // Initialize new tools
+  const workspaceManager = new SetWorkspaceFeature.WorkspaceManager(config, cache, indexer);
+  const statusReporter = new GetStatusFeature.StatusReporter(config, cache, indexer, embedder);
+  features[4].instance = workspaceManager;
+  features[5].instance = statusReporter;
 
   // Start indexing in background (non-blocking)
   console.error("[Server] Starting background indexing...");