smart-coding-mcp 1.2.4 → 1.3.1

package/lib/config.js

@@ -1,5 +1,6 @@
 import fs from "fs/promises";
 import path from "path";
+import { fileURLToPath } from "url";
 import { ProjectDetector } from "./project-detector.js";
 
 const DEFAULT_CONFIG = {
@@ -48,10 +49,11 @@ const DEFAULT_CONFIG = {
     "**/coverage/**",
     "**/.next/**",
     "**/target/**",
-    "**/vendor/**"
+    "**/vendor/**",
+    "**/.smart-coding-cache/**"
   ],
-  chunkSize: 15,
-  chunkOverlap: 3,
+  chunkSize: 25, // Lines per chunk (larger = fewer embeddings = faster indexing)
+  chunkOverlap: 5, // Overlap between chunks for context continuity
   batchSize: 100,
   maxFileSize: 1048576, // 1MB - skip files larger than this
   maxResults: 5,
@@ -59,6 +61,7 @@ const DEFAULT_CONFIG = {
   cacheDirectory: "./.smart-coding-cache",
   watchFiles: false,
   verbose: false,
+  workerThreads: "auto", // "auto" = CPU cores - 1, or set a number
   embeddingModel: "Xenova/all-MiniLM-L6-v2",
   semanticWeight: 0.7,
   exactMatchBoost: 1.5,
@@ -80,7 +83,7 @@ export async function loadConfig(workspaceDir = null) {
       console.error(`[Config] Workspace mode: ${baseDir}`);
     } else {
       // Server mode: load config from server directory
-      const scriptDir = path.dirname(new URL(import.meta.url).pathname);
+      const scriptDir = path.dirname(fileURLToPath(import.meta.url));
       baseDir = path.resolve(scriptDir, '..');
       configPath = path.join(baseDir, "config.json");
     }
@@ -212,6 +215,28 @@ export async function loadConfig(workspaceDir = null) {
     }
   }
   
+  if (process.env.SMART_CODING_EMBEDDING_MODEL !== undefined) {
+    const value = process.env.SMART_CODING_EMBEDDING_MODEL.trim();
+    if (value.length > 0) {
+      config.embeddingModel = value;
+      console.error(`[Config] Using custom embedding model: ${value}`);
+    }
+  }
+  
+  if (process.env.SMART_CODING_WORKER_THREADS !== undefined) {
+    const value = process.env.SMART_CODING_WORKER_THREADS.trim().toLowerCase();
+    if (value === 'auto') {
+      config.workerThreads = 'auto';
+    } else {
+      const numValue = parseInt(value, 10);
+      if (!isNaN(numValue) && numValue >= 1 && numValue <= 32) {
+        config.workerThreads = numValue;
+      } else {
+        console.error(`[Config] Invalid SMART_CODING_WORKER_THREADS: ${value}, using default (must be 'auto' or 1-32)`);
+      }
+    }
+  }
+  
   return config;
 }
 

package/lib/embedding-worker.js

@@ -0,0 +1,67 @@
+import { parentPort, workerData } from "worker_threads";
+import { pipeline } from "@xenova/transformers";
+
+let embedder = null;
+
+// Initialize the embedding model once when worker starts
+async function initializeEmbedder() {
+  if (!embedder) {
+    embedder = await pipeline("feature-extraction", workerData.embeddingModel);
+  }
+  return embedder;
+}
+
+/**
+ * Process chunks with optimized single-text embedding
+ * Note: Batch processing with transformers.js WASM backend doesn't improve speed
+ * because it loops internally. Single calls are actually faster.
+ */
+async function processChunks(chunks) {
+  const embedder = await initializeEmbedder();
+  const results = [];
+
+  for (const chunk of chunks) {
+    try {
+      const output = await embedder(chunk.text, { pooling: "mean", normalize: true });
+      results.push({
+        file: chunk.file,
+        startLine: chunk.startLine,
+        endLine: chunk.endLine,
+        content: chunk.text,
+        vector: Array.from(output.data),
+        success: true
+      });
+    } catch (error) {
+      results.push({
+        file: chunk.file,
+        startLine: chunk.startLine,
+        endLine: chunk.endLine,
+        error: error.message,
+        success: false
+      });
+    }
+  }
+
+  return results;
+}
+
+// Listen for messages from main thread
+parentPort.on("message", async (message) => {
+  if (message.type === "process") {
+    try {
+      const results = await processChunks(message.chunks);
+      parentPort.postMessage({ type: "results", results, batchId: message.batchId });
+    } catch (error) {
+      parentPort.postMessage({ type: "error", error: error.message, batchId: message.batchId });
+    }
+  } else if (message.type === "shutdown") {
+    process.exit(0);
+  }
+});
+
+// Signal that worker is ready
+initializeEmbedder().then(() => {
+  parentPort.postMessage({ type: "ready" });
+}).catch((error) => {
+  parentPort.postMessage({ type: "error", error: error.message });
+});

package/lib/tokenizer.js

@@ -0,0 +1,142 @@
+/**
+ * Token estimation and limits for embedding models
+ * 
+ * This module provides token counting utilities and model-specific limits
+ * to ensure text chunks don't exceed the model's maximum sequence length.
+ */
+
+/**
+ * Token limits for supported embedding models
+ * Each model has its own maximum sequence length
+ */
+export const MODEL_TOKEN_LIMITS = {
+  // Sentence Transformers / MiniLM family
+  "Xenova/all-MiniLM-L6-v2": 256,
+  "Xenova/all-MiniLM-L12-v2": 256,
+  "Xenova/paraphrase-MiniLM-L6-v2": 128,
+  "Xenova/paraphrase-MiniLM-L3-v2": 128,
+  
+  // MPNet models
+  "Xenova/all-mpnet-base-v2": 384,
+  "Xenova/paraphrase-mpnet-base-v2": 384,
+  
+  // Multilingual models
+  "Xenova/paraphrase-multilingual-MiniLM-L12-v2": 128,
+  "Xenova/paraphrase-multilingual-mpnet-base-v2": 256,
+  
+  // Code-specific models
+  "Xenova/codebert-base": 512,
+  "Xenova/graphcodebert-base": 512,
+  
+  // E5 models
+  "Xenova/e5-small-v2": 512,
+  "Xenova/e5-base-v2": 512,
+  "Xenova/e5-large-v2": 512,
+  
+  // BGE models
+  "Xenova/bge-small-en-v1.5": 512,
+  "Xenova/bge-base-en-v1.5": 512,
+  "Xenova/bge-large-en-v1.5": 512,
+  
+  // Default fallback
+  "default": 256
+};
+
+/**
+ * Get the maximum token limit for a given model
+ * Case-insensitive lookup for robustness
+ * @param {string} modelName - The model name (e.g., "Xenova/all-MiniLM-L6-v2")
+ * @returns {number} Maximum tokens supported by the model
+ */
+export function getModelTokenLimit(modelName) {
+  if (!modelName) return MODEL_TOKEN_LIMITS["default"];
+  
+  // Direct match first (fastest)
+  if (MODEL_TOKEN_LIMITS[modelName] !== undefined) {
+    return MODEL_TOKEN_LIMITS[modelName];
+  }
+  
+  // Case-insensitive search
+  const normalizedName = modelName.toLowerCase();
+  for (const [key, value] of Object.entries(MODEL_TOKEN_LIMITS)) {
+    if (key.toLowerCase() === normalizedName) {
+      return value;
+    }
+  }
+  
+  return MODEL_TOKEN_LIMITS["default"];
+}
+
+/**
+ * Get chunking parameters for a model
+ * Returns target and overlap tokens based on the model's limit
+ * @param {string} modelName - The model name
+ * @returns {{ maxTokens: number, targetTokens: number, overlapTokens: number }}
+ */
+export function getChunkingParams(modelName) {
+  const maxTokens = getModelTokenLimit(modelName);
+  
+  // Target: 85% of max to leave safety buffer
+  const targetTokens = Math.floor(maxTokens * 0.85);
+  
+  // Overlap: 15-20% of target for context continuity
+  const overlapTokens = Math.floor(targetTokens * 0.18);
+  
+  return {
+    maxTokens,
+    targetTokens,
+    overlapTokens
+  };
+}
+
+/**
+ * Estimate token count for text (conservative estimate for code)
+ * Uses a simple heuristic: counts words, special characters, and estimates subwords
+ * 
+ * This is conservative - actual tokenizers may produce fewer tokens.
+ * For most accurate results, use the actual tokenizer, but this is much faster.
+ * 
+ * @param {string} text - The text to estimate tokens for
+ * @returns {number} Estimated token count
+ */
+export function estimateTokens(text) {
+  if (!text || text.length === 0) return 0;
+  
+  // Count words (split by whitespace)
+  const words = text.split(/\s+/).filter(w => w.length > 0);
+  
+  // Count special characters/punctuation that often become separate tokens
+  const specialChars = (text.match(/[{}()\[\];:,.<>!=+\-*\/%&|^~@#$"'`\\]/g) || []).length;
+  
+  // Estimate: words + special chars + 2 (for [CLS] and [SEP] special tokens)
+  // For long words, add extra tokens due to subword tokenization
+  let tokenCount = 2; // [CLS] and [SEP]
+  
+  for (const word of words) {
+    if (word.length <= 4) {
+      tokenCount += 1;
+    } else if (word.length <= 10) {
+      tokenCount += 2;
+    } else {
+      // Long words get split into ~4-char subwords
+      tokenCount += Math.ceil(word.length / 4);
+    }
+  }
+  
+  // Many special chars merge with adjacent tokens, so count ~50%
+  tokenCount += Math.floor(specialChars * 0.5);
+  
+  return tokenCount;
+}
+
+/**
+ * Check if text exceeds the token limit for a model
+ * @param {string} text - The text to check
+ * @param {string} modelName - The model name
+ * @returns {boolean} True if the text exceeds the limit
+ */
+export function exceedsTokenLimit(text, modelName) {
+  const limit = getModelTokenLimit(modelName);
+  const tokens = estimateTokens(text);
+  return tokens > limit;
+}

package/lib/utils.js

@@ -1,5 +1,9 @@
 import crypto from "crypto";
 import path from "path";
+import { estimateTokens, getChunkingParams, getModelTokenLimit } from "./tokenizer.js";
+
+// Re-export tokenizer utilities
+export { estimateTokens, getChunkingParams, getModelTokenLimit, MODEL_TOKEN_LIMITS } from "./tokenizer.js";
 
 /**
  * Calculate cosine similarity between two vectors
@@ -22,13 +26,22 @@ export function hashContent(content) {
 }
 
 /**
- * Intelligent chunking: tries to split by function/class boundaries
+ * Intelligent chunking with token limit awareness
+ * Tries to split by function/class boundaries while respecting token limits
+ * 
+ * @param {string} content - File content to chunk
+ * @param {string} file - File path (for language detection)
+ * @param {object} config - Configuration object with embeddingModel
+ * @returns {Array<{text: string, startLine: number, endLine: number, tokenCount: number}>}
  */
 export function smartChunk(content, file, config) {
   const lines = content.split("\n");
   const chunks = [];
   const ext = path.extname(file);
   
+  // Get model-specific chunking parameters
+  const { targetTokens, overlapTokens } = getChunkingParams(config.embeddingModel);
+  
   // Language-specific patterns for function/class detection
   const patterns = {
     // JavaScript/TypeScript
@@ -42,6 +55,7 @@ export function smartChunk(content, file, config) {
     // Python
     py: /^(class|def|async\s+def)\s+\w+/,
     pyw: /^(class|def|async\s+def)\s+\w+/,
+    pyx: /^(cdef|cpdef|def|class)\s+\w+/, // Cython
     
     // Java/Kotlin/Scala
     java: /^(public|private|protected)?\s*(static\s+)?(class|interface|enum|void|int|String|boolean)\s+\w+/,
@@ -56,70 +70,144 @@ export function smartChunk(content, file, config) {
     cxx: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
     h: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
     hpp: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
+    hxx: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
     
     // C#
     cs: /^(public|private|protected)?\s*(static\s+)?(class|interface|struct|enum|void|int|string|bool)\s+\w+/,
+    csx: /^(public|private|protected)?\s*(static\s+)?(class|interface|struct|enum|void|int|string|bool)\s+\w+/,
     
     // Go
     go: /^(func|type|const|var)\s+\w+/,
     
     // Rust
-    rs: /^(pub\s+)?(fn|struct|enum|trait|impl|const|static)\s+\w+/,
+    rs: /^(pub\s+)?(fn|struct|enum|trait|impl|const|static|mod)\s+\w+/,
     
     // PHP
     php: /^(class|interface|trait|function|const)\s+\w+/,
+    phtml: /^(<\?php|class|interface|trait|function)\s*/,
     
     // Ruby
     rb: /^(class|module|def)\s+\w+/,
-    rake: /^(class|module|def|task)\s+\w+/,
+    rake: /^(class|module|def|task|namespace)\s+\w+/,
     
     // Swift
-    swift: /^(class|struct|enum|protocol|func|var|let)\s+\w+/,
+    swift: /^(class|struct|enum|protocol|func|var|let|extension)\s+\w+/,
     
     // R
-    r: /^(\w+)\s*<-\s*function/,
-    R: /^(\w+)\s*<-\s*function/,
+    r: /^(\w+)\s*(<-|=)\s*function/,
+    R: /^(\w+)\s*(<-|=)\s*function/,
     
     // Lua
     lua: /^(function|local\s+function)\s+\w+/,
+    
+    // Shell scripts
+    sh: /^(\w+\s*\(\)|function\s+\w+)/,
+    bash: /^(\w+\s*\(\)|function\s+\w+)/,
+    zsh: /^(\w+\s*\(\)|function\s+\w+)/,
+    fish: /^function\s+\w+/,
+    
+    // CSS/Styles
+    css: /^(\.|#|@media|@keyframes|@font-face|\w+)\s*[{,]/,
+    scss: /^(\$\w+:|@mixin|@function|@include|\.|#|@media)\s*/,
+    sass: /^(\$\w+:|=\w+|\+\w+|\.|#|@media)\s*/,
+    less: /^(@\w+:|\.|\#|@media)\s*/,
+    styl: /^(\$\w+\s*=|\w+\(|\.|\#)\s*/,
+    
+    // Markup/HTML
+    html: /^(<(div|section|article|header|footer|nav|main|aside|form|table|template|script|style)\b)/i,
+    htm: /^(<(div|section|article|header|footer|nav|main|aside|form|table|template|script|style)\b)/i,
+    xml: /^(<\w+|\s*<!\[CDATA\[)/,
+    svg: /^(<svg|<g|<path|<defs|<symbol)\b/,
+    
+    // Config files
+    json: /^(\s*"[\w-]+"\s*:\s*[\[{])/,
+    yaml: /^(\w[\w-]*:\s*[|>]?$|\w[\w-]*:\s*$)/,
+    yml: /^(\w[\w-]*:\s*[|>]?$|\w[\w-]*:\s*$)/,
+    toml: /^(\[\[?\w+\]?\]?|\w+\s*=)/,
+    ini: /^(\[\w+\]|\w+\s*=)/,
+    env: /^[A-Z_][A-Z0-9_]*=/,
+    
+    // Documentation
+    md: /^(#{1,6}\s+|```|\*{3}|_{3})/,
+    mdx: /^(#{1,6}\s+|```|import\s+|export\s+)/,
+    txt: /^.{50,}/, // Split on long paragraphs
+    rst: /^(={3,}|-{3,}|~{3,}|\.\.\s+\w+::)/,
+    
+    // Database
+    sql: /^(CREATE|ALTER|INSERT|UPDATE|DELETE|SELECT|DROP|GRANT|REVOKE|WITH|DECLARE|BEGIN|END)\s+/i,
+    
+    // Perl
+    pl: /^(sub|package|use|require)\s+\w+/,
+    pm: /^(sub|package|use|require)\s+\w+/,
+    
+    // Vim
+    vim: /^(function|command|autocmd|let\s+g:)\s*/,
   };
 
-
   const langPattern = patterns[ext.slice(1)] || patterns.js;
   let currentChunk = [];
   let chunkStartLine = 0;
+  let currentTokenCount = 0;
 
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
-    currentChunk.push(line);
-
-    // Check if we should start a new chunk
-    const shouldSplit = 
+    const lineTokens = estimateTokens(line);
+    
+    // Check if adding this line would exceed token limit
+    const wouldExceedLimit = (currentTokenCount + lineTokens) > targetTokens;
+    
+    // Check if this is a good split point (function/class boundary)
+    const isGoodSplitPoint = 
       langPattern.test(line.trim()) && 
-      currentChunk.length > config.chunkSize * 0.5;
+      currentChunk.length > 3; // At least a few lines before splitting
+    
+    // Split if we exceed limit OR at a good split point when near limit
+    const shouldSplit = wouldExceedLimit || (isGoodSplitPoint && currentTokenCount > targetTokens * 0.6);
 
-    if (shouldSplit || currentChunk.length >= config.chunkSize + config.chunkOverlap) {
-      if (currentChunk.join("\n").trim().length > 20) {
+    if (shouldSplit && currentChunk.length > 0) {
+      const chunkText = currentChunk.join("\n");
+      if (chunkText.trim().length > 20) {
         chunks.push({
-          text: currentChunk.join("\n"),
+          text: chunkText,
           startLine: chunkStartLine + 1,
-          endLine: i + 1
+          endLine: i,
+          tokenCount: currentTokenCount
         });
       }
       
-      // Keep overlap
-      currentChunk = currentChunk.slice(-config.chunkOverlap);
-      chunkStartLine = i - config.chunkOverlap + 1;
+      // Calculate overlap: keep last N lines that fit within overlapTokens
+      let overlapLines = [];
+      let overlapTokensCount = 0;
+      for (let j = currentChunk.length - 1; j >= 0 && overlapTokensCount < overlapTokens; j--) {
+        const lineT = estimateTokens(currentChunk[j]);
+        if (overlapTokensCount + lineT <= overlapTokens) {
+          overlapLines.unshift(currentChunk[j]);
+          overlapTokensCount += lineT;
+        } else {
+          break;
+        }
+      }
+      
+      currentChunk = overlapLines;
+      currentTokenCount = overlapTokensCount;
+      chunkStartLine = i - overlapLines.length;
     }
+
+    currentChunk.push(line);
+    currentTokenCount += lineTokens;
   }
 
   // Add remaining chunk
-  if (currentChunk.length > 0 && currentChunk.join("\n").trim().length > 20) {
-    chunks.push({
-      text: currentChunk.join("\n"),
-      startLine: chunkStartLine + 1,
-      endLine: lines.length
-    });
+  if (currentChunk.length > 0) {
+    const chunkText = currentChunk.join("\n");
+    if (chunkText.trim().length > 20) {
+      chunks.push({
+        text: chunkText,
+        startLine: chunkStartLine + 1,
+        endLine: lines.length,
+        tokenCount: currentTokenCount
+      });
+    }
   }
 
   return chunks;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smart-coding-mcp",
-  "version": "1.2.4",
+  "version": "1.3.1",
   "description": "An extensible MCP server that enhances coding productivity with AI-powered features including semantic code search, intelligent indexing, and more, using local LLMs",
   "type": "module",
   "main": "index.js",
@@ -10,6 +10,8 @@
   "scripts": {
     "start": "node index.js",
     "dev": "node --watch index.js",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "clear-cache": "node scripts/clear-cache.js"
   },
   "keywords": [
@@ -45,10 +47,14 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.0.4",
     "@xenova/transformers": "^2.17.2",
-    "glob": "^10.3.10",
-    "chokidar": "^3.5.3"
+    "chokidar": "^3.5.3",
+    "fdir": "^6.5.0",
+    "glob": "^10.3.10"
   },
   "engines": {
     "node": ">=18.0.0"
+  },
+  "devDependencies": {
+    "vitest": "^4.0.16"
   }
 }