semantic-code-mcp 2.0.0

package/lib/tokenizer.js

@@ -0,0 +1,149 @@
+/**
+ * Token estimation and limits for embedding models
+ * 
+ * This module provides token counting utilities and model-specific limits
+ * to ensure text chunks don't exceed the model's maximum sequence length.
+ */
+
+/**
+ * Token limits for supported embedding models
+ * Each model has its own maximum sequence length
+ */
+export const MODEL_TOKEN_LIMITS = {
+  // MRL / Nomic models (longer context)
+  "nomic-ai/nomic-embed-text-v1.5": 8192,
+  "nomic-ai/nomic-embed-text-v1": 2048,
+  
+  // Gemini embedding models
+  "gemini-embedding-001": 2048,
+  
+  // Sentence Transformers / MiniLM family
+  "Xenova/all-MiniLM-L6-v2": 256,
+  "Xenova/all-MiniLM-L12-v2": 256,
+  "Xenova/paraphrase-MiniLM-L6-v2": 128,
+  "Xenova/paraphrase-MiniLM-L3-v2": 128,
+  
+  // MPNet models
+  "Xenova/all-mpnet-base-v2": 384,
+  "Xenova/paraphrase-mpnet-base-v2": 384,
+  
+  // Multilingual models
+  "Xenova/paraphrase-multilingual-MiniLM-L12-v2": 128,
+  "Xenova/paraphrase-multilingual-mpnet-base-v2": 256,
+  
+  // Code-specific models
+  "Xenova/codebert-base": 512,
+  "Xenova/graphcodebert-base": 512,
+  
+  // E5 models
+  "Xenova/e5-small-v2": 512,
+  "Xenova/e5-base-v2": 512,
+  "Xenova/e5-large-v2": 512,
+  
+  // BGE models
+  "Xenova/bge-small-en-v1.5": 512,
+  "Xenova/bge-base-en-v1.5": 512,
+  "Xenova/bge-large-en-v1.5": 512,
+  
+  // Default fallback
+  "default": 256
+};
+
+/**
+ * Get the maximum token limit for a given model
+ * Case-insensitive lookup for robustness
+ * @param {string} modelName - The model name (e.g., "Xenova/all-MiniLM-L6-v2")
+ * @returns {number} Maximum tokens supported by the model
+ */
+export function getModelTokenLimit(modelName) {
+  if (!modelName) return MODEL_TOKEN_LIMITS["default"];
+  
+  // Direct match first (fastest)
+  if (MODEL_TOKEN_LIMITS[modelName] !== undefined) {
+    return MODEL_TOKEN_LIMITS[modelName];
+  }
+  
+  // Case-insensitive search
+  const normalizedName = modelName.toLowerCase();
+  for (const [key, value] of Object.entries(MODEL_TOKEN_LIMITS)) {
+    if (key.toLowerCase() === normalizedName) {
+      return value;
+    }
+  }
+  
+  return MODEL_TOKEN_LIMITS["default"];
+}
+
+/**
+ * Get chunking parameters for a model
+ * Returns target and overlap tokens based on the model's limit
+ * @param {string} modelName - The model name
+ * @returns {{ maxTokens: number, targetTokens: number, overlapTokens: number }}
+ */
+export function getChunkingParams(modelName) {
+  const maxTokens = getModelTokenLimit(modelName);
+  
+  // Target: 85% of max to leave safety buffer
+  const targetTokens = Math.floor(maxTokens * 0.85);
+  
+  // Overlap: 15-20% of target for context continuity
+  const overlapTokens = Math.floor(targetTokens * 0.18);
+  
+  return {
+    maxTokens,
+    targetTokens,
+    overlapTokens
+  };
+}
+
+/**
+ * Estimate token count for text (conservative estimate for code)
+ * Uses a simple heuristic: counts words, special characters, and estimates subwords
+ * 
+ * This is conservative - actual tokenizers may produce fewer tokens.
+ * For most accurate results, use the actual tokenizer, but this is much faster.
+ * 
+ * @param {string} text - The text to estimate tokens for
+ * @returns {number} Estimated token count
+ */
+export function estimateTokens(text) {
+  if (!text || text.length === 0) return 0;
+  
+  // Count words (split by whitespace)
+  const words = text.split(/\s+/).filter(w => w.length > 0);
+  
+  // Count special characters/punctuation that often become separate tokens
+  const specialChars = (text.match(/[{}()\[\];:,.<>!=+\-*\/%&|^~@#$"'`\\]/g) || []).length;
+  
+  // Estimate: words + special chars + 2 (for [CLS] and [SEP] special tokens)
+  // For long words, add extra tokens due to subword tokenization
+  let tokenCount = 2; // [CLS] and [SEP]
+  
+  for (const word of words) {
+    if (word.length <= 4) {
+      tokenCount += 1;
+    } else if (word.length <= 10) {
+      tokenCount += 2;
+    } else {
+      // Long words get split into ~4-char subwords
+      tokenCount += Math.ceil(word.length / 4);
+    }
+  }
+  
+  // Many special chars merge with adjacent tokens, so count ~50%
+  tokenCount += Math.floor(specialChars * 0.5);
+  
+  return tokenCount;
+}
+
+/**
+ * Check if text exceeds the token limit for a model
+ * @param {string} text - The text to check
+ * @param {string} modelName - The model name
+ * @returns {boolean} True if the text exceeds the limit
+ */
+export function exceedsTokenLimit(text, modelName) {
+  const limit = getModelTokenLimit(modelName);
+  const tokens = estimateTokens(text);
+  return tokens > limit;
+}

package/lib/utils.js

@@ -0,0 +1,214 @@
+import crypto from "crypto";
+import path from "path";
+import { estimateTokens, getChunkingParams, getModelTokenLimit } from "./tokenizer.js";
+
+// Re-export tokenizer utilities
+export { estimateTokens, getChunkingParams, getModelTokenLimit, MODEL_TOKEN_LIMITS } from "./tokenizer.js";
+
+/**
+ * Calculate cosine similarity between two vectors
+ */
+export function cosineSimilarity(a, b) {
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+/**
+ * Generate hash for file content to detect changes
+ */
+export function hashContent(content) {
+  return crypto.createHash("md5").update(content).digest("hex");
+}
+
+/**
+ * Intelligent chunking with token limit awareness
+ * Tries to split by function/class boundaries while respecting token limits
+ * 
+ * @param {string} content - File content to chunk
+ * @param {string} file - File path (for language detection)
+ * @param {object} config - Configuration object with embeddingModel
+ * @returns {Array<{text: string, startLine: number, endLine: number, tokenCount: number}>}
+ */
+export function smartChunk(content, file, config) {
+  const lines = content.split("\n");
+  const chunks = [];
+  const ext = path.extname(file);
+  
+  // Get model-specific chunking parameters
+  const { targetTokens, overlapTokens } = getChunkingParams(config.embeddingModel);
+  
+  // Language-specific patterns for function/class detection
+  const patterns = {
+    // JavaScript/TypeScript
+    js: /^(export\s+)?(async\s+)?(function|class|const|let|var)\s+\w+/,
+    jsx: /^(export\s+)?(async\s+)?(function|class|const|let|var)\s+\w+/,
+    ts: /^(export\s+)?(async\s+)?(function|class|const|let|var|interface|type)\s+\w+/,
+    tsx: /^(export\s+)?(async\s+)?(function|class|const|let|var|interface|type)\s+\w+/,
+    mjs: /^(export\s+)?(async\s+)?(function|class|const|let|var)\s+\w+/,
+    cjs: /^(export\s+)?(async\s+)?(function|class|const|let|var)\s+\w+/,
+    
+    // Python
+    py: /^(class|def|async\s+def)\s+\w+/,
+    pyw: /^(class|def|async\s+def)\s+\w+/,
+    pyx: /^(cdef|cpdef|def|class)\s+\w+/, // Cython
+    
+    // Java/Kotlin/Scala
+    java: /^(public|private|protected)?\s*(static\s+)?(class|interface|enum|void|int|String|boolean)\s+\w+/,
+    kt: /^(class|interface|object|fun|val|var)\s+\w+/,
+    kts: /^(class|interface|object|fun|val|var)\s+\w+/,
+    scala: /^(class|object|trait|def|val|var)\s+\w+/,
+    
+    // C/C++
+    c: /^(struct|enum|union|void|int|char|float|double)\s+\w+/,
+    cpp: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
+    cc: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
+    cxx: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
+    h: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
+    hpp: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
+    hxx: /^(class|struct|namespace|template|void|int|bool)\s+\w+/,
+    
+    // C#
+    cs: /^(public|private|protected)?\s*(static\s+)?(class|interface|struct|enum|void|int|string|bool)\s+\w+/,
+    csx: /^(public|private|protected)?\s*(static\s+)?(class|interface|struct|enum|void|int|string|bool)\s+\w+/,
+    
+    // Go
+    go: /^(func|type|const|var)\s+\w+/,
+    
+    // Rust
+    rs: /^(pub\s+)?(fn|struct|enum|trait|impl|const|static|mod)\s+\w+/,
+    
+    // PHP
+    php: /^(class|interface|trait|function|const)\s+\w+/,
+    phtml: /^(<\?php|class|interface|trait|function)\s*/,
+    
+    // Ruby
+    rb: /^(class|module|def)\s+\w+/,
+    rake: /^(class|module|def|task|namespace)\s+\w+/,
+    
+    // Swift
+    swift: /^(class|struct|enum|protocol|func|var|let|extension)\s+\w+/,
+    
+    // R
+    r: /^(\w+)\s*(<-|=)\s*function/,
+    R: /^(\w+)\s*(<-|=)\s*function/,
+    
+    // Lua
+    lua: /^(function|local\s+function)\s+\w+/,
+    
+    // Shell scripts
+    sh: /^(\w+\s*\(\)|function\s+\w+)/,
+    bash: /^(\w+\s*\(\)|function\s+\w+)/,
+    zsh: /^(\w+\s*\(\)|function\s+\w+)/,
+    fish: /^function\s+\w+/,
+    
+    // CSS/Styles
+    css: /^(\.|#|@media|@keyframes|@font-face|\w+)\s*[{,]/,
+    scss: /^(\$\w+:|@mixin|@function|@include|\.|#|@media)\s*/,
+    sass: /^(\$\w+:|=\w+|\+\w+|\.|#|@media)\s*/,
+    less: /^(@\w+:|\.|\#|@media)\s*/,
+    styl: /^(\$\w+\s*=|\w+\(|\.|\#)\s*/,
+    
+    // Markup/HTML
+    html: /^(<(div|section|article|header|footer|nav|main|aside|form|table|template|script|style)\b)/i,
+    htm: /^(<(div|section|article|header|footer|nav|main|aside|form|table|template|script|style)\b)/i,
+    xml: /^(<\w+|\s*<!\[CDATA\[)/,
+    svg: /^(<svg|<g|<path|<defs|<symbol)\b/,
+    
+    // Config files
+    json: /^(\s*"[\w-]+"\s*:\s*[\[{])/,
+    yaml: /^(\w[\w-]*:\s*[|>]?$|\w[\w-]*:\s*$)/,
+    yml: /^(\w[\w-]*:\s*[|>]?$|\w[\w-]*:\s*$)/,
+    toml: /^(\[\[?\w+\]?\]?|\w+\s*=)/,
+    ini: /^(\[\w+\]|\w+\s*=)/,
+    env: /^[A-Z_][A-Z0-9_]*=/,
+    
+    // Documentation
+    md: /^(#{1,6}\s+|```|\*{3}|_{3})/,
+    mdx: /^(#{1,6}\s+|```|import\s+|export\s+)/,
+    txt: /^.{50,}/, // Split on long paragraphs
+    rst: /^(={3,}|-{3,}|~{3,}|\.\.\s+\w+::)/,
+    
+    // Database
+    sql: /^(CREATE|ALTER|INSERT|UPDATE|DELETE|SELECT|DROP|GRANT|REVOKE|WITH|DECLARE|BEGIN|END)\s+/i,
+    
+    // Perl
+    pl: /^(sub|package|use|require)\s+\w+/,
+    pm: /^(sub|package|use|require)\s+\w+/,
+    
+    // Vim
+    vim: /^(function|command|autocmd|let\s+g:)\s*/,
+  };
+
+  const langPattern = patterns[ext.slice(1)] || patterns.js;
+  let currentChunk = [];
+  let chunkStartLine = 0;
+  let currentTokenCount = 0;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const lineTokens = estimateTokens(line);
+    
+    // Check if adding this line would exceed token limit
+    const wouldExceedLimit = (currentTokenCount + lineTokens) > targetTokens;
+    
+    // Check if this is a good split point (function/class boundary)
+    const isGoodSplitPoint = 
+      langPattern.test(line.trim()) && 
+      currentChunk.length > 3; // At least a few lines before splitting
+    
+    // Split if we exceed limit OR at a good split point when near limit
+    const shouldSplit = wouldExceedLimit || (isGoodSplitPoint && currentTokenCount > targetTokens * 0.6);
+
+    if (shouldSplit && currentChunk.length > 0) {
+      const chunkText = currentChunk.join("\n");
+      if (chunkText.trim().length > 20) {
+        chunks.push({
+          text: chunkText,
+          startLine: chunkStartLine + 1,
+          endLine: i,
+          tokenCount: currentTokenCount
+        });
+      }
+      
+      // Calculate overlap: keep last N lines that fit within overlapTokens
+      let overlapLines = [];
+      let overlapTokensCount = 0;
+      for (let j = currentChunk.length - 1; j >= 0 && overlapTokensCount < overlapTokens; j--) {
+        const lineT = estimateTokens(currentChunk[j]);
+        if (overlapTokensCount + lineT <= overlapTokens) {
+          overlapLines.unshift(currentChunk[j]);
+          overlapTokensCount += lineT;
+        } else {
+          break;
+        }
+      }
+      
+      currentChunk = overlapLines;
+      currentTokenCount = overlapTokensCount;
+      chunkStartLine = i - overlapLines.length;
+    }
+
+    currentChunk.push(line);
+    currentTokenCount += lineTokens;
+  }
+
+  // Add remaining chunk
+  if (currentChunk.length > 0) {
+    const chunkText = currentChunk.join("\n");
+    if (chunkText.trim().length > 20) {
+      chunks.push({
+        text: chunkText,
+        startLine: chunkStartLine + 1,
+        endLine: lines.length,
+        tokenCount: currentTokenCount
+      });
+    }
+  }
+
+  return chunks;
+}

package/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "semantic-code-mcp",
+  "version": "2.0.0",
+  "description": "AI-powered semantic code search for coding agents. MCP server with multi-provider embeddings and hybrid search.",
+  "type": "module",
+  "main": "index.js",
+  "bin": {
+    "semantic-code-mcp": "index.js"
+  },
+  "scripts": {
+    "start": "node index.js",
+    "dev": "node --watch index.js",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "clear-cache": "node scripts/clear-cache.js"
+  },
+  "keywords": [
+    "mcp",
+    "semantic-search",
+    "code-search",
+    "embeddings",
+    "ai",
+    "model-context-protocol",
+    "hybrid-search",
+    "code-intelligence",
+    "milvus",
+    "gemini",
+    "openai",
+    "tree-sitter",
+    "ast",
+    "vector-search"
+  ],
+  "author": {
+    "name": "bitkyc08",
+    "url": "https://github.com/bitkyc08-arch"
+  },
+  "contributors": [
+    {
+      "name": "Omar Haris",
+      "url": "https://www.linkedin.com/in/omarharis/"
+    }
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bitkyc08-arch/smart-coding-mcp.git"
+  },
+  "bugs": {
+    "url": "https://github.com/bitkyc08-arch/smart-coding-mcp/issues"
+  },
+  "homepage": "https://github.com/bitkyc08-arch/smart-coding-mcp#readme",
+  "license": "MIT",
+  "dependencies": {
+    "@huggingface/transformers": "^3.8.1",
+    "@modelcontextprotocol/sdk": "^1.0.4",
+    "@zilliz/milvus2-sdk-node": "^2.6.10",
+    "better-sqlite3": "^12.5.0",
+    "chokidar": "^3.5.3",
+    "fastembed": "^2.1.0",
+    "fdir": "^6.5.0",
+    "glob": "^10.3.10",
+    "google-auth-library": "^9.0.0",
+    "web-tree-sitter": "^0.24.6"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "devDependencies": {
+    "vitest": "^4.0.16"
+  }
+}

package/reindex.js

@@ -0,0 +1,109 @@
+#!/usr/bin/env node
+/**
+ * smart-coding-mcp 리인덱싱 스크립트 (shell 직접 실행용)
+ * MCP 서버 없이 직접 인덱싱, 로그 출력
+ *
+ * 사용법:
+ *   node reindex.js /path/to/workspace [--force]
+ *
+ * 환경 변수는 MCP config와 동일하게 설정 필요.
+ */
+import { loadConfig } from "./lib/config.js";
+import { createCache } from "./lib/cache-factory.js";
+import { createEmbedder } from "./lib/mrl-embedder.js";
+import { CodebaseIndexer } from "./features/index-codebase.js";
+import { parseArgs } from "util";
+
+const { values, positionals } = parseArgs({
+    allowPositionals: true,
+    options: {
+        force: { type: "boolean", short: "f", default: false },
+        help: { type: "boolean", short: "h", default: false },
+    },
+});
+
+if (values.help) {
+    console.log(`
+smart-coding-mcp 리인덱싱 (shell 직접 실행)
+
+Usage:
+  node reindex.js [workspace_path] [--force]
+
+Options:
+  -f, --force   전체 재인덱싱 (캐시 무시)
+  -h, --help    도움말
+
+Environment:
+  MCP config의 env를 그대로 사용합니다.
+  SMART_CODING_EMBEDDING_PROVIDER, SMART_CODING_GEMINI_BATCH_SIZE 등
+  `);
+    process.exit(0);
+}
+
+const workspaceDir = positionals[0] || process.cwd();
+const force = values.force;
+
+function log(msg) {
+    const ts = new Date().toLocaleTimeString("ko-KR", { hour12: false });
+    console.log(`[${ts}] ${msg}`);
+}
+
+async function main() {
+    log(`🚀 Reindex 시작: ${workspaceDir}`);
+    log(`   force=${force}`);
+
+    // 1. 설정 로드
+    const config = await loadConfig(workspaceDir);
+    log(`   searchDirectory: ${config.searchDirectory}`);
+    log(`   cacheDirectory: ${config.cacheDirectory}`);
+    log(`   extensions: ${config.fileExtensions?.length || "?"} types`);
+    log(`   excludePatterns: ${config.excludePatterns?.length || "?"} patterns`);
+    console.log();
+
+    // 2. 임베더 로드
+    log("🧠 임베더 로딩...");
+    const embedder = await createEmbedder(config);
+    log(`   model: ${embedder.modelName} (${embedder.dimension}d, device: ${embedder.device})`);
+
+    // 3. 캐시 로드
+    log("💾 캐시 로딩...");
+    const cache = createCache(config);
+    await cache.load();
+
+    const statsBefore = cache.getStats?.() || {};
+    log(`   캐시 항목: ${statsBefore.totalEntries ?? "?"}`);
+
+    // 4. 인덱서 생성 & 실행
+    log("📁 인덱싱 시작...");
+    console.log();
+
+    const t0 = Date.now();
+    const indexer = new CodebaseIndexer(embedder, cache, config);
+    const result = await indexer.indexAll(force);
+
+    const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
+
+    console.log();
+    log("🎉 완료!");
+    log(`   파일: ${result.filesProcessed ?? "?"}개`);
+    log(`   청크: ${result.chunksProcessed ?? "?"}개`);
+    log(`   새로운 파일: ${result.newFiles ?? "?"}개`);
+    log(`   업데이트: ${result.updatedFiles ?? "?"}개`);
+    log(`   스킵: ${result.skippedFiles ?? "?"}개`);
+    log(`   삭제: ${result.deletedFiles ?? "?"}개`);
+    log(`   소요: ${elapsed}s`);
+
+    // 5. 캐시 저장
+    if (cache.save) {
+        await cache.save();
+        log("💾 캐시 저장 완료");
+    }
+
+    process.exit(0);
+}
+
+main().catch((err) => {
+    console.error(`\n❌ 에러: ${err.message}`);
+    console.error(err.stack);
+    process.exit(1);
+});