smart-coding-mcp 2.3.1 → 2.3.3

package/lib/mrl-embedder.js

@@ -20,21 +20,24 @@ const VALID_DIMENSIONS = [64, 128, 256, 512, 768];
 function clearModelCache(modelName) {
   try {
     // Find the transformers package location
-    const transformersPath = dirname(fileURLToPath(import.meta.resolve('@huggingface/transformers')));
-    const cacheDir = join(transformersPath, '.cache', modelName.replace('/', '-'));
+    // import.meta.resolve may return .../dist/index.js, so check parent too
+    const resolvedPath = dirname(fileURLToPath(import.meta.resolve('@huggingface/transformers')));
+    const possibleRoots = [resolvedPath, dirname(resolvedPath)];
 
-    if (existsSync(cacheDir)) {
-      console.error(`[MRL] Clearing corrupted cache: ${cacheDir}`);
-      rmSync(cacheDir, { recursive: true, force: true });
-      return true;
-    }
+    for (const root of possibleRoots) {
+      // Try different cache path patterns
+      const cachePaths = [
+        join(root, '.cache', modelName.replace('/', '-')),      // nomic-ai-nomic-embed-text-v1.5
+        join(root, '.cache', ...modelName.split('/'))           // nomic-ai/nomic-embed-text-v1.5
+      ];
 
-    // Also try the model name with original slash (nomic-ai/nomic-embed-text-v1.5)
-    const cacheDir2 = join(transformersPath, '.cache', ...modelName.split('/'));
-    if (existsSync(cacheDir2)) {
-      console.error(`[MRL] Clearing corrupted cache: ${cacheDir2}`);
-      rmSync(cacheDir2, { recursive: true, force: true });
-      return true;
+      for (const cacheDir of cachePaths) {
+        if (existsSync(cacheDir)) {
+          console.error(`[MRL] Clearing corrupted cache: ${cacheDir}`);
+          rmSync(cacheDir, { recursive: true, force: true });
+          return true;
+        }
+      }
     }
   } catch (e) {
     console.error(`[MRL] Failed to clear cache: ${e.message}`);
@@ -74,43 +77,87 @@ export async function createMRLEmbedder(modelName, options = {}) {
   }
 
   let extractor;
+
+  // Helper to detect corruption errors
+  function isCorruptionError(err) {
+    if (!err.message) return false;
+    return err.message.includes('Protobuf parsing failed') ||
+           err.message.includes('Invalid model') ||
+           err.message.includes('ONNX') && err.message.includes('corrupt');
+  }
+
+  // Helper to load/reload the extractor
+  async function loadExtractor(clearCache = false) {
+    if (clearCache) {
+      console.error(`[MRL] Corrupted model detected, attempting auto-recovery...`);
+      clearModelCache(modelName);
+    }
+    return await pipeline('feature-extraction', modelName, pipelineOptions);
+  }
+
   try {
-    extractor = await pipeline('feature-extraction', modelName, pipelineOptions);
+    extractor = await loadExtractor();
   } catch (err) {
-    // Detect corrupted ONNX model (Protobuf parsing failed)
-    if (err.message && err.message.includes('Protobuf parsing failed')) {
-      console.error(`[MRL] Corrupted model detected, attempting auto-recovery...`);
-      if (clearModelCache(modelName)) {
-        // Retry after clearing cache
-        extractor = await pipeline('feature-extraction', modelName, pipelineOptions);
-      } else {
-        throw err;
-      }
+    if (isCorruptionError(err)) {
+      extractor = await loadExtractor(true);
     } else {
       throw err;
     }
   }
-  
+
   console.error(`[MRL] Model loaded on ${finalDevice}`);
-  
+
+  // Fallback embedder for when MRL model fails at runtime
+  let fallbackEmbedder = null;
+
   /**
    * Embed text with MRL dimension slicing
    * Compatible with existing embedder(text, options) signature
+   * Includes runtime auto-recovery for corrupted models with fallback
    */
   async function embed(text, embedOptions = {}) {
-    // Generate full 768d embedding
-    let embeddings = await extractor(text, { pooling: 'mean' });
-    
-    // Apply MRL: layer_norm -> slice -> normalize
-    embeddings = layer_norm(embeddings, [embeddings.dims[1]])
-      .slice(null, [0, targetDim])
-      .normalize(2, -1);
-    
-    // Return in format compatible with existing code (has .data property)
-    return {
-      data: embeddings.data,
-      dims: [embeddings.dims[0], targetDim]
-    };
+    // If we've fallen back to legacy, use it
+    if (fallbackEmbedder) {
+      return await fallbackEmbedder(text, embedOptions);
+    }
+
+    async function doEmbed() {
+      // Generate full 768d embedding
+      let embeddings = await extractor(text, { pooling: 'mean' });
+
+      // Apply MRL: layer_norm -> slice -> normalize
+      embeddings = layer_norm(embeddings, [embeddings.dims[1]])
+        .slice(null, [0, targetDim])
+        .normalize(2, -1);
+
+      // Return in format compatible with existing code (has .data property)
+      return {
+        data: embeddings.data,
+        dims: [embeddings.dims[0], targetDim]
+      };
+    }
+
+    try {
+      return await doEmbed();
+    } catch (err) {
+      // Runtime corruption detection - try reload first
+      if (isCorruptionError(err)) {
+        console.error(`[MRL] Runtime corruption detected, attempting reload...`);
+        try {
+          extractor = await loadExtractor(true);
+          return await doEmbed();
+        } catch (reloadErr) {
+          // Reload failed - fall back to legacy model
+          console.error(`[MRL] Reload failed, falling back to legacy model...`);
+          const { createLegacyEmbedder } = await import('./mrl-embedder.js');
+          fallbackEmbedder = await createLegacyEmbedder();
+          embed.dimension = fallbackEmbedder.dimension;
+          embed.modelName = fallbackEmbedder.modelName;
+          return await fallbackEmbedder(text, embedOptions);
+        }
+      }
+      throw err;
+    }
   }
   
   // Attach metadata

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smart-coding-mcp",
-  "version": "2.3.1",
+  "version": "2.3.3",
   "description": "An extensible MCP server that enhances coding productivity with AI-powered features including semantic code search, intelligent indexing, and more, using local LLMs",
   "type": "module",
   "main": "index.js",

package/test/mrl-embedder.test.js

@@ -7,7 +7,7 @@
  * - Semantic similarity at different dimensions
  */
 
-import { describe, it, expect, beforeAll } from 'vitest';
+import { describe, it, expect, beforeAll, vi } from 'vitest';
 import { createMRLEmbedder, createLegacyEmbedder, createEmbedder, VALID_DIMENSIONS } from '../lib/mrl-embedder.js';
 import { cosineSimilarity } from '../lib/utils.js';
 
@@ -101,8 +101,114 @@ describe('createEmbedder Factory', () => {
       embeddingModel: 'Xenova/all-MiniLM-L6-v2',
       device: 'cpu'
     };
-    
+
     const embedder = await createEmbedder(config);
     expect(embedder.dimension).toBe(384);
   }, 120000);
 });
+
+describe('Auto-Recovery Logic', () => {
+  describe('Corruption Error Detection', () => {
+    it('should detect Protobuf parsing errors', async () => {
+      // We test this indirectly through the createEmbedder fallback behavior
+      // When MRL fails, it should fall back to legacy
+      const config = {
+        embeddingModel: 'nomic-ai/nomic-embed-text-v1.5',
+        embeddingDimension: 256,
+        device: 'cpu'
+      };
+
+      // This should succeed (model loads or recovers)
+      const embedder = await createEmbedder(config);
+      expect(embedder).toBeDefined();
+      expect(typeof embedder).toBe('function');
+    }, 120000);
+  });
+
+  describe('Runtime Recovery', () => {
+    let embedder;
+
+    beforeAll(async () => {
+      embedder = await createMRLEmbedder('nomic-ai/nomic-embed-text-v1.5', { dimension: 256 });
+    }, 120000);
+
+    it('should successfully embed after model is loaded', async () => {
+      const result = await embedder('test recovery');
+      expect(result.data).toBeDefined();
+      expect(result.dims[1]).toBe(256);
+    });
+
+    it('should have correct metadata after successful embedding', () => {
+      expect(embedder.dimension).toBe(256);
+      expect(embedder.modelName).toBe('nomic-ai/nomic-embed-text-v1.5');
+    });
+
+    it('should handle multiple sequential embeddings', async () => {
+      const texts = ['first text', 'second text', 'third text'];
+
+      for (const text of texts) {
+        const result = await embedder(text);
+        expect(result.data).toBeDefined();
+        expect(Array.from(result.data).length).toBe(256);
+      }
+    });
+  });
+
+  describe('Fallback Behavior', () => {
+    it('createEmbedder should fall back to legacy when MRL fails completely', async () => {
+      // Test that the factory handles failures gracefully
+      // Using a known-working legacy model
+      const config = {
+        embeddingModel: 'Xenova/all-MiniLM-L6-v2',
+        device: 'cpu'
+      };
+
+      const embedder = await createEmbedder(config);
+      expect(embedder.dimension).toBe(384);
+      expect(embedder.modelName).toBe('Xenova/all-MiniLM-L6-v2');
+
+      // Verify it actually works
+      const result = await embedder('fallback test');
+      expect(result.data).toBeDefined();
+    }, 120000);
+
+    it('legacy embedder should produce valid embeddings', async () => {
+      const embedder = await createLegacyEmbedder();
+
+      expect(embedder.dimension).toBe(384);
+      expect(embedder.modelName).toBe('Xenova/all-MiniLM-L6-v2');
+
+      const result = await embedder('legacy embedding test');
+      const vector = Array.from(result.data);
+
+      expect(vector.length).toBe(384);
+
+      // Check it's normalized
+      const magnitude = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0));
+      expect(magnitude).toBeCloseTo(1, 2);
+    }, 120000);
+  });
+});
+
+describe('Auto-Recovery with Mocked Pipeline', () => {
+  it('should handle corruption and recovery flow', async () => {
+    // This test verifies the recovery logic exists and embedder is resilient
+    const embedder = await createMRLEmbedder('nomic-ai/nomic-embed-text-v1.5', { dimension: 128 });
+
+    // Verify embedder works
+    const result1 = await embedder('before corruption test');
+    expect(result1.dims[1]).toBe(128);
+
+    // Run multiple embeddings to ensure stability
+    const results = await Promise.all([
+      embedder('concurrent test 1'),
+      embedder('concurrent test 2'),
+      embedder('concurrent test 3')
+    ]);
+
+    results.forEach(result => {
+      expect(result.dims[1]).toBe(128);
+      expect(Array.from(result.data).length).toBe(128);
+    });
+  }, 120000);
+});

package/test/mrl-recovery-real.test.js

@@ -0,0 +1,151 @@
+/**
+ * Real Integration Tests for MRL Auto-Recovery
+ *
+ * Tests embedder stability and error handling with real models.
+ * Destructive corruption tests are in mrl-recovery.test.js (mocked).
+ */
+
+import { describe, it, expect, beforeAll } from 'vitest';
+import { existsSync, readdirSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { createMRLEmbedder, createLegacyEmbedder, createEmbedder } from '../lib/mrl-embedder.js';
+
+// Find the transformers cache directory
+function getTransformersCacheDir() {
+  const transformersPath = dirname(fileURLToPath(import.meta.resolve('@huggingface/transformers')));
+  const cacheInParent = join(dirname(transformersPath), '.cache');
+  if (existsSync(cacheInParent)) return cacheInParent;
+  return join(transformersPath, '.cache');
+}
+
+describe('MRL Embedder Integration', () => {
+  const modelName = 'nomic-ai/nomic-embed-text-v1.5';
+  let embedder;
+
+  beforeAll(async () => {
+    console.log('[Test] Loading MRL embedder...');
+    embedder = await createMRLEmbedder(modelName, { dimension: 256 });
+    console.log('[Test] MRL embedder loaded');
+  }, 300000);
+
+  it('should create embedder with correct metadata', () => {
+    expect(embedder.dimension).toBe(256);
+    expect(embedder.modelName).toBe(modelName);
+    expect(embedder.device).toBe('cpu');
+  });
+
+  it('should produce correct dimension embeddings', async () => {
+    const result = await embedder('test embedding');
+    expect(result.data).toBeDefined();
+    expect(result.dims[1]).toBe(256);
+    expect(Array.from(result.data).length).toBe(256);
+  });
+
+  it('should produce normalized vectors', async () => {
+    const result = await embedder('normalized test');
+    const vector = Array.from(result.data);
+    const magnitude = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0));
+    expect(magnitude).toBeCloseTo(1, 3);
+  });
+
+  it('should handle multiple sequential embeddings', async () => {
+    const texts = ['first', 'second', 'third', 'fourth', 'fifth'];
+
+    for (const text of texts) {
+      const result = await embedder(text);
+      expect(result.data).toBeDefined();
+      expect(Array.from(result.data).length).toBe(256);
+    }
+  });
+
+  it('should handle concurrent embeddings', async () => {
+    const results = await Promise.all([
+      embedder('concurrent 1'),
+      embedder('concurrent 2'),
+      embedder('concurrent 3')
+    ]);
+
+    results.forEach(result => {
+      expect(result.data).toBeDefined();
+      expect(result.dims[1]).toBe(256);
+    });
+  });
+
+  it('should handle various input types', async () => {
+    const inputs = [
+      'normal text',
+      'a'.repeat(500),  // long text
+      'special: @#$%^&*()',
+      'unicode: 你好世界',
+      '   whitespace   '
+    ];
+
+    for (const input of inputs) {
+      const result = await embedder(input);
+      expect(result.data).toBeDefined();
+    }
+  });
+});
+
+describe('Legacy Embedder Integration', () => {
+  let embedder;
+
+  beforeAll(async () => {
+    embedder = await createLegacyEmbedder();
+  }, 120000);
+
+  it('should create legacy embedder with correct metadata', () => {
+    expect(embedder.dimension).toBe(384);
+    expect(embedder.modelName).toBe('Xenova/all-MiniLM-L6-v2');
+  });
+
+  it('should produce 384d embeddings', async () => {
+    const result = await embedder('legacy test');
+    expect(Array.from(result.data).length).toBe(384);
+  });
+});
+
+describe('Factory Function', () => {
+  it('should create MRL embedder for nomic model', async () => {
+    const embedder = await createEmbedder({
+      embeddingModel: 'nomic-ai/nomic-embed-text-v1.5',
+      embeddingDimension: 128,
+      device: 'cpu'
+    });
+
+    expect(embedder.dimension).toBe(128);
+    expect(embedder.modelName).toContain('nomic');
+  }, 300000);
+
+  it('should create legacy embedder for MiniLM', async () => {
+    const embedder = await createEmbedder({
+      embeddingModel: 'Xenova/all-MiniLM-L6-v2',
+      device: 'cpu'
+    });
+
+    expect(embedder.dimension).toBe(384);
+    expect(embedder.modelName).toBe('Xenova/all-MiniLM-L6-v2');
+  }, 120000);
+});
+
+describe('Cache Location Verification', () => {
+  it('should find transformers cache directory', () => {
+    const cacheDir = getTransformersCacheDir();
+    expect(existsSync(cacheDir)).toBe(true);
+    console.log(`[Test] Cache directory: ${cacheDir}`);
+  });
+
+  it('should have model files in cache', () => {
+    const cacheDir = getTransformersCacheDir();
+    const modelDir = join(cacheDir, 'nomic-ai', 'nomic-embed-text-v1.5', 'onnx');
+
+    if (existsSync(modelDir)) {
+      const files = readdirSync(modelDir);
+      expect(files.some(f => f.endsWith('.onnx'))).toBe(true);
+      console.log(`[Test] Model files: ${files.join(', ')}`);
+    } else {
+      console.log('[Test] Model directory not found (may need download)');
+    }
+  });
+});

package/test/mrl-recovery.test.js

@@ -0,0 +1,248 @@
+/**
+ * Tests for MRL Embedder Auto-Recovery
+ *
+ * Uses mocked pipeline to test corruption detection and recovery:
+ * - Startup corruption → clear cache → reload
+ * - Runtime corruption → reload → retry
+ * - Runtime corruption → reload fails → fallback to legacy
+ */
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+
+// Mock the transformers pipeline
+const mockPipeline = vi.fn();
+const mockLayerNorm = vi.fn();
+
+vi.mock('@huggingface/transformers', () => ({
+  pipeline: (...args) => mockPipeline(...args),
+  layer_norm: (...args) => mockLayerNorm(...args)
+}));
+
+// Mock fs for cache clearing
+vi.mock('fs', () => ({
+  existsSync: vi.fn(() => true),
+  rmSync: vi.fn()
+}));
+
+describe('MRL Auto-Recovery (Mocked)', () => {
+  let callCount = 0;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    callCount = 0;
+
+    // Default mock implementation for layer_norm
+    mockLayerNorm.mockImplementation((tensor) => ({
+      slice: () => ({
+        normalize: () => ({
+          data: new Float32Array(256).fill(0.1),
+          dims: [1, 256]
+        })
+      }),
+      dims: tensor.dims
+    }));
+  });
+
+  afterEach(() => {
+    vi.resetModules();
+  });
+
+  describe('Startup Recovery', () => {
+    it('should recover from corruption on first load', async () => {
+      // First call throws corruption, second succeeds
+      mockPipeline
+        .mockRejectedValueOnce(new Error('Protobuf parsing failed'))
+        .mockResolvedValueOnce(async () => ({
+          data: new Float32Array(768).fill(0.1),
+          dims: [1, 768]
+        }));
+
+      const { createMRLEmbedder } = await import('../lib/mrl-embedder.js');
+      const embedder = await createMRLEmbedder('test-model', { dimension: 256 });
+
+      expect(mockPipeline).toHaveBeenCalledTimes(2);
+      expect(embedder).toBeDefined();
+    });
+
+    it('should throw if recovery also fails with non-corruption error', async () => {
+      mockPipeline.mockRejectedValue(new Error('Network error'));
+
+      const { createMRLEmbedder } = await import('../lib/mrl-embedder.js');
+
+      await expect(createMRLEmbedder('test-model', { dimension: 256 }))
+        .rejects.toThrow('Network error');
+    });
+  });
+
+  describe('Runtime Recovery', () => {
+    it('should reload model on runtime corruption and retry', async () => {
+      let embedCallCount = 0;
+      const mockExtractor = vi.fn().mockImplementation(async () => {
+        embedCallCount++;
+        if (embedCallCount === 1) {
+          throw new Error('Protobuf parsing failed');
+        }
+        return {
+          data: new Float32Array(768).fill(0.1),
+          dims: [1, 768]
+        };
+      });
+
+      mockPipeline.mockResolvedValue(mockExtractor);
+
+      const { createMRLEmbedder } = await import('../lib/mrl-embedder.js');
+      const embedder = await createMRLEmbedder('test-model', { dimension: 256 });
+
+      // First embed triggers corruption, then reload and retry
+      const result = await embedder('test text');
+
+      expect(result.data).toBeDefined();
+      // Pipeline called: 1 (init) + 1 (reload after corruption) = 2
+      expect(mockPipeline).toHaveBeenCalledTimes(2);
+    });
+
+    it('should detect various corruption error messages', async () => {
+      const corruptionErrors = [
+        'Protobuf parsing failed',
+        'Invalid model format',
+        'ONNX model is corrupt'
+      ];
+
+      for (const errorMsg of corruptionErrors) {
+        vi.resetModules();
+        vi.clearAllMocks();
+
+        let throwError = true;
+        const mockExtractor = vi.fn().mockImplementation(async () => {
+          if (throwError) {
+            throwError = false;
+            throw new Error(errorMsg);
+          }
+          return {
+            data: new Float32Array(768).fill(0.1),
+            dims: [1, 768]
+          };
+        });
+
+        mockPipeline.mockResolvedValue(mockExtractor);
+        mockLayerNorm.mockImplementation((tensor) => ({
+          slice: () => ({
+            normalize: () => ({
+              data: new Float32Array(256).fill(0.1),
+              dims: [1, 256]
+            })
+          }),
+          dims: tensor.dims
+        }));
+
+        const { createMRLEmbedder } = await import('../lib/mrl-embedder.js');
+        const embedder = await createMRLEmbedder('test-model', { dimension: 256 });
+
+        const result = await embedder('test');
+        expect(result.data).toBeDefined();
+      }
+    });
+  });
+
+  describe('Fallback to Legacy', () => {
+    it('should fall back to legacy when reload also fails', async () => {
+      let pipelineCallCount = 0;
+      let embedCallCount = 0;
+
+      // Mock extractor that fails on first embed call
+      const corruptExtractor = vi.fn().mockImplementation(async () => {
+        embedCallCount++;
+        if (embedCallCount === 1) {
+          throw new Error('Protobuf parsing failed');
+        }
+        return {
+          data: new Float32Array(768).fill(0.1),
+          dims: [1, 768]
+        };
+      });
+
+      // Mock legacy extractor that works
+      const legacyExtractor = vi.fn().mockResolvedValue({
+        data: new Float32Array(384).fill(0.2),
+        dims: [1, 384]
+      });
+
+      mockPipeline.mockImplementation(async (task, model) => {
+        pipelineCallCount++;
+        if (pipelineCallCount <= 2) {
+          if (pipelineCallCount === 2) {
+            // Reload attempt fails
+            throw new Error('Network timeout');
+          }
+          return corruptExtractor;
+        }
+        // Third call is legacy model
+        return legacyExtractor;
+      });
+
+      const { createMRLEmbedder } = await import('../lib/mrl-embedder.js');
+      const embedder = await createMRLEmbedder('nomic-ai/nomic-embed-text-v1.5', { dimension: 256 });
+
+      // This should trigger: corruption → reload fail → fallback to legacy
+      const result = await embedder('test text');
+
+      expect(result.data).toBeDefined();
+      // Dimension should update to legacy (384)
+      expect(embedder.dimension).toBe(384);
+    });
+
+    it('should use fallback for subsequent calls after switching', async () => {
+      let pipelineCallCount = 0;
+      let embedCallCount = 0;
+
+      const corruptExtractor = vi.fn().mockImplementation(async () => {
+        embedCallCount++;
+        throw new Error('Protobuf parsing failed');
+      });
+
+      const legacyExtractor = vi.fn().mockResolvedValue({
+        data: new Float32Array(384).fill(0.2),
+        dims: [1, 384]
+      });
+
+      mockPipeline.mockImplementation(async (task, model) => {
+        pipelineCallCount++;
+        if (model.includes('nomic')) {
+          if (pipelineCallCount >= 2) {
+            throw new Error('Model unavailable');
+          }
+          return corruptExtractor;
+        }
+        return legacyExtractor;
+      });
+
+      const { createMRLEmbedder } = await import('../lib/mrl-embedder.js');
+      const embedder = await createMRLEmbedder('nomic-ai/nomic-embed-text-v1.5', { dimension: 256 });
+
+      // First call triggers fallback
+      await embedder('first');
+
+      // Subsequent calls should use legacy directly
+      const legacyCallsBefore = legacyExtractor.mock.calls.length;
+      await embedder('second');
+      await embedder('third');
+
+      expect(legacyExtractor.mock.calls.length).toBe(legacyCallsBefore + 2);
+    });
+  });
+
+  describe('Non-Corruption Errors', () => {
+    it('should throw non-corruption errors without recovery attempt', async () => {
+      const mockExtractor = vi.fn().mockRejectedValue(new Error('Out of memory'));
+      mockPipeline.mockResolvedValue(mockExtractor);
+
+      const { createMRLEmbedder } = await import('../lib/mrl-embedder.js');
+      const embedder = await createMRLEmbedder('test-model', { dimension: 256 });
+
+      await expect(embedder('test')).rejects.toThrow('Out of memory');
+
+      // Should not have attempted reload (only initial load)
+      expect(mockPipeline).toHaveBeenCalledTimes(1);
+    });
+  });
+});