ruvector 0.2.14 → 0.2.15

package/bin/mcp-server.js

@@ -428,7 +428,7 @@ class Intelligence {
 const server = new Server(
   {
     name: 'ruvector',
-    version: '0.2.14',
+    version: '0.2.15',
   },
   {
     capabilities: {

package/dist/core/onnx/loader.js

@@ -0,0 +1,348 @@
+/**
+ * Model Loader for RuVector ONNX Embeddings WASM
+ *
+ * Provides easy loading of pre-trained models from HuggingFace Hub
+ */
+
+/**
+ * Pre-configured models with their HuggingFace URLs
+ */
+export const MODELS = {
+    // Sentence Transformers - Small & Fast
+    'all-MiniLM-L6-v2': {
+        name: 'all-MiniLM-L6-v2',
+        dimension: 384,
+        maxLength: 256,
+        size: '23MB',
+        description: 'Fast, general-purpose embeddings',
+        model: 'https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/onnx/model.onnx',
+        tokenizer: 'https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json',
+    },
+    'all-MiniLM-L12-v2': {
+        name: 'all-MiniLM-L12-v2',
+        dimension: 384,
+        maxLength: 256,
+        size: '33MB',
+        description: 'Better quality, balanced speed',
+        model: 'https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2/resolve/main/onnx/model.onnx',
+        tokenizer: 'https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2/resolve/main/tokenizer.json',
+    },
+
+    // BGE Models - State of the art
+    'bge-small-en-v1.5': {
+        name: 'bge-small-en-v1.5',
+        dimension: 384,
+        maxLength: 512,
+        size: '33MB',
+        description: 'State-of-the-art small model',
+        model: 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx',
+        tokenizer: 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json',
+    },
+    'bge-base-en-v1.5': {
+        name: 'bge-base-en-v1.5',
+        dimension: 768,
+        maxLength: 512,
+        size: '110MB',
+        description: 'Best overall quality',
+        model: 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx',
+        tokenizer: 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json',
+    },
+
+    // E5 Models - Microsoft
+    'e5-small-v2': {
+        name: 'e5-small-v2',
+        dimension: 384,
+        maxLength: 512,
+        size: '33MB',
+        description: 'Excellent for search & retrieval',
+        model: 'https://huggingface.co/intfloat/e5-small-v2/resolve/main/onnx/model.onnx',
+        tokenizer: 'https://huggingface.co/intfloat/e5-small-v2/resolve/main/tokenizer.json',
+    },
+
+    // GTE Models - Alibaba
+    'gte-small': {
+        name: 'gte-small',
+        dimension: 384,
+        maxLength: 512,
+        size: '33MB',
+        description: 'Good multilingual support',
+        model: 'https://huggingface.co/thenlper/gte-small/resolve/main/onnx/model.onnx',
+        tokenizer: 'https://huggingface.co/thenlper/gte-small/resolve/main/tokenizer.json',
+    },
+};
+
+/**
+ * Default model for quick start
+ */
+export const DEFAULT_MODEL = 'all-MiniLM-L6-v2';
+
+/**
+ * Model loader with caching support
+ */
+export class ModelLoader {
+    constructor(options = {}) {
+        this.cache = options.cache ?? true;
+        this.cacheStorage = options.cacheStorage ?? 'ruvector-models';
+        this.onProgress = options.onProgress ?? null;
+    }
+
+    /**
+     * Load a pre-configured model by name
+     * @param {string} modelName - Model name from MODELS
+     * @returns {Promise<{modelBytes: Uint8Array, tokenizerJson: string, config: object}>}
+     */
+    async loadModel(modelName = DEFAULT_MODEL) {
+        const modelConfig = MODELS[modelName];
+        if (!modelConfig) {
+            throw new Error(`Unknown model: ${modelName}. Available: ${Object.keys(MODELS).join(', ')}`);
+        }
+
+        console.log(`Loading model: ${modelConfig.name} (${modelConfig.size})`);
+
+        const [modelBytes, tokenizerJson] = await Promise.all([
+            this.fetchWithCache(modelConfig.model, `${modelName}-model.onnx`, 'arraybuffer'),
+            this.fetchWithCache(modelConfig.tokenizer, `${modelName}-tokenizer.json`, 'text'),
+        ]);
+
+        return {
+            modelBytes: new Uint8Array(modelBytes),
+            tokenizerJson,
+            config: modelConfig,
+        };
+    }
+
+    /**
+     * Load model from custom URLs
+     * @param {string} modelUrl - URL to ONNX model
+     * @param {string} tokenizerUrl - URL to tokenizer.json
+     * @returns {Promise<{modelBytes: Uint8Array, tokenizerJson: string}>}
+     */
+    async loadFromUrls(modelUrl, tokenizerUrl) {
+        const [modelBytes, tokenizerJson] = await Promise.all([
+            this.fetchWithCache(modelUrl, null, 'arraybuffer'),
+            this.fetchWithCache(tokenizerUrl, null, 'text'),
+        ]);
+
+        return {
+            modelBytes: new Uint8Array(modelBytes),
+            tokenizerJson,
+        };
+    }
+
+    /**
+     * Load model from local files (Node.js)
+     * @param {string} modelPath - Path to ONNX model
+     * @param {string} tokenizerPath - Path to tokenizer.json
+     * @returns {Promise<{modelBytes: Uint8Array, tokenizerJson: string}>}
+     */
+    async loadFromFiles(modelPath, tokenizerPath) {
+        // Node.js environment
+        if (typeof process !== 'undefined' && process.versions?.node) {
+            const fs = await import('fs/promises');
+            const [modelBytes, tokenizerJson] = await Promise.all([
+                fs.readFile(modelPath),
+                fs.readFile(tokenizerPath, 'utf8'),
+            ]);
+            return {
+                modelBytes: new Uint8Array(modelBytes),
+                tokenizerJson,
+            };
+        }
+        throw new Error('loadFromFiles is only available in Node.js');
+    }
+
+    /**
+     * Fetch with optional caching (uses Cache API in browsers)
+     */
+    async fetchWithCache(url, cacheKey, responseType) {
+        // Try cache first (browser only)
+        if (this.cache && typeof caches !== 'undefined' && cacheKey) {
+            try {
+                const cache = await caches.open(this.cacheStorage);
+                const cached = await cache.match(cacheKey);
+                if (cached) {
+                    console.log(`  Cache hit: ${cacheKey}`);
+                    return responseType === 'arraybuffer'
+                        ? await cached.arrayBuffer()
+                        : await cached.text();
+                }
+            } catch (e) {
+                // Cache API not available, continue with fetch
+            }
+        }
+
+        // Fetch from network
+        console.log(`  Downloading: ${url}`);
+        const response = await this.fetchWithProgress(url);
+
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+        }
+
+        // Clone for caching
+        const responseClone = response.clone();
+
+        // Cache the response (browser only)
+        if (this.cache && typeof caches !== 'undefined' && cacheKey) {
+            try {
+                const cache = await caches.open(this.cacheStorage);
+                await cache.put(cacheKey, responseClone);
+            } catch (e) {
+                // Cache write failed, continue
+            }
+        }
+
+        return responseType === 'arraybuffer'
+            ? await response.arrayBuffer()
+            : await response.text();
+    }
+
+    /**
+     * Fetch with progress reporting
+     */
+    async fetchWithProgress(url) {
+        const response = await fetch(url);
+
+        if (!this.onProgress || !response.body) {
+            return response;
+        }
+
+        const contentLength = response.headers.get('content-length');
+        if (!contentLength) {
+            return response;
+        }
+
+        const total = parseInt(contentLength, 10);
+        let loaded = 0;
+
+        const reader = response.body.getReader();
+        const chunks = [];
+
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) break;
+
+            chunks.push(value);
+            loaded += value.length;
+
+            this.onProgress({
+                loaded,
+                total,
+                percent: Math.round((loaded / total) * 100),
+            });
+        }
+
+        const body = new Uint8Array(loaded);
+        let position = 0;
+        for (const chunk of chunks) {
+            body.set(chunk, position);
+            position += chunk.length;
+        }
+
+        return new Response(body, {
+            headers: response.headers,
+            status: response.status,
+            statusText: response.statusText,
+        });
+    }
+
+    /**
+     * Clear cached models
+     */
+    async clearCache() {
+        if (typeof caches !== 'undefined') {
+            await caches.delete(this.cacheStorage);
+            console.log('Model cache cleared');
+        }
+    }
+
+    /**
+     * List available models
+     */
+    static listModels() {
+        return Object.entries(MODELS).map(([key, config]) => ({
+            id: key,
+            ...config,
+        }));
+    }
+}
+
+/**
+ * Quick helper to create an embedder with a pre-configured model
+ *
+ * @example
+ * ```javascript
+ * import { createEmbedder } from './loader.js';
+ *
+ * const embedder = await createEmbedder('all-MiniLM-L6-v2');
+ * const embedding = embedder.embedOne("Hello world");
+ * ```
+ */
+export async function createEmbedder(modelName = DEFAULT_MODEL, wasmModule = null) {
+    // Import WASM module if not provided
+    if (!wasmModule) {
+        wasmModule = await import('./pkg/ruvector_onnx_embeddings_wasm.js');
+        await wasmModule.default();
+    }
+
+    const loader = new ModelLoader();
+    const { modelBytes, tokenizerJson, config } = await loader.loadModel(modelName);
+
+    const embedderConfig = new wasmModule.WasmEmbedderConfig()
+        .setMaxLength(config.maxLength)
+        .setNormalize(true)
+        .setPooling(0); // Mean pooling
+
+    const embedder = wasmModule.WasmEmbedder.withConfig(
+        modelBytes,
+        tokenizerJson,
+        embedderConfig
+    );
+
+    return embedder;
+}
+
+/**
+ * Quick helper for one-off embedding (loads model, embeds, returns)
+ *
+ * @example
+ * ```javascript
+ * import { embed } from './loader.js';
+ *
+ * const embedding = await embed("Hello world");
+ * const embeddings = await embed(["Hello", "World"]);
+ * ```
+ */
+export async function embed(text, modelName = DEFAULT_MODEL) {
+    const embedder = await createEmbedder(modelName);
+
+    if (Array.isArray(text)) {
+        return embedder.embedBatch(text);
+    }
+    return embedder.embedOne(text);
+}
+
+/**
+ * Quick helper for similarity comparison
+ *
+ * @example
+ * ```javascript
+ * import { similarity } from './loader.js';
+ *
+ * const score = await similarity("I love dogs", "I adore puppies");
+ * console.log(score); // ~0.85
+ * ```
+ */
+export async function similarity(text1, text2, modelName = DEFAULT_MODEL) {
+    const embedder = await createEmbedder(modelName);
+    return embedder.similarity(text1, text2);
+}
+
+export default {
+    MODELS,
+    DEFAULT_MODEL,
+    ModelLoader,
+    createEmbedder,
+    embed,
+    similarity,
+};

package/dist/core/onnx/pkg/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 rUv
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.