ruvector 0.1.53 → 0.1.55

package/dist/core/onnx-embedder.d.ts

@@ -0,0 +1,81 @@
+/**
+ * ONNX WASM Embedder - Semantic embeddings for hooks
+ *
+ * Provides real transformer-based embeddings using all-MiniLM-L6-v2
+ * running in pure WASM (no native dependencies).
+ *
+ * Uses bundled ONNX WASM files from src/core/onnx/
+ *
+ * Features:
+ * - 384-dimensional semantic embeddings
+ * - Real semantic understanding (not hash-based)
+ * - Cached model loading (downloads from HuggingFace on first use)
+ * - Batch embedding support
+ */
+export interface OnnxEmbedderConfig {
+    modelId?: string;
+    maxLength?: number;
+    normalize?: boolean;
+    cacheDir?: string;
+}
+export interface EmbeddingResult {
+    embedding: number[];
+    dimension: number;
+    timeMs: number;
+}
+export interface SimilarityResult {
+    similarity: number;
+    timeMs: number;
+}
+/**
+ * Check if ONNX embedder is available (bundled files exist)
+ */
+export declare function isOnnxAvailable(): boolean;
+/**
+ * Initialize the ONNX embedder (downloads model if needed)
+ */
+export declare function initOnnxEmbedder(config?: OnnxEmbedderConfig): Promise<boolean>;
+/**
+ * Generate embedding for text
+ */
+export declare function embed(text: string): Promise<EmbeddingResult>;
+/**
+ * Generate embeddings for multiple texts
+ */
+export declare function embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
+/**
+ * Calculate cosine similarity between two texts
+ */
+export declare function similarity(text1: string, text2: string): Promise<SimilarityResult>;
+/**
+ * Calculate cosine similarity between two embeddings
+ */
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+/**
+ * Get embedding dimension
+ */
+export declare function getDimension(): number;
+/**
+ * Check if embedder is ready
+ */
+export declare function isReady(): boolean;
+/**
+ * Get embedder stats
+ */
+export declare function getStats(): {
+    ready: boolean;
+    dimension: number;
+    model: string;
+};
+export declare class OnnxEmbedder {
+    private config;
+    constructor(config?: OnnxEmbedderConfig);
+    init(): Promise<boolean>;
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[]): Promise<number[][]>;
+    similarity(text1: string, text2: string): Promise<number>;
+    get dimension(): number;
+    get ready(): boolean;
+}
+export default OnnxEmbedder;
+//# sourceMappingURL=onnx-embedder.d.ts.map

package/dist/core/onnx-embedder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"onnx-embedder.d.ts","sourceRoot":"","sources":["../../src/core/onnx-embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAUH,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB;AAYD;;GAEG;AACH,wBAAgB,eAAe,IAAI,OAAO,CAOzC;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,MAAM,GAAE,kBAAuB,GAAG,OAAO,CAAC,OAAO,CAAC,CA8DxF;AAED;;GAEG;AACH,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAiBlE;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAyB5E;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAaxF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAiBjE;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED;;GAEG;AACH,wBAAgB,OAAO,IAAI,OAAO,CAEjC;AAED;;GAEG;AACH,wBAAgB,QAAQ,IAAI;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAM/E;AAGD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAqB;gBAEvB,MAAM,GAAE,kBAAuB;IAIrC,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC;IAIxB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKtC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAKhD,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK/D,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,IAAI,KAAK,IAAI,OAAO,CAEnB;CACF;AAED,eAAe,YAAY,CAAC"}

package/dist/core/onnx-embedder.js

@@ -0,0 +1,269 @@
+"use strict";
+/**
+ * ONNX WASM Embedder - Semantic embeddings for hooks
+ *
+ * Provides real transformer-based embeddings using all-MiniLM-L6-v2
+ * running in pure WASM (no native dependencies).
+ *
+ * Uses bundled ONNX WASM files from src/core/onnx/
+ *
+ * Features:
+ * - 384-dimensional semantic embeddings
+ * - Real semantic understanding (not hash-based)
+ * - Cached model loading (downloads from HuggingFace on first use)
+ * - Batch embedding support
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OnnxEmbedder = void 0;
+exports.isOnnxAvailable = isOnnxAvailable;
+exports.initOnnxEmbedder = initOnnxEmbedder;
+exports.embed = embed;
+exports.embedBatch = embedBatch;
+exports.similarity = similarity;
+exports.cosineSimilarity = cosineSimilarity;
+exports.getDimension = getDimension;
+exports.isReady = isReady;
+exports.getStats = getStats;
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+// Force native dynamic import (avoids TypeScript transpiling to require)
+// eslint-disable-next-line @typescript-eslint/no-implied-eval
+const dynamicImport = new Function('specifier', 'return import(specifier)');
+// Lazy-loaded module state
+let wasmModule = null;
+let embedder = null;
+let loadError = null;
+let loadPromise = null;
+let isInitialized = false;
+// Default model
+const DEFAULT_MODEL = 'all-MiniLM-L6-v2';
+/**
+ * Check if ONNX embedder is available (bundled files exist)
+ */
+function isOnnxAvailable() {
+    try {
+        const pkgPath = path.join(__dirname, 'onnx', 'pkg', 'ruvector_onnx_embeddings_wasm.js');
+        return fs.existsSync(pkgPath);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Initialize the ONNX embedder (downloads model if needed)
+ */
+async function initOnnxEmbedder(config = {}) {
+    if (isInitialized)
+        return true;
+    if (loadError)
+        throw loadError;
+    if (loadPromise) {
+        await loadPromise;
+        return isInitialized;
+    }
+    loadPromise = (async () => {
+        try {
+            // Paths to bundled ONNX files
+            const pkgPath = path.join(__dirname, 'onnx', 'pkg', 'ruvector_onnx_embeddings_wasm.js');
+            const loaderPath = path.join(__dirname, 'onnx', 'loader.js');
+            if (!fs.existsSync(pkgPath)) {
+                throw new Error('ONNX WASM files not bundled. The onnx/ directory is missing.');
+            }
+            // Dynamic import of bundled modules
+            wasmModule = await dynamicImport(pkgPath);
+            // Initialize WASM module (loads the .wasm file)
+            const wasmPath = path.join(__dirname, 'onnx', 'pkg', 'ruvector_onnx_embeddings_wasm_bg.wasm');
+            if (wasmModule.default && typeof wasmModule.default === 'function') {
+                // For bundler-style initialization, pass the wasm buffer
+                const wasmBytes = fs.readFileSync(wasmPath);
+                await wasmModule.default(wasmBytes);
+            }
+            const loaderModule = await dynamicImport(loaderPath);
+            const { ModelLoader } = loaderModule;
+            // Create model loader with caching
+            const modelLoader = new ModelLoader({
+                cache: true,
+                cacheDir: config.cacheDir || path.join(process.env.HOME || '/tmp', '.ruvector', 'models'),
+            });
+            // Load model (downloads from HuggingFace on first use)
+            const modelId = config.modelId || DEFAULT_MODEL;
+            console.error(`Loading ONNX model: ${modelId}...`);
+            const { modelBytes, tokenizerJson, config: modelConfig } = await modelLoader.loadModel(modelId);
+            // Create embedder with config
+            const embedderConfig = new wasmModule.WasmEmbedderConfig()
+                .setMaxLength(config.maxLength || modelConfig.maxLength || 256)
+                .setNormalize(config.normalize !== false)
+                .setPooling(0); // Mean pooling
+            embedder = wasmModule.WasmEmbedder.withConfig(modelBytes, tokenizerJson, embedderConfig);
+            console.error(`ONNX embedder ready: ${embedder.dimension()}d`);
+            isInitialized = true;
+        }
+        catch (e) {
+            loadError = new Error(`Failed to initialize ONNX embedder: ${e.message}`);
+            throw loadError;
+        }
+    })();
+    await loadPromise;
+    return isInitialized;
+}
+/**
+ * Generate embedding for text
+ */
+async function embed(text) {
+    if (!isInitialized) {
+        await initOnnxEmbedder();
+    }
+    if (!embedder) {
+        throw new Error('ONNX embedder not initialized');
+    }
+    const start = performance.now();
+    const embedding = embedder.embedOne(text);
+    const timeMs = performance.now() - start;
+    return {
+        embedding: Array.from(embedding),
+        dimension: embedding.length,
+        timeMs,
+    };
+}
+/**
+ * Generate embeddings for multiple texts
+ */
+async function embedBatch(texts) {
+    if (!isInitialized) {
+        await initOnnxEmbedder();
+    }
+    if (!embedder) {
+        throw new Error('ONNX embedder not initialized');
+    }
+    const start = performance.now();
+    const batchEmbeddings = embedder.embedBatch(texts);
+    const totalTime = performance.now() - start;
+    const dimension = embedder.dimension();
+    const results = [];
+    for (let i = 0; i < texts.length; i++) {
+        const embedding = batchEmbeddings.slice(i * dimension, (i + 1) * dimension);
+        results.push({
+            embedding: Array.from(embedding),
+            dimension,
+            timeMs: totalTime / texts.length,
+        });
+    }
+    return results;
+}
+/**
+ * Calculate cosine similarity between two texts
+ */
+async function similarity(text1, text2) {
+    if (!isInitialized) {
+        await initOnnxEmbedder();
+    }
+    if (!embedder) {
+        throw new Error('ONNX embedder not initialized');
+    }
+    const start = performance.now();
+    const sim = embedder.similarity(text1, text2);
+    const timeMs = performance.now() - start;
+    return { similarity: sim, timeMs };
+}
+/**
+ * Calculate cosine similarity between two embeddings
+ */
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+        throw new Error('Embeddings must have same dimension');
+    }
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dotProduct += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+    return magnitude === 0 ? 0 : dotProduct / magnitude;
+}
+/**
+ * Get embedding dimension
+ */
+function getDimension() {
+    return embedder ? embedder.dimension() : 384;
+}
+/**
+ * Check if embedder is ready
+ */
+function isReady() {
+    return isInitialized;
+}
+/**
+ * Get embedder stats
+ */
+function getStats() {
+    return {
+        ready: isInitialized,
+        dimension: embedder ? embedder.dimension() : 384,
+        model: DEFAULT_MODEL,
+    };
+}
+// Export class wrapper for compatibility
+class OnnxEmbedder {
+    constructor(config = {}) {
+        this.config = config;
+    }
+    async init() {
+        return initOnnxEmbedder(this.config);
+    }
+    async embed(text) {
+        const result = await embed(text);
+        return result.embedding;
+    }
+    async embedBatch(texts) {
+        const results = await embedBatch(texts);
+        return results.map(r => r.embedding);
+    }
+    async similarity(text1, text2) {
+        const result = await similarity(text1, text2);
+        return result.similarity;
+    }
+    get dimension() {
+        return getDimension();
+    }
+    get ready() {
+        return isReady();
+    }
+}
+exports.OnnxEmbedder = OnnxEmbedder;
+exports.default = OnnxEmbedder;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ruvector",
-  "version": "0.1.53",
+  "version": "0.1.55",
   "description": "High-performance vector database for Node.js with automatic native/WASM fallback",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -38,7 +38,10 @@
     "lora",
     "ewc",
     "adaptive-learning",
-    "continual-learning"
+    "continual-learning",
+    "onnx",
+    "semantic-embeddings",
+    "minilm"
   ],
   "author": "ruv.io Team <info@ruv.io> (https://ruv.io)",
   "homepage": "https://ruv.io",