ruvector 0.1.26 → 0.1.27

package/dist/services/embedding-service.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Embedding Service - Unified embedding generation and management
+ *
+ * This service provides a unified interface for generating, caching, and
+ * managing embeddings from various sources (local models, APIs, etc.)
+ */
+/**
+ * Embedding provider interface
+ */
+export interface EmbeddingProvider {
+    /** Provider name */
+    name: string;
+    /** Generate embeddings for texts */
+    embed(texts: string[]): Promise<number[][]>;
+    /** Get embedding dimensions */
+    getDimensions(): number;
+}
+/**
+ * Embedding service configuration
+ */
+export interface EmbeddingServiceConfig {
+    /** Default provider to use */
+    defaultProvider?: string;
+    /** Maximum cache size */
+    maxCacheSize?: number;
+    /** Cache TTL in milliseconds */
+    cacheTtl?: number;
+    /** Batch size for embedding generation */
+    batchSize?: number;
+}
+/**
+ * Mock embedding provider for testing
+ */
+export declare class MockEmbeddingProvider implements EmbeddingProvider {
+    name: string;
+    private dimensions;
+    constructor(dimensions?: number);
+    embed(texts: string[]): Promise<number[][]>;
+    getDimensions(): number;
+}
+/**
+ * Simple local embedding using character n-grams
+ * This is a fallback when no external provider is available
+ */
+export declare class LocalNGramProvider implements EmbeddingProvider {
+    name: string;
+    private dimensions;
+    private ngramSize;
+    constructor(dimensions?: number, ngramSize?: number);
+    embed(texts: string[]): Promise<number[][]>;
+    private embedSingle;
+    private hashNgram;
+    getDimensions(): number;
+}
+/**
+ * Embedding service with caching and batching
+ */
+export declare class EmbeddingService {
+    private providers;
+    private cache;
+    private config;
+    constructor(config?: EmbeddingServiceConfig);
+    /**
+     * Register an embedding provider
+     */
+    registerProvider(provider: EmbeddingProvider): void;
+    /**
+     * Get a registered provider
+     */
+    getProvider(name?: string): EmbeddingProvider;
+    /**
+     * Generate embeddings for texts with caching
+     *
+     * @param texts - Texts to embed
+     * @param provider - Provider name (uses default if not specified)
+     * @returns Array of embeddings
+     */
+    embed(texts: string[], provider?: string): Promise<number[][]>;
+    /**
+     * Generate a single embedding
+     */
+    embedOne(text: string, provider?: string): Promise<number[]>;
+    /**
+     * Add entry to cache with LRU eviction
+     */
+    private addToCache;
+    /**
+     * Compute cosine similarity between two embeddings
+     */
+    cosineSimilarity(a: number[], b: number[]): number;
+    /**
+     * Find most similar texts from a corpus
+     */
+    findSimilar(query: string, corpus: string[], k?: number, provider?: string): Promise<{
+        text: string;
+        similarity: number;
+        index: number;
+    }[]>;
+    /**
+     * Get cache statistics
+     */
+    getCacheStats(): {
+        size: number;
+        maxSize: number;
+        hitRate: number;
+    };
+    /**
+     * Clear the cache
+     */
+    clearCache(): void;
+    /**
+     * Get embedding dimensions for a provider
+     */
+    getDimensions(provider?: string): number;
+    /**
+     * List available providers
+     */
+    listProviders(): string[];
+}
+/**
+ * Create an embedding service instance
+ */
+export declare function createEmbeddingService(config?: EmbeddingServiceConfig): EmbeddingService;
+/**
+ * Get the default embedding service instance
+ */
+export declare function getDefaultEmbeddingService(): EmbeddingService;
+declare const _default: {
+    EmbeddingService: typeof EmbeddingService;
+    LocalNGramProvider: typeof LocalNGramProvider;
+    MockEmbeddingProvider: typeof MockEmbeddingProvider;
+    createEmbeddingService: typeof createEmbeddingService;
+    getDefaultEmbeddingService: typeof getDefaultEmbeddingService;
+};
+export default _default;
+//# sourceMappingURL=embedding-service.d.ts.map

package/dist/services/embedding-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-service.d.ts","sourceRoot":"","sources":["../../src/services/embedding-service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5C,+BAA+B;IAC/B,aAAa,IAAI,MAAM,CAAC;CACzB;AAWD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,8BAA8B;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAeD;;GAEG;AACH,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,IAAI,SAAU;IACd,OAAO,CAAC,UAAU,CAAS;gBAEf,UAAU,GAAE,MAAY;IAI9B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAoBjD,aAAa,IAAI,MAAM;CAGxB;AAED;;;GAGG;AACH,qBAAa,kBAAmB,YAAW,iBAAiB;IAC1D,IAAI,SAAiB;IACrB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAEd,UAAU,GAAE,MAAY,EAAE,SAAS,GAAE,MAAU;IAKrD,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAIjD,OAAO,CAAC,WAAW;IAiBnB,OAAO,CAAC,SAAS;IAQjB,aAAa,IAAI,MAAM;CAGxB;AAED;;GAEG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,SAAS,CAA6C;IAC9D,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,MAAM,CAAmC;gBAErC,MAAM,GAAE,sBAA2B;IAa/C;;OAEG;IACH,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,IAAI;IAInD;;OAEG;IACH,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,iBAAiB;IAS7C;;;;;;OAMG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAkDpE;;OAEG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKlE;;OAEG;IACH,OAAO,CAAC,UAAU;IAwBlB;;OAEG;IACH,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM;IAmBlD;;OAEG;IACG,WAAW,CACf,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,EAChB,CAAC,GAAE,MAAU,EACb,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAcjE;;OAEG;IACH,aAAa,IAAI;QACf,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACjB;IAaD;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,aAAa,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IAIxC;;OAEG;IACH,aAAa,IAAI,MAAM,EAAE;CAG1B;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,CAAC,EAAE,sBAAsB,GAC9B,gBAAgB,CAElB;AAKD;;GAEG;AACH,wBAAgB,0BAA0B,IAAI,gBAAgB,CAK7D;;;;;;;;AAED,wBAME"}

package/dist/services/embedding-service.js

@@ -0,0 +1,294 @@
+"use strict";
+/**
+ * Embedding Service - Unified embedding generation and management
+ *
+ * This service provides a unified interface for generating, caching, and
+ * managing embeddings from various sources (local models, APIs, etc.)
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EmbeddingService = exports.LocalNGramProvider = exports.MockEmbeddingProvider = void 0;
+exports.createEmbeddingService = createEmbeddingService;
+exports.getDefaultEmbeddingService = getDefaultEmbeddingService;
+/**
+ * Simple hash function for cache keys
+ */
+function hashText(text) {
+    let hash = 0;
+    for (let i = 0; i < text.length; i++) {
+        const char = text.charCodeAt(i);
+        hash = ((hash << 5) - hash) + char;
+        hash = hash & hash;
+    }
+    return `h${hash.toString(36)}`;
+}
+/**
+ * Mock embedding provider for testing
+ */
+class MockEmbeddingProvider {
+    constructor(dimensions = 384) {
+        this.name = 'mock';
+        this.dimensions = dimensions;
+    }
+    async embed(texts) {
+        return texts.map(text => {
+            // Generate deterministic pseudo-random embeddings based on text
+            const embedding = [];
+            let seed = 0;
+            for (let i = 0; i < text.length; i++) {
+                seed = ((seed << 5) - seed + text.charCodeAt(i)) | 0;
+            }
+            for (let i = 0; i < this.dimensions; i++) {
+                seed = (seed * 1103515245 + 12345) | 0;
+                embedding.push((seed % 1000) / 1000 - 0.5);
+            }
+            // Normalize
+            const norm = Math.sqrt(embedding.reduce((s, v) => s + v * v, 0));
+            return embedding.map(v => v / (norm || 1));
+        });
+    }
+    getDimensions() {
+        return this.dimensions;
+    }
+}
+exports.MockEmbeddingProvider = MockEmbeddingProvider;
+/**
+ * Simple local embedding using character n-grams
+ * This is a fallback when no external provider is available
+ */
+class LocalNGramProvider {
+    constructor(dimensions = 256, ngramSize = 3) {
+        this.name = 'local-ngram';
+        this.dimensions = dimensions;
+        this.ngramSize = ngramSize;
+    }
+    async embed(texts) {
+        return texts.map(text => this.embedSingle(text));
+    }
+    embedSingle(text) {
+        const embedding = new Array(this.dimensions).fill(0);
+        const normalized = text.toLowerCase().replace(/[^a-z0-9]/g, ' ');
+        // Generate n-grams and hash them into embedding dimensions
+        for (let i = 0; i <= normalized.length - this.ngramSize; i++) {
+            const ngram = normalized.slice(i, i + this.ngramSize);
+            const hash = this.hashNgram(ngram);
+            const idx = Math.abs(hash) % this.dimensions;
+            embedding[idx] += hash > 0 ? 1 : -1;
+        }
+        // Normalize
+        const norm = Math.sqrt(embedding.reduce((s, v) => s + v * v, 0));
+        return embedding.map(v => v / (norm || 1));
+    }
+    hashNgram(ngram) {
+        let hash = 0;
+        for (let i = 0; i < ngram.length; i++) {
+            hash = ((hash << 5) - hash + ngram.charCodeAt(i)) | 0;
+        }
+        return hash;
+    }
+    getDimensions() {
+        return this.dimensions;
+    }
+}
+exports.LocalNGramProvider = LocalNGramProvider;
+/**
+ * Embedding service with caching and batching
+ */
+class EmbeddingService {
+    constructor(config = {}) {
+        this.providers = new Map();
+        this.cache = new Map();
+        this.config = {
+            defaultProvider: config.defaultProvider ?? 'local-ngram',
+            maxCacheSize: config.maxCacheSize ?? 10000,
+            cacheTtl: config.cacheTtl ?? 3600000, // 1 hour
+            batchSize: config.batchSize ?? 32,
+        };
+        // Register default providers
+        this.registerProvider(new LocalNGramProvider());
+        this.registerProvider(new MockEmbeddingProvider());
+    }
+    /**
+     * Register an embedding provider
+     */
+    registerProvider(provider) {
+        this.providers.set(provider.name, provider);
+    }
+    /**
+     * Get a registered provider
+     */
+    getProvider(name) {
+        const providerName = name ?? this.config.defaultProvider;
+        const provider = this.providers.get(providerName);
+        if (!provider) {
+            throw new Error(`Provider not found: ${providerName}`);
+        }
+        return provider;
+    }
+    /**
+     * Generate embeddings for texts with caching
+     *
+     * @param texts - Texts to embed
+     * @param provider - Provider name (uses default if not specified)
+     * @returns Array of embeddings
+     */
+    async embed(texts, provider) {
+        const providerInstance = this.getProvider(provider);
+        const providerName = providerInstance.name;
+        const now = Date.now();
+        // Check cache and collect texts that need embedding
+        const results = new Array(texts.length).fill(null);
+        const uncachedIndices = [];
+        const uncachedTexts = [];
+        for (let i = 0; i < texts.length; i++) {
+            const cacheKey = `${providerName}:${hashText(texts[i])}`;
+            const cached = this.cache.get(cacheKey);
+            if (cached && now - cached.timestamp < this.config.cacheTtl) {
+                results[i] = cached.embedding;
+                cached.hits++;
+            }
+            else {
+                uncachedIndices.push(i);
+                uncachedTexts.push(texts[i]);
+            }
+        }
+        // Generate embeddings for uncached texts in batches
+        if (uncachedTexts.length > 0) {
+            const batches = [];
+            for (let i = 0; i < uncachedTexts.length; i += this.config.batchSize) {
+                batches.push(uncachedTexts.slice(i, i + this.config.batchSize));
+            }
+            let batchOffset = 0;
+            for (const batch of batches) {
+                const embeddings = await providerInstance.embed(batch);
+                for (let j = 0; j < embeddings.length; j++) {
+                    const originalIndex = uncachedIndices[batchOffset + j];
+                    results[originalIndex] = embeddings[j];
+                    // Cache the result
+                    const cacheKey = `${providerName}:${hashText(texts[originalIndex])}`;
+                    this.addToCache(cacheKey, embeddings[j], now);
+                }
+                batchOffset += batch.length;
+            }
+        }
+        return results;
+    }
+    /**
+     * Generate a single embedding
+     */
+    async embedOne(text, provider) {
+        const results = await this.embed([text], provider);
+        return results[0];
+    }
+    /**
+     * Add entry to cache with LRU eviction
+     */
+    addToCache(key, embedding, timestamp) {
+        // Evict old entries if cache is full
+        if (this.cache.size >= this.config.maxCacheSize) {
+            // Find and remove least recently used entry
+            let oldestKey = '';
+            let oldestTime = Infinity;
+            let lowestHits = Infinity;
+            for (const [k, v] of this.cache.entries()) {
+                if (v.hits < lowestHits || (v.hits === lowestHits && v.timestamp < oldestTime)) {
+                    oldestKey = k;
+                    oldestTime = v.timestamp;
+                    lowestHits = v.hits;
+                }
+            }
+            if (oldestKey) {
+                this.cache.delete(oldestKey);
+            }
+        }
+        this.cache.set(key, { embedding, timestamp, hits: 0 });
+    }
+    /**
+     * Compute cosine similarity between two embeddings
+     */
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length) {
+            throw new Error('Embeddings must have same dimensions');
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        const denom = Math.sqrt(normA) * Math.sqrt(normB);
+        return denom === 0 ? 0 : dotProduct / denom;
+    }
+    /**
+     * Find most similar texts from a corpus
+     */
+    async findSimilar(query, corpus, k = 5, provider) {
+        const [queryEmbed, ...corpusEmbeds] = await this.embed([query, ...corpus], provider);
+        const results = corpusEmbeds.map((embed, i) => ({
+            text: corpus[i],
+            similarity: this.cosineSimilarity(queryEmbed, embed),
+            index: i,
+        }));
+        return results
+            .sort((a, b) => b.similarity - a.similarity)
+            .slice(0, k);
+    }
+    /**
+     * Get cache statistics
+     */
+    getCacheStats() {
+        let totalHits = 0;
+        for (const entry of this.cache.values()) {
+            totalHits += entry.hits;
+        }
+        return {
+            size: this.cache.size,
+            maxSize: this.config.maxCacheSize,
+            hitRate: this.cache.size > 0 ? totalHits / this.cache.size : 0,
+        };
+    }
+    /**
+     * Clear the cache
+     */
+    clearCache() {
+        this.cache.clear();
+    }
+    /**
+     * Get embedding dimensions for a provider
+     */
+    getDimensions(provider) {
+        return this.getProvider(provider).getDimensions();
+    }
+    /**
+     * List available providers
+     */
+    listProviders() {
+        return Array.from(this.providers.keys());
+    }
+}
+exports.EmbeddingService = EmbeddingService;
+/**
+ * Create an embedding service instance
+ */
+function createEmbeddingService(config) {
+    return new EmbeddingService(config);
+}
+// Singleton instance
+let defaultService = null;
+/**
+ * Get the default embedding service instance
+ */
+function getDefaultEmbeddingService() {
+    if (!defaultService) {
+        defaultService = new EmbeddingService();
+    }
+    return defaultService;
+}
+exports.default = {
+    EmbeddingService,
+    LocalNGramProvider,
+    MockEmbeddingProvider,
+    createEmbeddingService,
+    getDefaultEmbeddingService,
+};

package/dist/services/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Services module exports
+ */
+export * from './embedding-service';
+export { default as embeddingService } from './embedding-service';
+//# sourceMappingURL=index.d.ts.map

package/dist/services/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/services/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,qBAAqB,CAAC;AACpC,OAAO,EAAE,OAAO,IAAI,gBAAgB,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/services/index.js

@@ -0,0 +1,26 @@
+"use strict";
+/**
+ * Services module exports
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.embeddingService = void 0;
+__exportStar(require("./embedding-service"), exports);
+var embedding_service_1 = require("./embedding-service");
+Object.defineProperty(exports, "embeddingService", { enumerable: true, get: function () { return __importDefault(embedding_service_1).default; } });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ruvector",
-  "version": "0.1.26",
+  "version": "0.1.27",
   "description": "High-performance vector database for Node.js with automatic native/WASM fallback",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -33,7 +33,12 @@
     "attention",
     "transformer",
     "flash-attention",
-    "hyperbolic"
+    "hyperbolic",
+    "sona",
+    "lora",
+    "ewc",
+    "adaptive-learning",
+    "continual-learning"
   ],
   "author": "ruv.io Team <info@ruv.io> (https://ruv.io)",
   "homepage": "https://ruv.io",
@@ -54,7 +59,8 @@
     "ora": "^5.4.1"
   },
   "optionalDependencies": {
-    "@ruvector/attention": "^0.1.1"
+    "@ruvector/attention": "^0.1.1",
+    "@ruvector/sona": "^0.1.3"
   },
   "devDependencies": {
     "@types/node": "^20.10.5",