ruvector 0.1.30 → 0.1.31

package/src/services/embedding-service.ts

@@ -0,0 +1,386 @@
+/**
+ * Embedding Service - Unified embedding generation and management
+ *
+ * This service provides a unified interface for generating, caching, and
+ * managing embeddings from various sources (local models, APIs, etc.)
+ */
+
+/**
+ * Embedding provider interface
+ */
+export interface EmbeddingProvider {
+  /** Provider name */
+  name: string;
+  /** Generate embeddings for texts */
+  embed(texts: string[]): Promise<number[][]>;
+  /** Get embedding dimensions */
+  getDimensions(): number;
+}
+
+/**
+ * Cached embedding entry
+ */
+interface CacheEntry {
+  embedding: number[];
+  timestamp: number;
+  hits: number;
+}
+
+/**
+ * Embedding service configuration
+ */
+export interface EmbeddingServiceConfig {
+  /** Default provider to use */
+  defaultProvider?: string;
+  /** Maximum cache size */
+  maxCacheSize?: number;
+  /** Cache TTL in milliseconds */
+  cacheTtl?: number;
+  /** Batch size for embedding generation */
+  batchSize?: number;
+}
+
+/**
+ * Simple hash function for cache keys
+ */
+function hashText(text: string): string {
+  let hash = 0;
+  for (let i = 0; i < text.length; i++) {
+    const char = text.charCodeAt(i);
+    hash = ((hash << 5) - hash) + char;
+    hash = hash & hash;
+  }
+  return `h${hash.toString(36)}`;
+}
+
+/**
+ * Mock embedding provider for testing
+ */
+export class MockEmbeddingProvider implements EmbeddingProvider {
+  name = 'mock';
+  private dimensions: number;
+
+  constructor(dimensions: number = 384) {
+    this.dimensions = dimensions;
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    return texts.map(text => {
+      // Generate deterministic pseudo-random embeddings based on text
+      const embedding: number[] = [];
+      let seed = 0;
+      for (let i = 0; i < text.length; i++) {
+        seed = ((seed << 5) - seed + text.charCodeAt(i)) | 0;
+      }
+
+      for (let i = 0; i < this.dimensions; i++) {
+        seed = (seed * 1103515245 + 12345) | 0;
+        embedding.push((seed % 1000) / 1000 - 0.5);
+      }
+
+      // Normalize
+      const norm = Math.sqrt(embedding.reduce((s, v) => s + v * v, 0));
+      return embedding.map(v => v / (norm || 1));
+    });
+  }
+
+  getDimensions(): number {
+    return this.dimensions;
+  }
+}
+
+/**
+ * Simple local embedding using character n-grams
+ * This is a fallback when no external provider is available
+ */
+export class LocalNGramProvider implements EmbeddingProvider {
+  name = 'local-ngram';
+  private dimensions: number;
+  private ngramSize: number;
+
+  constructor(dimensions: number = 256, ngramSize: number = 3) {
+    this.dimensions = dimensions;
+    this.ngramSize = ngramSize;
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    return texts.map(text => this.embedSingle(text));
+  }
+
+  private embedSingle(text: string): number[] {
+    const embedding = new Array(this.dimensions).fill(0);
+    const normalized = text.toLowerCase().replace(/[^a-z0-9]/g, ' ');
+
+    // Generate n-grams and hash them into embedding dimensions
+    for (let i = 0; i <= normalized.length - this.ngramSize; i++) {
+      const ngram = normalized.slice(i, i + this.ngramSize);
+      const hash = this.hashNgram(ngram);
+      const idx = Math.abs(hash) % this.dimensions;
+      embedding[idx] += hash > 0 ? 1 : -1;
+    }
+
+    // Normalize
+    const norm = Math.sqrt(embedding.reduce((s, v) => s + v * v, 0));
+    return embedding.map(v => v / (norm || 1));
+  }
+
+  private hashNgram(ngram: string): number {
+    let hash = 0;
+    for (let i = 0; i < ngram.length; i++) {
+      hash = ((hash << 5) - hash + ngram.charCodeAt(i)) | 0;
+    }
+    return hash;
+  }
+
+  getDimensions(): number {
+    return this.dimensions;
+  }
+}
+
+/**
+ * Embedding service with caching and batching
+ */
+export class EmbeddingService {
+  private providers: Map<string, EmbeddingProvider> = new Map();
+  private cache: Map<string, CacheEntry> = new Map();
+  private config: Required<EmbeddingServiceConfig>;
+
+  constructor(config: EmbeddingServiceConfig = {}) {
+    this.config = {
+      defaultProvider: config.defaultProvider ?? 'local-ngram',
+      maxCacheSize: config.maxCacheSize ?? 10000,
+      cacheTtl: config.cacheTtl ?? 3600000, // 1 hour
+      batchSize: config.batchSize ?? 32,
+    };
+
+    // Register default providers
+    this.registerProvider(new LocalNGramProvider());
+    this.registerProvider(new MockEmbeddingProvider());
+  }
+
+  /**
+   * Register an embedding provider
+   */
+  registerProvider(provider: EmbeddingProvider): void {
+    this.providers.set(provider.name, provider);
+  }
+
+  /**
+   * Get a registered provider
+   */
+  getProvider(name?: string): EmbeddingProvider {
+    const providerName = name ?? this.config.defaultProvider;
+    const provider = this.providers.get(providerName);
+    if (!provider) {
+      throw new Error(`Provider not found: ${providerName}`);
+    }
+    return provider;
+  }
+
+  /**
+   * Generate embeddings for texts with caching
+   *
+   * @param texts - Texts to embed
+   * @param provider - Provider name (uses default if not specified)
+   * @returns Array of embeddings
+   */
+  async embed(texts: string[], provider?: string): Promise<number[][]> {
+    const providerInstance = this.getProvider(provider);
+    const providerName = providerInstance.name;
+    const now = Date.now();
+
+    // Check cache and collect texts that need embedding
+    const results: (number[] | null)[] = new Array(texts.length).fill(null);
+    const uncachedIndices: number[] = [];
+    const uncachedTexts: string[] = [];
+
+    for (let i = 0; i < texts.length; i++) {
+      const cacheKey = `${providerName}:${hashText(texts[i])}`;
+      const cached = this.cache.get(cacheKey);
+
+      if (cached && now - cached.timestamp < this.config.cacheTtl) {
+        results[i] = cached.embedding;
+        cached.hits++;
+      } else {
+        uncachedIndices.push(i);
+        uncachedTexts.push(texts[i]);
+      }
+    }
+
+    // Generate embeddings for uncached texts in batches
+    if (uncachedTexts.length > 0) {
+      const batches: string[][] = [];
+      for (let i = 0; i < uncachedTexts.length; i += this.config.batchSize) {
+        batches.push(uncachedTexts.slice(i, i + this.config.batchSize));
+      }
+
+      let batchOffset = 0;
+      for (const batch of batches) {
+        const embeddings = await providerInstance.embed(batch);
+
+        for (let j = 0; j < embeddings.length; j++) {
+          const originalIndex = uncachedIndices[batchOffset + j];
+          results[originalIndex] = embeddings[j];
+
+          // Cache the result
+          const cacheKey = `${providerName}:${hashText(texts[originalIndex])}`;
+          this.addToCache(cacheKey, embeddings[j], now);
+        }
+
+        batchOffset += batch.length;
+      }
+    }
+
+    return results as number[][];
+  }
+
+  /**
+   * Generate a single embedding
+   */
+  async embedOne(text: string, provider?: string): Promise<number[]> {
+    const results = await this.embed([text], provider);
+    return results[0];
+  }
+
+  /**
+   * Add entry to cache with LRU eviction
+   */
+  private addToCache(key: string, embedding: number[], timestamp: number): void {
+    // Evict old entries if cache is full
+    if (this.cache.size >= this.config.maxCacheSize) {
+      // Find and remove least recently used entry
+      let oldestKey = '';
+      let oldestTime = Infinity;
+      let lowestHits = Infinity;
+
+      for (const [k, v] of this.cache.entries()) {
+        if (v.hits < lowestHits || (v.hits === lowestHits && v.timestamp < oldestTime)) {
+          oldestKey = k;
+          oldestTime = v.timestamp;
+          lowestHits = v.hits;
+        }
+      }
+
+      if (oldestKey) {
+        this.cache.delete(oldestKey);
+      }
+    }
+
+    this.cache.set(key, { embedding, timestamp, hits: 0 });
+  }
+
+  /**
+   * Compute cosine similarity between two embeddings
+   */
+  cosineSimilarity(a: number[], b: number[]): number {
+    if (a.length !== b.length) {
+      throw new Error('Embeddings must have same dimensions');
+    }
+
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i] * b[i];
+      normA += a[i] * a[i];
+      normB += b[i] * b[i];
+    }
+
+    const denom = Math.sqrt(normA) * Math.sqrt(normB);
+    return denom === 0 ? 0 : dotProduct / denom;
+  }
+
+  /**
+   * Find most similar texts from a corpus
+   */
+  async findSimilar(
+    query: string,
+    corpus: string[],
+    k: number = 5,
+    provider?: string
+  ): Promise<{ text: string; similarity: number; index: number }[]> {
+    const [queryEmbed, ...corpusEmbeds] = await this.embed([query, ...corpus], provider);
+
+    const results = corpusEmbeds.map((embed, i) => ({
+      text: corpus[i],
+      similarity: this.cosineSimilarity(queryEmbed, embed),
+      index: i,
+    }));
+
+    return results
+      .sort((a, b) => b.similarity - a.similarity)
+      .slice(0, k);
+  }
+
+  /**
+   * Get cache statistics
+   */
+  getCacheStats(): {
+    size: number;
+    maxSize: number;
+    hitRate: number;
+  } {
+    let totalHits = 0;
+    for (const entry of this.cache.values()) {
+      totalHits += entry.hits;
+    }
+
+    return {
+      size: this.cache.size,
+      maxSize: this.config.maxCacheSize,
+      hitRate: this.cache.size > 0 ? totalHits / this.cache.size : 0,
+    };
+  }
+
+  /**
+   * Clear the cache
+   */
+  clearCache(): void {
+    this.cache.clear();
+  }
+
+  /**
+   * Get embedding dimensions for a provider
+   */
+  getDimensions(provider?: string): number {
+    return this.getProvider(provider).getDimensions();
+  }
+
+  /**
+   * List available providers
+   */
+  listProviders(): string[] {
+    return Array.from(this.providers.keys());
+  }
+}
+
+/**
+ * Create an embedding service instance
+ */
+export function createEmbeddingService(
+  config?: EmbeddingServiceConfig
+): EmbeddingService {
+  return new EmbeddingService(config);
+}
+
+// Singleton instance
+let defaultService: EmbeddingService | null = null;
+
+/**
+ * Get the default embedding service instance
+ */
+export function getDefaultEmbeddingService(): EmbeddingService {
+  if (!defaultService) {
+    defaultService = new EmbeddingService();
+  }
+  return defaultService;
+}
+
+export default {
+  EmbeddingService,
+  LocalNGramProvider,
+  MockEmbeddingProvider,
+  createEmbeddingService,
+  getDefaultEmbeddingService,
+};

package/src/services/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Services module exports
+ */
+
+export * from './embedding-service';
+export { default as embeddingService } from './embedding-service';

package/src/types.ts

@@ -0,0 +1,161 @@
+/**
+ * Vector entry representing a document with its embedding
+ */
+export interface VectorEntry {
+  /** Unique identifier for the vector */
+  id: string;
+  /** Vector embedding (array of floats) */
+  vector: number[];
+  /** Optional metadata associated with the vector */
+  metadata?: Record<string, any>;
+}
+
+/**
+ * Search query parameters
+ */
+export interface SearchQuery {
+  /** Query vector to search for */
+  vector: number[];
+  /** Number of results to return */
+  k?: number;
+  /** Optional metadata filters */
+  filter?: Record<string, any>;
+  /** Minimum similarity threshold (0-1) */
+  threshold?: number;
+}
+
+/**
+ * Search result containing matched vector and similarity score
+ */
+export interface SearchResult {
+  /** ID of the matched vector */
+  id: string;
+  /** Similarity score (0-1, higher is better) */
+  score: number;
+  /** Vector data */
+  vector: number[];
+  /** Associated metadata */
+  metadata?: Record<string, any>;
+}
+
+/**
+ * Database configuration options
+ */
+export interface DbOptions {
+  /** Vector dimension size */
+  dimension: number;
+  /** Distance metric to use */
+  metric?: 'cosine' | 'euclidean' | 'dot';
+  /** Path to persist database */
+  path?: string;
+  /** Enable auto-persistence */
+  autoPersist?: boolean;
+  /** HNSW index parameters */
+  hnsw?: {
+    /** Maximum number of connections per layer */
+    m?: number;
+    /** Size of the dynamic candidate list */
+    efConstruction?: number;
+    /** Size of the dynamic candidate list for search */
+    efSearch?: number;
+  };
+}
+
+/**
+ * Database statistics
+ */
+export interface DbStats {
+  /** Total number of vectors */
+  count: number;
+  /** Vector dimension */
+  dimension: number;
+  /** Distance metric */
+  metric: string;
+  /** Memory usage in bytes */
+  memoryUsage?: number;
+  /** Index type */
+  indexType?: string;
+}
+
+/**
+ * Main VectorDB class interface
+ */
+export interface VectorDB {
+  /**
+   * Create a new vector database
+   * @param options Database configuration
+   */
+  new(options: DbOptions): VectorDB;
+
+  /**
+   * Insert a single vector
+   * @param entry Vector entry to insert
+   */
+  insert(entry: VectorEntry): void;
+
+  /**
+   * Insert multiple vectors in batch
+   * @param entries Array of vector entries
+   */
+  insertBatch(entries: VectorEntry[]): void;
+
+  /**
+   * Search for similar vectors
+   * @param query Search query parameters
+   * @returns Array of search results
+   */
+  search(query: SearchQuery): SearchResult[];
+
+  /**
+   * Get vector by ID
+   * @param id Vector ID
+   * @returns Vector entry or null
+   */
+  get(id: string): VectorEntry | null;
+
+  /**
+   * Delete vector by ID
+   * @param id Vector ID
+   * @returns true if deleted, false if not found
+   */
+  delete(id: string): boolean;
+
+  /**
+   * Update vector metadata
+   * @param id Vector ID
+   * @param metadata New metadata
+   */
+  updateMetadata(id: string, metadata: Record<string, any>): void;
+
+  /**
+   * Get database statistics
+   */
+  stats(): DbStats;
+
+  /**
+   * Save database to disk
+   * @param path Optional path (uses configured path if not provided)
+   */
+  save(path?: string): void;
+
+  /**
+   * Load database from disk
+   * @param path Path to database file
+   */
+  load(path: string): void;
+
+  /**
+   * Clear all vectors from database
+   */
+  clear(): void;
+
+  /**
+   * Build HNSW index for faster search
+   */
+  buildIndex(): void;
+
+  /**
+   * Optimize database (rebuild indices, compact storage)
+   */
+  optimize(): void;
+}