simile-search 0.3.1 → 0.4.0

package/dist/cache.js

@@ -0,0 +1,179 @@
+/**
+ * Vector Cache - LRU cache for embedding vectors with text hashing.
+ * Avoids re-embedding duplicate or previously seen texts.
+ */
+/**
+ * MurmurHash3 - Fast, collision-resistant hash function.
+ * Used for creating cache keys from text content.
+ */
+export function murmurHash3(str, seed = 0) {
+    let h1 = seed >>> 0;
+    const c1 = 0xcc9e2d51;
+    const c2 = 0x1b873593;
+    for (let i = 0; i < str.length; i++) {
+        let k1 = str.charCodeAt(i);
+        k1 = Math.imul(k1, c1);
+        k1 = (k1 << 15) | (k1 >>> 17);
+        k1 = Math.imul(k1, c2);
+        h1 ^= k1;
+        h1 = (h1 << 13) | (h1 >>> 19);
+        h1 = Math.imul(h1, 5) + 0xe6546b64;
+    }
+    h1 ^= str.length;
+    h1 ^= h1 >>> 16;
+    h1 = Math.imul(h1, 0x85ebca6b);
+    h1 ^= h1 >>> 13;
+    h1 = Math.imul(h1, 0xc2b2ae35);
+    h1 ^= h1 >>> 16;
+    return (h1 >>> 0).toString(16).padStart(8, '0');
+}
+/**
+ * Create a cache key from text content.
+ * Uses double hashing for better collision resistance.
+ */
+export function createCacheKey(text, model) {
+    const textHash = murmurHash3(text, 0);
+    const modelHash = murmurHash3(model, 1);
+    return `${textHash}-${modelHash}`;
+}
+/**
+ * LRU (Least Recently Used) Vector Cache.
+ * Provides O(1) get/set operations with automatic eviction.
+ */
+export class VectorCache {
+    constructor(options = {}) {
+        this.hits = 0;
+        this.misses = 0;
+        this.maxSize = options.maxSize ?? 10000;
+        this.enableStats = options.enableStats ?? false;
+        this.cache = new Map();
+    }
+    /**
+     * Get a cached vector by text content.
+     * Returns undefined if not in cache.
+     */
+    get(key) {
+        const vector = this.cache.get(key);
+        if (vector !== undefined) {
+            // Move to end for LRU (delete and re-add)
+            this.cache.delete(key);
+            this.cache.set(key, vector);
+            if (this.enableStats)
+                this.hits++;
+            return vector;
+        }
+        if (this.enableStats)
+            this.misses++;
+        return undefined;
+    }
+    /**
+     * Cache a vector for a text content.
+     */
+    set(key, vector) {
+        // If key exists, delete first to update LRU order
+        if (this.cache.has(key)) {
+            this.cache.delete(key);
+        }
+        // Evict oldest entries if at capacity
+        while (this.cache.size >= this.maxSize) {
+            const oldestKey = this.cache.keys().next().value;
+            if (oldestKey !== undefined) {
+                this.cache.delete(oldestKey);
+            }
+        }
+        this.cache.set(key, vector);
+    }
+    /**
+     * Check if a key exists in cache.
+     */
+    has(key) {
+        return this.cache.has(key);
+    }
+    /**
+     * Clear all cached entries.
+     */
+    clear() {
+        this.cache.clear();
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /**
+     * Get current cache size.
+     */
+    get size() {
+        return this.cache.size;
+    }
+    /**
+     * Get cache statistics.
+     */
+    getStats() {
+        const total = this.hits + this.misses;
+        return {
+            hits: this.hits,
+            misses: this.misses,
+            hitRate: total > 0 ? this.hits / total : 0,
+            size: this.cache.size,
+        };
+    }
+    /**
+     * Reset statistics counters.
+     */
+    resetStats() {
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /**
+     * Serialize cache for persistence.
+     */
+    serialize() {
+        const entries = [];
+        for (const [key, vector] of this.cache) {
+            const buffer = Buffer.from(vector.buffer);
+            entries.push([key, buffer.toString('base64')]);
+        }
+        return {
+            entries,
+            maxSize: this.maxSize,
+        };
+    }
+    /**
+     * Deserialize and restore cache from saved state.
+     */
+    static deserialize(data, options = {}) {
+        const cache = new VectorCache({
+            maxSize: data.maxSize,
+            ...options,
+        });
+        for (const [key, base64] of data.entries) {
+            const buffer = Buffer.from(base64, 'base64');
+            const vector = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / 4);
+            cache.cache.set(key, vector);
+        }
+        return cache;
+    }
+    /**
+     * Pre-warm cache with existing vectors.
+     */
+    warmup(entries) {
+        for (const { key, vector } of entries) {
+            this.set(key, vector);
+        }
+    }
+    /**
+     * Get all keys currently in cache.
+     */
+    keys() {
+        return Array.from(this.cache.keys());
+    }
+    /**
+     * Estimate memory usage in bytes.
+     */
+    getMemoryUsage() {
+        let bytes = 0;
+        for (const [key, vector] of this.cache) {
+            bytes += key.length * 2; // UTF-16 string
+            bytes += vector.byteLength;
+        }
+        return bytes;
+    }
+}

package/dist/embedder.d.ts

@@ -1,10 +1,61 @@
+import { VectorCache, CacheOptions } from "./cache.js";
+export interface BatchConfig {
+    /** Maximum items per batch (default: 32) */
+    maxBatchSize?: number;
+    /** Maximum estimated tokens per batch (default: 8000) */
+    maxTokensPerBatch?: number;
+    /** Enable adaptive batch sizing based on text length (default: true) */
+    adaptive?: boolean;
+    /** Use cache for embeddings (default: true if cache exists) */
+    useCache?: boolean;
+    /** Progress callback for long operations */
+    onProgress?: (processed: number, total: number) => void;
+}
+/**
+ * Set the global vector cache instance.
+ */
+export declare function setGlobalCache(cache: VectorCache | null): void;
+/**
+ * Get the global vector cache instance.
+ */
+export declare function getGlobalCache(): VectorCache | null;
+/**
+ * Create a new cache and set it as global.
+ */
+export declare function createCache(options?: CacheOptions): VectorCache;
 export declare function getEmbedder(model?: string): Promise<any>;
-export declare function embed(text: string, model?: string): Promise<Float32Array>;
 /**
- * Batch embed multiple texts at once for better performance.
- * This is significantly faster than embedding one by one.
+ * Estimate token count for a text (rough approximation).
+ * Uses ~4 characters per token heuristic.
+ */
+export declare function estimateTokens(text: string): number;
+export declare function embed(text: string, model?: string, useCache?: boolean): Promise<Float32Array>;
+/**
+ * Batch embed multiple texts with dynamic batching and caching.
+ *
+ * Features:
+ * - Adaptive batch sizing based on text length
+ * - Cache integration to skip already-embedded texts
+ * - Progress callback for long operations
+ */
+export declare function embedBatch(texts: string[], model?: string, config?: BatchConfig): Promise<Float32Array[]>;
+/**
+ * Pre-warm the cache with known text-vector pairs.
+ * Useful when loading from a snapshot.
+ */
+export declare function warmupCache(entries: Array<{
+    text: string;
+    vector: Float32Array;
+    model?: string;
+}>): void;
+/**
+ * Clear the global cache.
+ */
+export declare function clearCache(): void;
+/**
+ * Get cache statistics.
  */
-export declare function embedBatch(texts: string[], model?: string): Promise<Float32Array[]>;
+export declare function getCacheStats(): import("./cache.js").CacheStats;
 /** Serialize Float32Array to base64 string for storage */
 export declare function vectorToBase64(vector: Float32Array): string;
 /** Deserialize base64 string back to Float32Array */

package/dist/embedder.js

@@ -1,6 +1,28 @@
 import { pipeline } from "@xenova/transformers";
+import { VectorCache, createCacheKey } from "./cache.js";
 let extractor;
 let currentModel = "";
+// Global cache instance (can be replaced via setGlobalCache)
+let globalCache = null;
+/**
+ * Set the global vector cache instance.
+ */
+export function setGlobalCache(cache) {
+    globalCache = cache;
+}
+/**
+ * Get the global vector cache instance.
+ */
+export function getGlobalCache() {
+    return globalCache;
+}
+/**
+ * Create a new cache and set it as global.
+ */
+export function createCache(options = {}) {
+    globalCache = new VectorCache(options);
+    return globalCache;
+}
 export async function getEmbedder(model = "Xenova/all-MiniLM-L6-v2") {
     if (!extractor || currentModel !== model) {
         extractor = await pipeline("feature-extraction", model);
@@ -8,30 +30,140 @@ export async function getEmbedder(model = "Xenova/all-MiniLM-L6-v2") {
     }
     return extractor;
 }
-export async function embed(text, model) {
+/**
+ * Estimate token count for a text (rough approximation).
+ * Uses ~4 characters per token heuristic.
+ */
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+export async function embed(text, model, useCache = true) {
+    const actualModel = model ?? "Xenova/all-MiniLM-L6-v2";
+    // Check cache first
+    if (useCache && globalCache) {
+        const cacheKey = createCacheKey(text, actualModel);
+        const cached = globalCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+    }
     const embedder = await getEmbedder(model);
     const output = await embedder(text, {
         pooling: "mean",
         normalize: true,
     });
-    return output.data;
+    const vector = output.data;
+    // Store in cache
+    if (useCache && globalCache) {
+        const cacheKey = createCacheKey(text, actualModel);
+        globalCache.set(cacheKey, vector);
+    }
+    return vector;
 }
 /**
- * Batch embed multiple texts at once for better performance.
- * This is significantly faster than embedding one by one.
+ * Batch embed multiple texts with dynamic batching and caching.
+ *
+ * Features:
+ * - Adaptive batch sizing based on text length
+ * - Cache integration to skip already-embedded texts
+ * - Progress callback for long operations
  */
-export async function embedBatch(texts, model) {
+export async function embedBatch(texts, model, config = {}) {
+    const { maxBatchSize = 32, maxTokensPerBatch = 8000, adaptive = true, useCache = true, onProgress, } = config;
+    const actualModel = model ?? "Xenova/all-MiniLM-L6-v2";
     const embedder = await getEmbedder(model);
-    const results = [];
-    // Process in batches of 32 for memory efficiency
-    const BATCH_SIZE = 32;
-    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
-        const batch = texts.slice(i, i + BATCH_SIZE);
-        const outputs = await Promise.all(batch.map((text) => embedder(text, { pooling: "mean", normalize: true })));
-        results.push(...outputs.map((o) => o.data));
+    const results = new Array(texts.length).fill(null);
+    // First pass: check cache and collect uncached texts
+    const uncachedIndices = [];
+    if (useCache && globalCache) {
+        for (let i = 0; i < texts.length; i++) {
+            const cacheKey = createCacheKey(texts[i], actualModel);
+            const cached = globalCache.get(cacheKey);
+            if (cached) {
+                results[i] = cached;
+            }
+            else {
+                uncachedIndices.push(i);
+            }
+        }
+    }
+    else {
+        for (let i = 0; i < texts.length; i++) {
+            uncachedIndices.push(i);
+        }
+    }
+    // Report initial progress (cached items)
+    const cachedCount = texts.length - uncachedIndices.length;
+    if (onProgress && cachedCount > 0) {
+        onProgress(cachedCount, texts.length);
+    }
+    // Second pass: embed uncached texts with dynamic batching
+    let processed = cachedCount;
+    let batchStart = 0;
+    while (batchStart < uncachedIndices.length) {
+        let batchEnd = batchStart;
+        let batchTokens = 0;
+        // Build batch with adaptive sizing
+        while (batchEnd < uncachedIndices.length) {
+            const idx = uncachedIndices[batchEnd];
+            const textTokens = adaptive ? estimateTokens(texts[idx]) : 0;
+            // Check if adding this text would exceed limits
+            if (batchEnd > batchStart) {
+                if (batchEnd - batchStart >= maxBatchSize)
+                    break;
+                if (adaptive && batchTokens + textTokens > maxTokensPerBatch)
+                    break;
+            }
+            batchTokens += textTokens;
+            batchEnd++;
+        }
+        // Process batch
+        const batchIndices = uncachedIndices.slice(batchStart, batchEnd);
+        const batchTexts = batchIndices.map(i => texts[i]);
+        const outputs = await Promise.all(batchTexts.map((text) => embedder(text, { pooling: "mean", normalize: true })));
+        // Store results and update cache
+        for (let i = 0; i < batchIndices.length; i++) {
+            const originalIdx = batchIndices[i];
+            const vector = outputs[i].data;
+            results[originalIdx] = vector;
+            if (useCache && globalCache) {
+                const cacheKey = createCacheKey(texts[originalIdx], actualModel);
+                globalCache.set(cacheKey, vector);
+            }
+        }
+        processed += batchIndices.length;
+        if (onProgress) {
+            onProgress(processed, texts.length);
+        }
+        batchStart = batchEnd;
     }
     return results;
 }
+/**
+ * Pre-warm the cache with known text-vector pairs.
+ * Useful when loading from a snapshot.
+ */
+export function warmupCache(entries) {
+    if (!globalCache) {
+        globalCache = new VectorCache();
+    }
+    for (const { text, vector, model } of entries) {
+        const cacheKey = createCacheKey(text, model ?? "Xenova/all-MiniLM-L6-v2");
+        globalCache.set(cacheKey, vector);
+    }
+}
+/**
+ * Clear the global cache.
+ */
+export function clearCache() {
+    globalCache?.clear();
+}
+/**
+ * Get cache statistics.
+ */
+export function getCacheStats() {
+    return globalCache?.getStats() ?? { hits: 0, misses: 0, hitRate: 0, size: 0 };
+}
 /** Serialize Float32Array to base64 string for storage */
 export function vectorToBase64(vector) {
     const buffer = Buffer.from(vector.buffer);

package/dist/engine.d.ts

@@ -1,10 +1,14 @@
-import { SearchItem, SearchResult, SearchOptions, SimileConfig, SimileSnapshot, HybridWeights } from "./types.js";
+import { SearchItem, SearchResult, SearchOptions, SimileConfig, SimileSnapshot, HybridWeights, IndexInfo } from "./types.js";
 export declare class Simile<T = any> {
     private items;
     private vectors;
     private itemIndex;
     private config;
+    private cache;
+    private annIndex;
+    private updater;
     private constructor();
+    private buildANNIndex;
     /**
      * Extract searchable text from an item using configured paths.
      */
@@ -14,6 +18,10 @@ export declare class Simile<T = any> {
      * This will embed all items (slow for first run, but cached after).
      */
     static from<T>(items: SearchItem<T>[], config?: SimileConfig): Promise<Simile<T>>;
+    /**
+     * Internal helper for embedding text with caching.
+     */
+    private embedWithCache;
     /**
      * Load a Simile instance from a previously saved snapshot.
      * This is INSTANT - no embedding needed!
@@ -32,10 +40,15 @@ export declare class Simile<T = any> {
      * Export as JSON string for file storage
      */
     toJSON(): string;
+    add(items: SearchItem<T>[]): Promise<void>;
     /**
-     * Add new items to the index
+     * Queue items for background indexing (non-blocking).
      */
-    add(items: SearchItem<T>[]): Promise<void>;
+    enqueue(items: SearchItem<T>[]): void;
+    /**
+     * Get indexing information and stats.
+     */
+    getIndexInfo(): IndexInfo;
     /**
      * Remove items by ID
      */