squish-memory 1.1.5 → 1.2.0

package/dist/core/embeddings/transformers-local.js

@@ -0,0 +1,213 @@
+/**
+ * Transformers.js Local Embedding Provider
+ *
+ * Uses ONNX-based transformer models for high-quality local embeddings.
+ * Supports Hugging Face ONNX embedding models.
+ *
+ * Usage:
+ *   Set SQUISH_EMBEDDINGS_PROVIDER=transformers
+ *   Required: SQUISH_LOCAL_MODEL=<huggingface-onnx-model>
+ *
+ * Download models automatically on first use. Models cached in HuggingFace cache directory.
+ */
+import { pipeline } from '@huggingface/transformers';
+import { logger } from '../logger.js';
+import { config } from '../../config.js';
+const DEFAULT_CONFIG = {
+    model: '',
+    device: 'cpu',
+    dtype: 'q8', // Quantized for smaller size + faster loading
+};
+// Singleton pipeline instance
+let embeddingPipeline = null;
+let isLoading = false;
+let loadPromise = null;
+/**
+ * Get or initialize the embedding pipeline (lazy loading)
+ */
+async function getPipeline() {
+    // Return existing pipeline
+    if (embeddingPipeline) {
+        return embeddingPipeline;
+    }
+    // Already loading - wait for it
+    if (isLoading && loadPromise) {
+        return loadPromise;
+    }
+    // Start loading
+    isLoading = true;
+    const model = config.transformersLocalModel;
+    if (!model) {
+        isLoading = false;
+        throw new Error('Transformers provider requires SQUISH_LOCAL_MODEL to be set');
+    }
+    logger.info(`Loading transformers local model: ${model}`);
+    loadPromise = (async () => {
+        try {
+            embeddingPipeline = await pipeline('feature-extraction', model, {
+                device: DEFAULT_CONFIG.device,
+                dtype: DEFAULT_CONFIG.dtype,
+            });
+            logger.info(`Transformers local model loaded: ${model}`);
+            return embeddingPipeline;
+        }
+        catch (error) {
+            logger.error(`Failed to load transformers model: ${error}`);
+            embeddingPipeline = null;
+            throw error;
+        }
+        finally {
+            isLoading = false;
+            loadPromise = null;
+        }
+    })();
+    return loadPromise;
+}
+/**
+ * Check if pipeline is loaded
+ */
+export function isReady() {
+    return embeddingPipeline !== null;
+}
+/**
+ * Get embedding dimension for current model
+ */
+export function getEmbeddingDimension() {
+    return 0;
+}
+/**
+ * Generate embedding for a single text input
+ * Uses mean pooling + L2 normalization
+ */
+export async function getEmbedding(text) {
+    if (!text || typeof text !== 'string') {
+        return null;
+    }
+    try {
+        const pipeline = await getPipeline();
+        if (!pipeline) {
+            return null;
+        }
+        // Truncate to max sequence length (256 tokens)
+        const truncatedText = text.slice(0, 512);
+        // Run feature extraction with pooling
+        const output = await pipeline(truncatedText, {
+            pooling: 'mean',
+            normalize: true,
+        });
+        // Convert tensor to array
+        const embedding = Array.from(output.data);
+        return embedding;
+    }
+    catch (error) {
+        logger.warn(`Transformers embedding error: ${error}`);
+        return null;
+    }
+}
+/**
+ * Generate embeddings for multiple texts in batch
+ * Processes efficiently with batching
+ */
+export async function getBatchEmbeddings(texts, batchSize = 8) {
+    if (texts.length === 0) {
+        return [];
+    }
+    const results = new Array(texts.length).fill(null);
+    // Process in batches
+    for (let i = 0; i < texts.length; i += batchSize) {
+        const batchEnd = Math.min(i + batchSize, texts.length);
+        const batch = texts.slice(i, batchEnd);
+        const indices = [...Array(batch.length).keys()].map(j => i + j);
+        try {
+            const pipeline = await getPipeline();
+            if (!pipeline) {
+                continue;
+            }
+            // Truncate each text
+            const truncatedBatch = batch.map(t => t.slice(0, 512));
+            // Process batch
+            const outputs = await Promise.all(truncatedBatch.map(text => pipeline(text, { pooling: 'mean', normalize: true })));
+            // Store results
+            for (let j = 0; j < outputs.length; j++) {
+                results[indices[j]] = Array.from(outputs[j].data);
+            }
+        }
+        catch (error) {
+            logger.warn(`Transformers batch embedding error: ${error}`);
+        }
+    }
+    return results;
+}
+/**
+ * Check health of the transformers provider
+ */
+export async function checkHealth() {
+    const model = config.transformersLocalModel;
+    if (!model) {
+        return {
+            available: false,
+            error: 'SQUISH_LOCAL_MODEL is not configured',
+        };
+    }
+    // Check if library is available
+    try {
+        await import('@huggingface/transformers');
+    }
+    catch (error) {
+        return {
+            available: false,
+            error: '@huggingface/transformers not installed',
+        };
+    }
+    // Try quick embedding
+    const start = Date.now();
+    try {
+        const embedding = await getEmbedding('health check test');
+        const latency = Date.now() - start;
+        return {
+            available: embedding !== null && embedding.length > 0,
+            latencyMs: latency,
+            model,
+            dimension: embedding?.length || 0,
+        };
+    }
+    catch (error) {
+        return {
+            available: false,
+            error: error.message,
+            model,
+        };
+    }
+}
+/**
+ * Unload the pipeline (for testing or memory management)
+ */
+export async function unload() {
+    if (embeddingPipeline) {
+        // Pipeline doesn't have explicit cleanup, just release reference
+        embeddingPipeline = null;
+        logger.info('Transformers pipeline unloaded');
+    }
+}
+/**
+ * Warm up the model with a test input
+ */
+export async function warmup() {
+    try {
+        const result = await getEmbedding('warmup test');
+        return result !== null && result.length > 0;
+    }
+    catch {
+        return false;
+    }
+}
+export default {
+    isReady,
+    getEmbeddingDimension,
+    getEmbedding,
+    getBatchEmbeddings,
+    checkHealth,
+    unload,
+    warmup,
+};
+//# sourceMappingURL=transformers-local.js.map

package/dist/core/embeddings.d.ts

@@ -1,29 +1,2 @@
-import { MultimodalInput } from './embeddings/google-multimodal.js';
-export type EmbeddingProvider = 'local' | 'openai' | 'ollama' | 'lmstudio' | 'google' | 'none' | 'auto';
-export declare function getEmbedding(input: string | MultimodalInput): Promise<number[] | null>;
-/**
- * Get embeddings for multiple inputs in parallel batches
- * Processes inputs in batches to respect rate limits while parallelizing
- */
-export declare function getBatchEmbeddings(inputs: string[], batchSize?: number): Promise<Array<number[] | null>>;
-/**
- * Clear the embedding cache
- */
-export declare function clearEmbeddingCache(): void;
-/**
- * Get embedding cache statistics
- */
-export declare function getEmbeddingCacheStats(): {
-    size: number;
-    maxSize: number;
-};
-/**
- * Check health of all configured embedding providers
- * Returns availability and latency for each provider
- */
-export declare function checkEmbeddingProviderHealth(): Promise<Map<string, {
-    available: boolean;
-    latencyMs?: number;
-    error?: string;
-}>>;
+export * from './embeddings/embeddings';
 //# sourceMappingURL=embeddings.d.ts.map

package/dist/core/embeddings.js

@@ -1,454 +1,3 @@
-import { config } from '../config.js';
-import { getGoogleMultimodalEmbedding, isMultimodalInput } from './embeddings/google-multimodal.js';
-import { logger } from './logger.js';
-// Retry utility with exponential backoff
-async function withRetry(fn, maxRetries = config.embeddingsMaxRetries, baseDelayMs = config.embeddingsRetryDelayMs) {
-    let lastError;
-    for (let attempt = 0; attempt < maxRetries; attempt++) {
-        try {
-            return await fn();
-        }
-        catch (error) {
-            lastError = error;
-            // Only retry on network errors (5xx, ECONNRESET, ETIMEDOUT, etc.)
-            if (error instanceof Error && shouldRetryError(error)) {
-                const delay = baseDelayMs * Math.pow(2, attempt) + Math.random() * baseDelayMs;
-                logger.debug(`Embedding request failed (attempt ${attempt + 1}/${maxRetries}), retrying in ${delay.toFixed(0)}ms`, { error: error });
-                await new Promise(resolve => setTimeout(resolve, delay));
-                continue;
-            }
-            // Don't retry on 4xx errors or non-retryable errors
-            break;
-        }
-    }
-    throw lastError;
-}
-function shouldRetryError(error) {
-    const message = error.message.toLowerCase();
-    // Network errors that are typically transient
-    const retryablePatterns = [
-        'econnreset',
-        'etimedout',
-        'econnrefused',
-        'esocket',
-        'network error',
-        'fetch failed',
-        'timeout',
-        'request timed out',
-        'service unavailable',
-        'too many requests',
-        'rate limit',
-        'internal server error',
-        'bad gateway',
-        'gateway timeout',
-    ];
-    return retryablePatterns.some(pattern => message.includes(pattern));
-}
-// Timeout wrapper using AbortController
-async function withTimeout(promise, timeoutMs) {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
-    try {
-        return await promise;
-    }
-    finally {
-        clearTimeout(timeoutId);
-    }
-}
-// Fetch wrapper that combines retry and timeout
-async function fetchWithRetryAndTimeout(url, options, timeoutMs = config.embeddingsTimeoutMs) {
-    return withRetry(async () => {
-        return withTimeout(fetch(url, options), timeoutMs);
-    });
-}
-// Simple in-memory cache for embeddings (LRU with 1000 entries)
-const embeddingCache = new Map();
-const MAX_CACHE_SIZE = 1000;
-function getCacheKey(input, provider) {
-    // Simple hash of input + provider
-    let hash = 0;
-    const str = input + provider;
-    for (let i = 0; i < str.length; i++) {
-        const char = str.charCodeAt(i);
-        hash = ((hash << 5) - hash) + char;
-        hash = hash & hash; // Convert to 32bit integer
-    }
-    return hash.toString();
-}
-function getCachedEmbedding(key) {
-    return embeddingCache.get(key);
-}
-function setCachedEmbedding(key, embedding) {
-    if (embeddingCache.size >= MAX_CACHE_SIZE) {
-        // Remove oldest entry (first one)
-        const firstKey = embeddingCache.keys().next().value;
-        if (firstKey) {
-            embeddingCache.delete(firstKey);
-        }
-    }
-    embeddingCache.set(key, embedding);
-}
-export async function getEmbedding(input) {
-    if (!input || (typeof input !== 'string' && !isMultimodalInput(input))) {
-        return null;
-    }
-    const provider = config.embeddingsProvider;
-    const cacheKey = typeof input === 'string'
-        ? getCacheKey(input, provider)
-        : getCacheKey(JSON.stringify(input), provider);
-    // Check cache first
-    const cached = getCachedEmbedding(cacheKey);
-    if (cached) {
-        return cached;
-    }
-    let result = null;
-    // Handle multimodal input
-    if (isMultimodalInput(input) && provider === 'google') {
-        const multimodalResult = await getGoogleMultimodalEmbedding(input);
-        if (multimodalResult) {
-            result = multimodalResult.embedding;
-        }
-    }
-    // Handle text-only input
-    if (!result && typeof input === 'string') {
-        const textInput = input;
-        if (provider === 'none') {
-            result = null;
-        }
-        else if (provider === 'google') {
-            const multimodalResult = await getGoogleMultimodalEmbedding({ text: textInput });
-            result = multimodalResult?.embedding || null;
-        }
-        else if (provider === 'openai') {
-            result = await getOpenAiEmbedding(textInput);
-        }
-        else if (provider === 'ollama') {
-            result = await getOllamaEmbedding(textInput);
-        }
-        else if (provider === 'lmstudio') {
-            result = await getLmStudioEmbedding(textInput);
-        }
-        else if (provider === 'local') {
-            result = getLocalEmbedding(textInput);
-        }
-        else {
-            // Auto mode: try cloud providers first if configured, then fall back to local
-            if (config.openAiApiKey) {
-                result = await getOpenAiEmbedding(textInput);
-            }
-            if (!result && config.ollamaUrl) {
-                result = await getOllamaEmbedding(textInput);
-            }
-            if (!result && config.lmStudioUrl) {
-                result = await getLmStudioEmbedding(textInput);
-            }
-            if (!result) {
-                result = getLocalEmbedding(textInput);
-            }
-        }
-    }
-    // Cache the result if valid
-    if (result) {
-        setCachedEmbedding(cacheKey, result);
-    }
-    return result;
-}
-/**
- * Get embeddings for multiple inputs in parallel batches
- * Processes inputs in batches to respect rate limits while parallelizing
- */
-export async function getBatchEmbeddings(inputs, batchSize = 20) {
-    if (inputs.length === 0)
-        return [];
-    const results = new Array(inputs.length).fill(null);
-    const provider = config.embeddingsProvider;
-    // Check cache for all inputs first
-    const uncachedIndices = [];
-    const uncachedInputs = [];
-    for (let i = 0; i < inputs.length; i++) {
-        const cacheKey = getCacheKey(inputs[i], provider);
-        const cached = getCachedEmbedding(cacheKey);
-        if (cached) {
-            results[i] = cached;
-        }
-        else {
-            uncachedIndices.push(i);
-            uncachedInputs.push(inputs[i]);
-        }
-    }
-    // Process only uncached inputs in batches
-    for (let i = 0; i < uncachedInputs.length; i += batchSize) {
-        const batchEnd = Math.min(i + batchSize, uncachedInputs.length);
-        const batch = uncachedInputs.slice(i, batchEnd);
-        const indices = uncachedIndices.slice(i, batchEnd);
-        // Parallelize embeddings within batch using Promise.all
-        const batchResults = await Promise.all(batch.map((input) => getEmbedding(input)));
-        // Store results in correct positions and cache
-        for (let j = 0; j < batchResults.length; j++) {
-            results[indices[j]] = batchResults[j];
-        }
-    }
-    return results;
-}
-/**
- * Clear the embedding cache
- */
-export function clearEmbeddingCache() {
-    embeddingCache.clear();
-}
-/**
- * Get embedding cache statistics
- */
-export function getEmbeddingCacheStats() {
-    return {
-        size: embeddingCache.size,
-        maxSize: MAX_CACHE_SIZE,
-    };
-}
-/**
- * Local TF-IDF embedding using character n-grams and word hashing
- * Creates a 768-dimensional vector (same size as OpenAI text-embedding-3-small)
- * Fast, no API calls, works offline
- */
-function getLocalEmbedding(input) {
-    if (!input || typeof input !== 'string') {
-        return null;
-    }
-    // Normalize text
-    const text = input.toLowerCase()
-        .replace(/[^a-z0-9\s]/g, ' ')
-        .replace(/\s+/g, ' ')
-        .trim();
-    if (text.length === 0) {
-        return null;
-    }
-    // Embedding dimensions
-    const dimensions = 768;
-    const vector = new Array(dimensions).fill(0);
-    // Character n-grams (3-5 grams for semantic similarity)
-    const ngrams = [3, 4, 5];
-    for (const n of ngrams) {
-        for (let i = 0; i <= text.length - n; i++) {
-            const gram = text.substring(i, i + n);
-            const hash = djb2Hash(gram);
-            const idx = Math.abs(hash) % dimensions;
-            vector[idx] += 1;
-        }
-    }
-    // Word-level hashing for semantic capture
-    const words = text.split(/\s+/).filter(w => w.length > 2);
-    for (const word of words) {
-        const hash = djb2Hash(word);
-        const idx = Math.abs(hash) % dimensions;
-        vector[idx] += 2; // Words weighted higher than n-grams
-        // Bigrams
-        if (words.length > 1) {
-            const idx2 = words.indexOf(word);
-            if (idx2 < words.length - 1) {
-                const bigram = `${word}_${words[idx2 + 1]}`;
-                const bigramHash = djb2Hash(bigram);
-                const bigramIdx = Math.abs(bigramHash) % dimensions;
-                vector[bigramIdx] += 3; // Bigrams weighted highest
-            }
-        }
-    }
-    // Apply TF-IDF-like scaling: square root to dampen high frequencies
-    for (let i = 0; i < dimensions; i++) {
-        if (vector[i] > 0) {
-            vector[i] = Math.sqrt(vector[i]);
-        }
-    }
-    // L2 normalize
-    const norm = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
-    if (norm > 0) {
-        for (let i = 0; i < dimensions; i++) {
-            vector[i] /= norm;
-        }
-    }
-    return vector;
-}
-/**
- * DJB2 hash function - fast, good distribution
- */
-function djb2Hash(str) {
-    let hash = 5381;
-    for (let i = 0; i < str.length; i++) {
-        hash = ((hash << 5) + hash) + str.charCodeAt(i);
-    }
-    return hash;
-}
-async function getOpenAiEmbedding(input) {
-    if (!config.openAiApiKey)
-        return null;
-    try {
-        const response = await fetchWithRetryAndTimeout(config.openAiApiUrl, {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json',
-                Authorization: `Bearer ${config.openAiApiKey}`,
-            },
-            body: JSON.stringify({
-                model: config.openAiEmbeddingModel,
-                input,
-            }),
-        }, config.openAiTimeoutMs);
-        if (!response.ok) {
-            const message = await response.text();
-            logger.warn(`OpenAI embeddings failed: ${response.status} ${message}`);
-            return null; // Return null to allow fallback
-        }
-        const payload = await response.json();
-        const embedding = payload.data?.[0]?.embedding;
-        return embedding ?? null;
-    }
-    catch (error) {
-        logger.warn('OpenAI embeddings error:', { error: error });
-        return null; // Return null to allow fallback
-    }
-}
-async function getOllamaEmbedding(input) {
-    try {
-        const response = await fetchWithRetryAndTimeout(`${config.ollamaUrl}/api/embeddings`, {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({
-                model: config.ollamaEmbeddingModel || 'nomic-embed-text',
-                prompt: input,
-            }),
-        }, config.ollamaTimeoutMs);
-        if (!response.ok) {
-            const message = await response.text();
-            logger.warn(`Ollama embeddings failed: ${response.status} ${message}`);
-            return null; // Return null to allow fallback
-        }
-        const payload = await response.json();
-        return payload.embedding ?? null;
-    }
-    catch (error) {
-        logger.warn('Ollama embeddings error:', { error: error });
-        return null; // Return null to allow fallback
-    }
-}
-// LM Studio uses OpenAI-compatible API
-async function getLmStudioEmbedding(input) {
-    try {
-        const response = await fetchWithRetryAndTimeout(`${config.lmStudioUrl}/v1/embeddings`, {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({
-                model: config.lmStudioEmbeddingModel || 'text-embedding-nomic-embed-text-v1.5',
-                input: input,
-            }),
-        }, config.ollamaTimeoutMs); // Reuse Ollama timeout
-        if (!response.ok) {
-            const message = await response.text();
-            logger.warn(`LM Studio embeddings failed: ${response.status} ${message}`);
-            return null; // Return null to allow fallback
-        }
-        const payload = await response.json();
-        return payload.data?.[0]?.embedding ?? null;
-    }
-    catch (error) {
-        logger.warn('LM Studio embeddings error:', { error: error });
-        return null; // Return null to allow fallback
-    }
-}
-/**
- * Check health of all configured embedding providers
- * Returns availability and latency for each provider
- */
-export async function checkEmbeddingProviderHealth() {
-    const results = new Map();
-    const providers = ['local', 'openai', 'ollama', 'lmstudio', 'google', 'none', 'auto'];
-    // Test local provider (always available)
-    results.set('local', { available: true, latencyMs: 0 });
-    // Test OpenAI if configured
-    if (config.openAiApiKey) {
-        const start = Date.now();
-        try {
-            const testInput = 'health check';
-            const embedding = await withRetry(() => withTimeout(getOpenAiEmbedding(testInput), config.openAiTimeoutMs), config.embeddingsMaxRetries, config.embeddingsRetryDelayMs);
-            const latency = Date.now() - start;
-            results.set('openai', {
-                available: embedding !== null && embedding.length > 0,
-                latencyMs: latency
-            });
-        }
-        catch (error) {
-            results.set('openai', {
-                available: false,
-                error: error.message
-            });
-        }
-    }
-    else {
-        results.set('openai', { available: false, error: 'Not configured' });
-    }
-    // Test Ollama if configured
-    if (config.ollamaUrl) {
-        const start = Date.now();
-        try {
-            const testInput = 'health check';
-            const embedding = await withRetry(() => withTimeout(getOllamaEmbedding(testInput), config.ollamaTimeoutMs), config.embeddingsMaxRetries, config.embeddingsRetryDelayMs);
-            const latency = Date.now() - start;
-            results.set('ollama', {
-                available: embedding !== null && embedding.length > 0,
-                latencyMs: latency
-            });
-        }
-        catch (error) {
-            results.set('ollama', {
-                available: false,
-                error: error.message
-            });
-        }
-    }
-    else {
-        results.set('ollama', { available: false, error: 'Not configured' });
-    }
-    // Test LM Studio if configured
-    if (config.lmStudioUrl) {
-        const start = Date.now();
-        try {
-            const testInput = 'health check';
-            const embedding = await withRetry(() => withTimeout(getLmStudioEmbedding(testInput), config.ollamaTimeoutMs), config.embeddingsMaxRetries, config.embeddingsRetryDelayMs);
-            const latency = Date.now() - start;
-            results.set('lmstudio', {
-                available: embedding !== null && embedding.length > 0,
-                latencyMs: latency
-            });
-        }
-        catch (error) {
-            results.set('lmstudio', {
-                available: false,
-                error: error.message
-            });
-        }
-    }
-    else {
-        results.set('lmstudio', { available: false, error: 'Not configured' });
-    }
-    // Test Google if configured
-    if (config.googleCloudApiKey || config.googleCloudProject) {
-        const start = Date.now();
-        try {
-            const result = await withRetry(() => withTimeout(getGoogleMultimodalEmbedding({ text: 'health check' }), config.googleTimeoutMs), config.embeddingsMaxRetries, config.embeddingsRetryDelayMs);
-            const latency = Date.now() - start;
-            results.set('google', {
-                available: result !== null && result.embedding.length > 0,
-                latencyMs: latency
-            });
-        }
-        catch (error) {
-            results.set('google', {
-                available: false,
-                error: error.message
-            });
-        }
-    }
-    else {
-        results.set('google', { available: false, error: 'Not configured' });
-    }
-    return results;
-}
+// Re-export from embeddings directory
+export * from './embeddings/embeddings';
 //# sourceMappingURL=embeddings.js.map