rag-lite-ts 1.0.1 → 2.0.0

package/dist/{reranker.js → text/reranker.js}

@@ -1,82 +1,82 @@
-import { AutoTokenizer, AutoModelForSequenceClassification } from '@huggingface/transformers';
-import { config } from './config.js';
+// Set up polyfills immediately before any other imports
+// Force polyfills in Node.js environment regardless of window state
+if (typeof globalThis !== 'undefined') {
+    if (typeof globalThis.self === 'undefined') {
+        globalThis.self = globalThis;
+    }
+    if (typeof global.self === 'undefined') {
+        global.self = global;
+    }
+}
+import '../dom-polyfills.js';
+import { config } from '../core/config.js';
 /**
- * Cross-encoder reranker for improving search result quality
- * Uses a cross-encoder model to rerank initial vector search results
+ * Embedding-based reranker for improving search result quality
+ * Uses embedding similarity to rerank initial vector search results
  */
 export class CrossEncoderReranker {
     model = null; // Use any to avoid complex transformers.js typing issues
     tokenizer = null;
-    modelName = 'Xenova/ms-marco-MiniLM-L-6-v2'; // Use working model as default
-    // Alternative models in case the primary fails
-    static FALLBACK_MODELS = [
-        'Xenova/ms-marco-MiniLM-L-6-v2', // Primary - optimized for transformers.js
-        'cross-encoder/ms-marco-MiniLM-L-6-v2', // Original (likely to fail)
-        'cross-encoder/ms-marco-MiniLM-L-2-v2', // Smaller original (likely to fail)
-        // Note: sentence-transformers/all-MiniLM-L6-v2 is a bi-encoder, not cross-encoder, so removed
-    ];
+    modelName = 'Xenova/ms-marco-MiniLM-L-6-v2'; // Use working cross-encoder model
     /**
-     * Load the cross-encoder model with graceful fallback
+     * Ensure DOM polyfills are set up for transformers.js
      */
-    async loadModel() {
-        // Try primary model first (should work since it's Xenova)
-        if (await this.tryLoadModel(this.modelName)) {
-            return;
-        }
-        // Try fallback models if primary fails
-        console.warn(`Primary model ${this.modelName} failed, trying fallbacks...`);
-        for (const fallbackModel of CrossEncoderReranker.FALLBACK_MODELS) {
-            if (fallbackModel === this.modelName)
-                continue; // Skip already tried model
-            console.warn(`Trying fallback model: ${fallbackModel}`);
-            if (await this.tryLoadModel(fallbackModel)) {
-                this.modelName = fallbackModel;
-                return;
+    ensurePolyfills() {
+        // Use the exact same approach as the working standalone version
+        if (typeof window === 'undefined' && typeof globalThis !== 'undefined') {
+            if (typeof globalThis.self === 'undefined') {
+                globalThis.self = globalThis;
             }
+            if (typeof global.self === 'undefined') {
+                global.self = global;
+            }
+            // Polyfills already set up at module level
         }
-        console.warn('All cross-encoder models failed to load. Reranking will be disabled.');
-        this.model = null;
-        this.tokenizer = null;
+    }
+    /**
+     * Load the embedding model
+     */
+    async loadModel() {
+        await this.tryLoadModel(this.modelName);
     }
     /**
      * Try to load a specific model
      */
     async tryLoadModel(modelName) {
-        try {
-            console.log(`Loading cross-encoder model: ${modelName}`);
-            // Load model and tokenizer separately for proper cross-encoder usage
-            this.model = await AutoModelForSequenceClassification.from_pretrained(modelName, {
-                cache_dir: config.model_cache_path,
-                dtype: 'fp32' // Suppress dtype warnings
-            });
-            this.tokenizer = await AutoTokenizer.from_pretrained(modelName, {
-                cache_dir: config.model_cache_path
-            });
-            console.log(`Cross-encoder model loaded successfully: ${modelName}`);
-            return true;
-        }
-        catch (error) {
-            console.warn(`Failed to load model ${modelName}: ${error instanceof Error ? error.message : 'Unknown error'}`);
-            return false;
-        }
+        console.log(`Loading cross-encoder model: ${modelName}`);
+        // Ensure polyfills are set up exactly like the working standalone version
+        this.ensurePolyfills();
+        // Use the exact same approach as the working standalone test
+        const { AutoTokenizer, AutoModelForSequenceClassification } = await import('@huggingface/transformers');
+        console.log('Loading model...');
+        this.model = await AutoModelForSequenceClassification.from_pretrained(modelName, {
+            cache_dir: config.model_cache_path,
+            dtype: 'fp32'
+        });
+        console.log('Loading tokenizer...');
+        this.tokenizer = await AutoTokenizer.from_pretrained(modelName, {
+            cache_dir: config.model_cache_path
+        });
+        console.log(`Cross-encoder model loaded successfully: ${modelName}`);
     }
     /**
-     * Rerank search results using cross-encoder scoring
+     * Rerank search results using embedding similarity scoring
      * @param query - Original search query
      * @param results - Initial search results from vector search
      * @returns Promise resolving to reranked results
      */
     async rerank(query, results) {
-        if (!this.model || !this.tokenizer) {
+        if (!this.model) {
             throw new Error('Cross-encoder model not loaded. Call loadModel() first.');
         }
         if (results.length === 0) {
             return results;
         }
         try {
-            // Prepare queries and documents for proper cross-encoder format
+            // Use cross-encoder approach - prepare queries and documents for proper cross-encoder format
             const queries = results.map(() => query);
-            const documents = results.map(result => result.text);
+            const documents = results.map(result => result.content);
+            console.log('Tokenizing query-document pairs...');
             // Tokenize using the proper cross-encoder format with text_pair
             const features = this.tokenizer(queries, {
                 text_pair: documents,
@@ -84,6 +84,7 @@ export class CrossEncoderReranker {
                 truncation: true,
                 return_tensors: 'pt'
             });
+            console.log('Running cross-encoder inference...');
             // Get model predictions
             const output = await this.model(features);
             // Extract logits - these are the raw relevance scores
@@ -134,15 +135,33 @@ export class CrossEncoderReranker {
     getModelName() {
         return this.modelName;
     }
+    /**
+     * Compute cosine similarity between two Float32Array embeddings
+     */
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length) {
+            return 0;
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+        return magnitude === 0 ? 0 : dotProduct / magnitude;
+    }
     /**
      * Simple text-based reranking using keyword matching and text similarity
-     * This is a fallback when cross-encoder models don't work well for the content
+     * This is a fallback when embedding similarity doesn't provide good discrimination
      */
     simpleTextReranking(query, results) {
         const queryLower = query.toLowerCase();
         const queryWords = queryLower.split(/\s+/).filter(word => word.length > 2);
         const rerankedResults = results.map(result => {
-            const textLower = result.text.toLowerCase();
+            const textLower = result.content.toLowerCase();
             const titleLower = result.document.title?.toLowerCase() || '';
             let score = result.score; // Start with vector search score
             let bonus = 0;
@@ -209,4 +228,70 @@ export class CrossEncoderReranker {
         return rerankedResults;
     }
 }
+/*
+*
+ * Create a RerankFunction implementation using the embedding-based reranker
+ * This function implements the core RerankFunction interface for dependency injection
+ * @param modelName - Optional model name override
+ * @returns RerankFunction that can be injected into core components
+ */
+export function createTextRerankFunction(modelName) {
+    let reranker = null;
+    const rerankFunction = async (query, results, contentType) => {
+        // Only support text content type
+        if (contentType && contentType !== 'text') {
+            throw new Error(`Text reranker only supports 'text' content type, got: ${contentType}`);
+        }
+        // Initialize reranker if not already done
+        if (!reranker) {
+            reranker = new CrossEncoderReranker();
+            if (modelName) {
+                // Set custom model name if provided
+                reranker.modelName = modelName;
+            }
+            await reranker.loadModel();
+        }
+        // If reranker failed to load, return results unchanged
+        if (!reranker.isLoaded()) {
+            console.warn('Text reranker not loaded, returning results unchanged');
+            return results;
+        }
+        // Use the existing rerank method
+        return await reranker.rerank(query, results);
+    };
+    return rerankFunction;
+}
+/**
+ * Create a text reranker factory function
+ * @param modelName - Optional model name override
+ * @returns Factory function that creates initialized rerankers
+ */
+export function createTextReranker(modelName) {
+    return {
+        async rerank(query, results) {
+            const reranker = new CrossEncoderReranker();
+            if (modelName) {
+                reranker.modelName = modelName;
+            }
+            await reranker.loadModel();
+            if (!reranker.isLoaded()) {
+                console.warn('Text reranker not loaded, returning results unchanged');
+                return results;
+            }
+            return reranker.rerank(query, results);
+        },
+        async loadModel() {
+            const reranker = new CrossEncoderReranker();
+            if (modelName) {
+                reranker.modelName = modelName;
+            }
+            await reranker.loadModel();
+        },
+        isLoaded() {
+            // For the factory version, we create new instances each time
+            // so we can't track loaded state
+            return true;
+        }
+    };
+}
 //# sourceMappingURL=reranker.js.map

package/dist/text/sentence-transformer-embedder.d.ts

@@ -0,0 +1,96 @@
+/**
+ * TEXT IMPLEMENTATION — Sentence Transformer Embedder Implementation
+ * Implements UniversalEmbedder interface for sentence-transformer models
+ * Adapts existing text embedding logic to the universal interface
+ */
+import '../dom-polyfills.js';
+import { BaseUniversalEmbedder, type EmbedderOptions } from '../core/abstract-embedder.js';
+import type { EmbeddingResult } from '../types.js';
+/**
+ * Sentence transformer embedder implementation
+ * Supports sentence-transformers/all-MiniLM-L6-v2 and Xenova/all-mpnet-base-v2
+ * Ensures consistent EmbeddingResult format with contentType='text'
+ * Adapts existing EmbeddingEngine to UniversalEmbedder interface
+ */
+export declare class SentenceTransformerEmbedder extends BaseUniversalEmbedder {
+    private embeddingEngine;
+    private resourceManager;
+    private embedderResourceId?;
+    private engineResourceId?;
+    constructor(modelName: string, options?: EmbedderOptions);
+    /**
+     * Load the sentence transformer model using existing EmbeddingEngine
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Clean up model resources with comprehensive disposal
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Embed text using the existing EmbeddingEngine
+     */
+    embedText(text: string): Promise<EmbeddingResult>;
+    /**
+     * Optimized batch processing using existing EmbeddingEngine and BatchProcessingOptimizer
+     * Overrides the base implementation for better performance with progress reporting
+     */
+    protected processBatch(batch: Array<{
+        content: string;
+        contentType: string;
+        metadata?: Record<string, any>;
+    }>): Promise<EmbeddingResult[]>;
+    /**
+     * Get model-specific information
+     */
+    getModelInfo(): {
+        capabilities: {
+            supportsSemanticSimilarity: boolean;
+            supportsTextClassification: boolean;
+            supportsTextClustering: boolean;
+            recommendedUseCase: string;
+            supportsText: boolean;
+            supportsImages: boolean;
+            supportsBatchProcessing: boolean;
+            supportsMetadata: boolean;
+            maxBatchSize?: number;
+            maxTextLength?: number;
+            supportedImageFormats?: readonly string[];
+            supportsMultimodal?: boolean;
+            supportsCrossModalSearch?: boolean;
+            unifiedEmbeddingSpace?: boolean;
+            reliableImplementation?: boolean;
+        };
+        name: string;
+        type: import("../core/universal-embedder.js").ModelType;
+        dimensions: number;
+        version: string;
+        supportedContentTypes: readonly string[];
+        requirements: import("../types.js").ModelRequirements;
+    };
+    /**
+     * Check if the model is suitable for a specific task
+     */
+    isSuitableForTask(task: 'similarity' | 'classification' | 'clustering' | 'retrieval'): boolean;
+    /**
+     * Embed document batch using existing EmbeddingEngine's optimized method
+     * This method provides compatibility with the existing document ingestion pipeline
+     */
+    embedDocumentBatch(chunks: string[]): Promise<EmbeddingResult[]>;
+    /**
+     * Get the model version from the underlying EmbeddingEngine
+     */
+    getModelVersion(): string;
+    /**
+     * Get the batch size from the underlying EmbeddingEngine
+     */
+    getBatchSize(): number;
+    /**
+     * Check if the underlying EmbeddingEngine is loaded
+     */
+    isEngineLoaded(): boolean;
+    /**
+     * Override isLoaded to check both internal state and engine state
+     */
+    isLoaded(): boolean;
+}
+//# sourceMappingURL=sentence-transformer-embedder.d.ts.map