rag-lite-ts 2.0.4 → 2.1.0

package/dist/core/reranking-config.js

@@ -17,15 +17,13 @@ export const DEFAULT_MULTIMODAL_RERANKING_CONFIG = {
         semantic: 0.7,
         metadata: 0.3
     },
-    fallback: 'metadata'
+    fallback: 'disabled'
 };
 // Strategy validation without complex interface patterns
 export function validateRerankingStrategy(strategy) {
     const validStrategies = [
         'cross-encoder',
         'text-derived',
-        'metadata',
-        'hybrid',
         'disabled'
     ];
     return validStrategies.includes(strategy);
@@ -36,7 +34,7 @@ export function validateRerankingConfig(config) {
         throw new Error('Reranking strategy is required');
     }
     if (!validateRerankingStrategy(config.strategy)) {
-        const validStrategies = ['cross-encoder', 'text-derived', 'metadata', 'hybrid', 'disabled'];
+        const validStrategies = ['cross-encoder', 'text-derived', 'disabled'];
         throw new Error(`Invalid reranking strategy '${config.strategy}'. ` +
             `Valid strategies: ${validStrategies.join(', ')}`);
     }
@@ -52,23 +50,16 @@ export function validateRerankingConfig(config) {
         if (visual !== undefined && (visual < 0 || visual > 1)) {
             throw new Error('Visual weight must be between 0 and 1');
         }
-        // Ensure weights sum to reasonable value for hybrid strategy
-        if (config.strategy === 'hybrid') {
-            const totalWeight = (semantic || 0) + (metadata || 0) + (visual || 0);
-            if (totalWeight === 0) {
-                throw new Error('Hybrid strategy requires at least one weight to be greater than 0');
-            }
-        }
     }
     // Validate fallback strategy if provided
     if (config.fallback && !validateRerankingStrategy(config.fallback)) {
-        const validStrategies = ['cross-encoder', 'text-derived', 'metadata', 'hybrid', 'disabled'];
+        const validStrategies = ['cross-encoder', 'text-derived', 'disabled'];
         throw new Error(`Invalid fallback strategy '${config.fallback}'. ` +
             `Valid strategies: ${validStrategies.join(', ')}`);
     }
     return {
         strategy: config.strategy,
-        enabled: config.enabled ?? true,
+        enabled: config.strategy === 'disabled' ? false : (config.enabled ?? true),
         model: config.model,
         weights: config.weights,
         fallback: config.fallback || 'disabled'
@@ -91,7 +82,7 @@ export function isStrategySupported(strategy, mode) {
         case 'text':
             return strategy === 'cross-encoder' || strategy === 'disabled';
         case 'multimodal':
-            return ['text-derived', 'metadata', 'hybrid', 'disabled'].includes(strategy);
+            return ['text-derived', 'disabled'].includes(strategy);
         default:
             return false;
     }
@@ -102,7 +93,7 @@ export function getSupportedStrategies(mode) {
         case 'text':
             return ['cross-encoder', 'disabled'];
         case 'multimodal':
-            return ['text-derived', 'metadata', 'hybrid', 'disabled'];
+            return ['text-derived', 'disabled'];
         default:
             return ['disabled'];
     }
@@ -145,7 +136,7 @@ export class RerankingConfigBuilder {
             .strategy('text-derived')
             .enabled(true)
             .weights({ semantic: 0.7, metadata: 0.3 })
-            .fallback('metadata');
+            .fallback('disabled');
     }
     static disabled() {
         return new RerankingConfigBuilder()

package/dist/core/reranking-factory.js

@@ -6,7 +6,7 @@
  * principle of using simple functions over complex factory patterns.
  */
 import { getDefaultRerankingConfig, isStrategySupported, getSupportedStrategies, validateRerankingConfig } from './reranking-config.js';
-import { createCrossEncoderRerankFunction, createTextDerivedRerankFunction, createMetadataRerankFunction } from './reranking-strategies.js';
+import { createCrossEncoderRerankFunction, createTextDerivedRerankFunction } from './reranking-strategies.js';
 /**
  * Simple reranking creation function with conditional logic
  *
@@ -102,23 +102,6 @@ function createRerankingFunction(mode, strategy, config) {
                 undefined // Use default cross-encoder model
                 );
                 break;
-            case 'metadata':
-                console.log(`Creating metadata reranker for ${mode} mode`);
-                reranker = createMetadataRerankFunction({
-                    weights: config.weights ? {
-                        filename: config.weights.metadata || 0.4,
-                        contentType: 0.3,
-                        metadata: config.weights.metadata || 0.3
-                    } : undefined
-                });
-                break;
-            case 'hybrid':
-                if (mode !== 'multimodal') {
-                    throw new RerankingStrategyError(strategy, mode, 'Hybrid strategy only supported in multimodal mode', 'UNSUPPORTED_MODE');
-                }
-                console.log('Creating hybrid reranker for multimodal mode');
-                reranker = createHybridRerankFunction(config);
-                break;
             case 'disabled':
                 console.log('Reranking explicitly disabled');
                 return undefined;
@@ -241,172 +224,10 @@ function wrapRerankFunctionWithErrorRecovery(reranker, strategy, mode) {
     };
 }
 /**
- * Create hybrid reranking function that combines multiple strategies with enhanced error recovery
+ * Hybrid reranking strategy removed in Phase 3 - throwing error for backward compatibility
  */
 function createHybridRerankFunction(config) {
-    // Default weights if not specified
-    const weights = config.weights || {
-        semantic: 0.6,
-        metadata: 0.4,
-        visual: 0.0 // Not implemented yet
-    };
-    // Track which strategies are available
-    const availableStrategies = {};
-    // Initialize strategies with error handling
-    try {
-        if (weights.semantic && weights.semantic > 0) {
-            availableStrategies.textDerived = createTextDerivedRerankFunction();
-            console.log('✅ Text-derived strategy initialized for hybrid reranking');
-        }
-    }
-    catch (error) {
-        console.warn(`⚠️ Text-derived strategy initialization failed for hybrid reranking: ${error instanceof Error ? error.message : 'Unknown error'}`);
-    }
-    try {
-        if (weights.metadata && weights.metadata > 0) {
-            availableStrategies.metadata = createMetadataRerankFunction();
-            console.log('✅ Metadata strategy initialized for hybrid reranking');
-        }
-    }
-    catch (error) {
-        console.warn(`⚠️ Metadata strategy initialization failed for hybrid reranking: ${error instanceof Error ? error.message : 'Unknown error'}`);
-    }
-    // Check if any strategies are available
-    const hasAvailableStrategies = Object.keys(availableStrategies).length > 0;
-    if (!hasAvailableStrategies) {
-        throw new RerankingStrategyError('hybrid', 'multimodal', 'No hybrid reranking strategies could be initialized', 'NO_STRATEGIES_AVAILABLE');
-    }
-    console.log(`Hybrid reranking initialized with ${Object.keys(availableStrategies).length} available strategies`);
-    return async (query, results, contentType) => {
-        const startTime = Date.now();
-        const strategyResults = {};
-        try {
-            console.log(`🔄 Running hybrid reranking with ${Object.keys(availableStrategies).length} strategies`);
-            // Start with original results
-            let hybridResults = [...results];
-            let successfulStrategies = 0;
-            // Apply text-derived reranking if available and enabled
-            if (availableStrategies.textDerived && weights.semantic && weights.semantic > 0) {
-                const strategyStartTime = Date.now();
-                try {
-                    console.log(`🔧 Applying text-derived reranking (weight: ${weights.semantic})`);
-                    const textDerivedResults = await availableStrategies.textDerived(query, hybridResults, contentType);
-                    // Combine scores with semantic weight
-                    hybridResults = hybridResults.map((result, index) => {
-                        const textDerivedScore = textDerivedResults[index]?.score || result.score;
-                        const combinedScore = result.score * (1 - weights.semantic) + textDerivedScore * weights.semantic;
-                        return {
-                            ...result,
-                            score: combinedScore,
-                            metadata: {
-                                ...result.metadata,
-                                hybridScores: {
-                                    ...(result.metadata?.hybridScores || {}),
-                                    textDerived: textDerivedScore,
-                                    semantic: combinedScore
-                                }
-                            }
-                        };
-                    });
-                    const strategyDuration = Date.now() - strategyStartTime;
-                    strategyResults.textDerived = { success: true, duration: strategyDuration };
-                    successfulStrategies++;
-                    console.log(`✅ Text-derived reranking completed (${strategyDuration}ms)`);
-                }
-                catch (error) {
-                    const strategyDuration = Date.now() - strategyStartTime;
-                    const errorMessage = error instanceof Error ? error.message : 'Unknown error';
-                    strategyResults.textDerived = { success: false, error: errorMessage, duration: strategyDuration };
-                    console.warn(`❌ Text-derived reranking failed in hybrid mode (${strategyDuration}ms): ${errorMessage}`);
-                }
-            }
-            // Apply metadata reranking if available and enabled
-            if (availableStrategies.metadata && weights.metadata && weights.metadata > 0) {
-                const strategyStartTime = Date.now();
-                try {
-                    console.log(`🔧 Applying metadata reranking (weight: ${weights.metadata})`);
-                    const metadataResults = await availableStrategies.metadata(query, hybridResults, contentType);
-                    // Combine scores with metadata weight
-                    hybridResults = hybridResults.map((result, index) => {
-                        const metadataScore = metadataResults[index]?.score || result.score;
-                        const currentScore = result.score;
-                        const combinedScore = currentScore * (1 - weights.metadata) + metadataScore * weights.metadata;
-                        return {
-                            ...result,
-                            score: combinedScore,
-                            metadata: {
-                                ...result.metadata,
-                                hybridScores: {
-                                    ...(result.metadata?.hybridScores || {}),
-                                    metadata: metadataScore,
-                                    combined: combinedScore
-                                }
-                            }
-                        };
-                    });
-                    const strategyDuration = Date.now() - strategyStartTime;
-                    strategyResults.metadata = { success: true, duration: strategyDuration };
-                    successfulStrategies++;
-                    console.log(`✅ Metadata reranking completed (${strategyDuration}ms)`);
-                }
-                catch (error) {
-                    const strategyDuration = Date.now() - strategyStartTime;
-                    const errorMessage = error instanceof Error ? error.message : 'Unknown error';
-                    strategyResults.metadata = { success: false, error: errorMessage, duration: strategyDuration };
-                    console.warn(`❌ Metadata reranking failed in hybrid mode (${strategyDuration}ms): ${errorMessage}`);
-                }
-            }
-            // Sort by final combined scores
-            hybridResults.sort((a, b) => b.score - a.score);
-            const totalDuration = Date.now() - startTime;
-            // Add hybrid reranking metadata to results
-            hybridResults = hybridResults.map(result => ({
-                ...result,
-                metadata: {
-                    ...result.metadata,
-                    hybridRerankingInfo: {
-                        totalDuration,
-                        successfulStrategies,
-                        strategyResults,
-                        weights
-                    }
-                }
-            }));
-            if (successfulStrategies > 0) {
-                console.log(`✅ Hybrid reranking completed successfully (${totalDuration}ms, ${successfulStrategies}/${Object.keys(availableStrategies).length} strategies succeeded)`);
-            }
-            else {
-                console.warn(`⚠️ Hybrid reranking completed with no successful strategies (${totalDuration}ms), returning original results`);
-                return results; // Return original results if no strategies succeeded
-            }
-            return hybridResults;
-        }
-        catch (error) {
-            const totalDuration = Date.now() - startTime;
-            const errorMessage = error instanceof Error ? error.message : 'Unknown error';
-            console.warn(`❌ Hybrid reranking failed (${totalDuration}ms): ${errorMessage}. ` +
-                `Returning original results.`);
-            // Log detailed error information
-            console.error('Hybrid reranking error details:', {
-                query: query.substring(0, 100) + (query.length > 100 ? '...' : ''),
-                resultCount: results.length,
-                contentType,
-                availableStrategies: Object.keys(availableStrategies),
-                weights,
-                strategyResults,
-                error: errorMessage
-            });
-            return results.map(result => ({
-                ...result,
-                metadata: {
-                    ...result.metadata,
-                    hybridRerankingFailed: true,
-                    hybridRerankingError: errorMessage,
-                    fallbackToVectorSimilarity: true
-                }
-            }));
-        }
-    };
+    throw new RerankingStrategyError('hybrid', 'multimodal', 'Hybrid reranking strategy has been removed in this version. Use text-derived instead.', 'STRATEGY_REMOVED');
 }
 /**
  * Create reranker with automatic mode detection
@@ -582,8 +403,6 @@ export function getRerankingStats() {
         strategiesUsed: {
             'cross-encoder': 0,
             'text-derived': 0,
-            'metadata': 0,
-            'hybrid': 0,
             'disabled': 0
         }
     };

package/dist/core/reranking-strategies.js

@@ -194,7 +194,7 @@ export class TextDerivedRerankingStrategy {
         catch (error) {
             console.warn(`Failed to generate description for image ${imagePath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
             // Fallback to filename-based description
-            const filename = imagePath.split('/').pop() || imagePath;
+            const filename = imagePath.split('/').pop() || imagePath.split('\\').pop() || imagePath;
             return `Image file: ${filename}`;
         }
     }
@@ -211,16 +211,17 @@ export class TextDerivedRerankingStrategy {
             // Step 1: Convert images to text descriptions
             const processedResults = await Promise.all(results.map(async (result) => {
                 if (result.contentType === 'image') {
-                    // Generate text description for image
-                    const description = await this.generateImageDescription(result.content);
+                    // Generate text description for image using the file path from document.source
+                    const description = await this.generateImageDescription(result.document.source);
                     return {
                         ...result,
                         content: description,
+                        contentType: 'text', // Change to 'text' so cross-encoder will process it
                         originalContent: result.content,
                         originalContentType: result.contentType,
                         metadata: {
                             ...result.metadata,
-                            originalImagePath: result.content,
+                            originalImagePath: result.document.source,
                             generatedDescription: description
                         }
                     };

package/dist/core/search.d.ts

@@ -80,6 +80,16 @@ export declare class SearchEngine {
      * @returns Promise resolving to array of search results
      */
     search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Perform semantic search using a pre-computed embedding vector
+     * Useful for image-based search or when embedding is computed externally
+     * @param queryVector - Pre-computed query embedding vector
+     * @param options - Search options including top_k and rerank settings
+     * @param originalQuery - Optional original query for reranking (text or image path)
+     * @param embeddingTime - Optional embedding time for logging
+     * @returns Promise resolving to array of search results
+     */
+    searchWithVector(queryVector: Float32Array, options?: SearchOptions, originalQuery?: string, embeddingTime?: number): Promise<SearchResult[]>;
     /**
      * Format search results with proper structure
      * @param chunks - Database chunks with metadata

package/dist/core/search.js

@@ -106,18 +106,40 @@ export class SearchEngine {
             return [];
         }
         const startTime = performance.now();
-        const topK = options.top_k || config.top_k || 10;
-        const shouldRerank = options.rerank !== undefined ? options.rerank : (this.rerankFn !== undefined);
         try {
             // Step 1: Build query embedding using injected embed function
             const embeddingStartTime = performance.now();
             const queryEmbedding = await this.embedFn(query);
             const embeddingTime = performance.now() - embeddingStartTime;
-            // Step 2: Search using IndexManager (which handles hash mapping properly)
+            // Step 2: Search with the vector
+            const results = await this.searchWithVector(queryEmbedding.vector, options, query, embeddingTime);
+            return results;
+        }
+        catch (error) {
+            throw new Error(`Search failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Perform semantic search using a pre-computed embedding vector
+     * Useful for image-based search or when embedding is computed externally
+     * @param queryVector - Pre-computed query embedding vector
+     * @param options - Search options including top_k and rerank settings
+     * @param originalQuery - Optional original query for reranking (text or image path)
+     * @param embeddingTime - Optional embedding time for logging
+     * @returns Promise resolving to array of search results
+     */
+    async searchWithVector(queryVector, options = {}, originalQuery, embeddingTime) {
+        const startTime = performance.now();
+        const topK = options.top_k || config.top_k || 10;
+        // Phase 1: Disable reranking by default for better performance
+        // Users must explicitly opt-in with --rerank flag
+        const shouldRerank = options.rerank === true;
+        try {
+            // Step 1: Search using IndexManager (which handles hash mapping properly)
             const searchStartTime = performance.now();
             let searchResult;
             try {
-                searchResult = this.indexManager.search(queryEmbedding.vector, topK);
+                searchResult = this.indexManager.search(queryVector, topK);
             }
             catch (error) {
                 if (error instanceof Error && error.message.includes('No embedding ID found for hash')) {
@@ -133,18 +155,18 @@ export class SearchEngine {
                 console.log(`No similar documents found (${totalTime.toFixed(2)}ms total)`);
                 return [];
             }
-            // Step 3: Retrieve chunks from database using embedding IDs
+            // Step 2: Retrieve chunks from database using embedding IDs
             const retrievalStartTime = performance.now();
             const chunks = await getChunksByEmbeddingIds(this.db, searchResult.embeddingIds);
             const retrievalTime = performance.now() - retrievalStartTime;
-            // Step 4: Format results as JSON with text, score, and document metadata
+            // Step 3: Format results as JSON with text, score, and document metadata
             let results = this.formatSearchResults(chunks, searchResult.distances, searchResult.embeddingIds);
-            // Step 5: Optional reranking with injected rerank function
+            // Step 4: Optional reranking with injected rerank function
             let rerankTime = 0;
-            if (shouldRerank && this.rerankFn && results.length > 1) {
+            if (shouldRerank && this.rerankFn && results.length > 1 && originalQuery) {
                 try {
                     const rerankStartTime = performance.now();
-                    results = await this.rerankFn(query, results);
+                    results = await this.rerankFn(originalQuery, results);
                     rerankTime = performance.now() - rerankStartTime;
                 }
                 catch (error) {
@@ -154,13 +176,14 @@ export class SearchEngine {
             }
             const totalTime = performance.now() - startTime;
             // Measure latency without premature optimization - just log for monitoring
+            const embedTimeStr = embeddingTime !== undefined ? `embed: ${embeddingTime.toFixed(2)}ms, ` : '';
             console.log(`Search completed: ${results.length} results in ${totalTime.toFixed(2)}ms ` +
-                `(embed: ${embeddingTime.toFixed(2)}ms, vector: ${vectorSearchTime.toFixed(2)}ms, ` +
+                `(${embedTimeStr}vector: ${vectorSearchTime.toFixed(2)}ms, ` +
                 `retrieval: ${retrievalTime.toFixed(2)}ms${rerankTime > 0 ? `, rerank: ${rerankTime.toFixed(2)}ms` : ''})`);
             return results;
         }
         catch (error) {
-            throw new Error(`Search failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            throw new Error(`Vector search failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
         }
     }
     /**

package/dist/factories/ingestion-factory.js

@@ -323,7 +323,9 @@ export class IngestionFactory {
         const { getSystemInfo, setSystemInfo } = await import('../core/db.js');
         // Determine the effective mode and reranking strategy
         const effectiveMode = options.mode || 'text';
-        const effectiveRerankingStrategy = options.rerankingStrategy || 'cross-encoder';
+        // Phase 1: Fix mode-specific reranking strategy defaults
+        const effectiveRerankingStrategy = options.rerankingStrategy ||
+            (effectiveMode === 'multimodal' ? 'text-derived' : 'cross-encoder');
         // Determine model type based on model name
         let modelType;
         if (effectiveModel.includes('clip')) {