rag-lite-ts 2.0.0 → 2.0.2

package/dist/factories/polymorphic-factory.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Polymorphic factory for creating mode-aware search engines
+ * Automatically detects mode from database and uses appropriate embedder
+ *
+ * This factory implements the Chameleon Architecture principle:
+ * - Detects mode (text/multimodal) from database configuration
+ * - Uses appropriate embedder based on detected mode
+ * - Provides seamless polymorphic behavior without user intervention
+ *
+ * @example
+ * ```typescript
+ * // Automatically detects mode and creates appropriate search engine
+ * const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
+ *
+ * // Works for both text and multimodal modes
+ * const results = await search.search('query');
+ * ```
+ */
+import { SearchEngine } from '../core/search.js';
+export interface PolymorphicSearchOptions {
+    /** Whether to enable reranking (default: true) */
+    enableReranking?: boolean;
+    /** Top-k results to return (default: from config) */
+    topK?: number;
+}
+/**
+ * Factory for creating mode-aware search engines
+ * Automatically detects mode from database and uses appropriate embedder
+ */
+export declare class PolymorphicSearchFactory {
+    /**
+     * Create a SearchEngine that automatically adapts to the mode stored in the database
+     *
+     * This method:
+     * 1. Validates that required files exist
+     * 2. Opens database and reads system configuration
+     * 3. Detects mode (text/multimodal) from database
+     * 4. Creates appropriate embedder based on mode
+     * 5. Optionally creates reranker based on configuration
+     * 6. Returns fully configured SearchEngine
+     *
+     * @param indexPath - Path to the vector index file (must exist)
+     * @param dbPath - Path to the SQLite database file (must exist)
+     * @param options - Optional configuration overrides
+     * @returns Promise resolving to configured SearchEngine
+     * @throws {Error} If required files don't exist or initialization fails
+     */
+    static create(indexPath: string, dbPath: string, options?: PolymorphicSearchOptions): Promise<SearchEngine>;
+}
+//# sourceMappingURL=polymorphic-factory.d.ts.map

package/dist/factories/polymorphic-factory.js

@@ -0,0 +1,159 @@
+/**
+ * Polymorphic factory for creating mode-aware search engines
+ * Automatically detects mode from database and uses appropriate embedder
+ *
+ * This factory implements the Chameleon Architecture principle:
+ * - Detects mode (text/multimodal) from database configuration
+ * - Uses appropriate embedder based on detected mode
+ * - Provides seamless polymorphic behavior without user intervention
+ *
+ * @example
+ * ```typescript
+ * // Automatically detects mode and creates appropriate search engine
+ * const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
+ *
+ * // Works for both text and multimodal modes
+ * const results = await search.search('query');
+ * ```
+ */
+import { SearchEngine } from '../core/search.js';
+import { IndexManager } from '../index-manager.js';
+import { openDatabase, getSystemInfo } from '../core/db.js';
+import { createTextEmbedFunction } from '../text/embedder.js';
+import { createTextRerankFunction } from '../text/reranker.js';
+import { config, getModelDefaults } from '../core/config.js';
+import { existsSync } from 'fs';
+import { createMissingFileError, createInvalidPathError, createFactoryCreationError } from '../core/actionable-error-messages.js';
+/**
+ * Factory for creating mode-aware search engines
+ * Automatically detects mode from database and uses appropriate embedder
+ */
+export class PolymorphicSearchFactory {
+    /**
+     * Create a SearchEngine that automatically adapts to the mode stored in the database
+     *
+     * This method:
+     * 1. Validates that required files exist
+     * 2. Opens database and reads system configuration
+     * 3. Detects mode (text/multimodal) from database
+     * 4. Creates appropriate embedder based on mode
+     * 5. Optionally creates reranker based on configuration
+     * 6. Returns fully configured SearchEngine
+     *
+     * @param indexPath - Path to the vector index file (must exist)
+     * @param dbPath - Path to the SQLite database file (must exist)
+     * @param options - Optional configuration overrides
+     * @returns Promise resolving to configured SearchEngine
+     * @throws {Error} If required files don't exist or initialization fails
+     */
+    static async create(indexPath, dbPath, options = {}) {
+        try {
+            console.log('🏭 PolymorphicSearchFactory: Initializing mode-aware search engine...');
+            // Validate input paths
+            if (!indexPath || !dbPath) {
+                throw createInvalidPathError([
+                    { name: 'indexPath', value: indexPath },
+                    { name: 'dbPath', value: dbPath }
+                ], { operationContext: 'PolymorphicSearchFactory.create' });
+            }
+            // Check if required files exist
+            if (!existsSync(indexPath)) {
+                throw createMissingFileError(indexPath, 'index', {
+                    operationContext: 'PolymorphicSearchFactory.create'
+                });
+            }
+            if (!existsSync(dbPath)) {
+                throw createMissingFileError(dbPath, 'database', {
+                    operationContext: 'PolymorphicSearchFactory.create'
+                });
+            }
+            // Step 1: Open database and detect mode
+            console.log('💾 Opening database and detecting mode...');
+            const db = await openDatabase(dbPath);
+            let mode = 'text';
+            let embeddingModel;
+            let modelDimensions;
+            try {
+                const systemInfo = await getSystemInfo(db);
+                if (systemInfo) {
+                    mode = systemInfo.mode;
+                    embeddingModel = systemInfo.modelName;
+                    modelDimensions = systemInfo.modelDimensions;
+                    console.log(`📊 Detected mode: ${mode}`);
+                    console.log(`📊 Detected model: ${embeddingModel} (${modelDimensions} dimensions)`);
+                }
+                else {
+                    // Fallback to default if no system info
+                    embeddingModel = config.embedding_model;
+                    const modelDefaults = getModelDefaults(embeddingModel);
+                    modelDimensions = modelDefaults.dimensions;
+                    console.log(`📊 No system info found, using default: ${embeddingModel} (${modelDimensions} dimensions)`);
+                }
+            }
+            catch (error) {
+                // If getSystemInfo fails, use defaults
+                embeddingModel = config.embedding_model;
+                const modelDefaults = getModelDefaults(embeddingModel);
+                modelDimensions = modelDefaults.dimensions;
+                console.log(`📊 Using default configuration: ${embeddingModel} (${modelDimensions} dimensions)`);
+            }
+            // Step 2: Create appropriate embedder based on mode
+            let embedFn;
+            if (mode === 'multimodal') {
+                console.log('📊 Loading CLIP embedder for multimodal mode...');
+                const { createEmbedder } = await import('../core/embedder-factory.js');
+                const clipEmbedder = await createEmbedder(embeddingModel);
+                // Wrap CLIP embedder to match EmbedFunction signature
+                embedFn = async (content, contentType) => {
+                    if (contentType === 'image') {
+                        return await clipEmbedder.embedImage(content);
+                    }
+                    return await clipEmbedder.embedText(content);
+                };
+                console.log('✓ CLIP embedder loaded for multimodal mode');
+            }
+            else {
+                console.log('📊 Loading text embedder for text mode...');
+                embedFn = createTextEmbedFunction(embeddingModel);
+                console.log('✓ Text embedder loaded');
+            }
+            // Step 3: Initialize reranking function (optional)
+            let rerankFn;
+            if (options.enableReranking === true) {
+                console.log('🔄 Loading reranking model...');
+                rerankFn = createTextRerankFunction();
+                await rerankFn('test query', []);
+                console.log('✓ Reranking model loaded successfully');
+            }
+            else {
+                console.log('🔄 Reranking disabled (local-first, fast mode)');
+            }
+            // Step 4: Initialize database schema
+            const { initializeSchema } = await import('../core/db.js');
+            await initializeSchema(db);
+            console.log('✓ Database connection established');
+            // Step 5: Initialize index manager
+            console.log('📇 Loading vector index...');
+            const indexManager = new IndexManager(indexPath, dbPath, modelDimensions, embeddingModel);
+            await indexManager.initialize();
+            console.log('✓ Vector index loaded successfully');
+            // Step 6: Create ContentResolver
+            console.log('📁 Initializing content resolver...');
+            const { ContentResolver } = await import('../core/content-resolver.js');
+            const contentResolver = new ContentResolver(db);
+            console.log('✓ Content resolver ready');
+            // Step 7: Create SearchEngine with dependency injection
+            const searchEngine = new SearchEngine(embedFn, indexManager, db, rerankFn, contentResolver);
+            // Step 8: Validate the setup
+            const stats = await searchEngine.getStats();
+            console.log(`✓ Search engine ready: ${stats.totalChunks} chunks indexed, mode: ${mode}, reranking ${stats.rerankingEnabled ? 'enabled' : 'disabled'}`);
+            console.log('🎉 PolymorphicSearchFactory: Mode-aware search engine initialized successfully');
+            return searchEngine;
+        }
+        catch (error) {
+            console.error('❌ PolymorphicSearchFactory: Failed to create search engine');
+            throw createFactoryCreationError('PolymorphicSearchFactory', error instanceof Error ? error.message : 'Unknown error', { operationContext: 'polymorphic search engine creation' });
+        }
+    }
+}
+//# sourceMappingURL=polymorphic-factory.js.map

package/dist/file-processor.js

@@ -346,24 +346,35 @@ function extractTitle(content, filePath) {
  * Cache for image-to-text pipeline to avoid reloading
  */
 let imageToTextPipeline = null;
+let imageToTextPipelinePromise = null;
 /**
- * Initialize the image-to-text pipeline
+ * Initialize the image-to-text pipeline with proper async locking
  */
 async function initializeImageToTextPipeline(modelName = 'Xenova/vit-gpt2-image-captioning') {
+    // Return cached pipeline if available
     if (imageToTextPipeline) {
         return imageToTextPipeline;
     }
-    try {
-        const { pipeline } = await import('@huggingface/transformers');
-        console.log(`Loading image-to-text model: ${modelName}`);
-        imageToTextPipeline = await pipeline('image-to-text', modelName);
-        console.log(`Successfully loaded image-to-text model: ${modelName}`);
-        return imageToTextPipeline;
-    }
-    catch (error) {
-        console.error(`Failed to load image-to-text model ${modelName}:`, error);
-        throw new Error(`Failed to initialize image-to-text pipeline: ${error instanceof Error ? error.message : String(error)}`);
+    // If pipeline is currently loading, wait for it
+    if (imageToTextPipelinePromise) {
+        return imageToTextPipelinePromise;
     }
+    // Start loading pipeline
+    imageToTextPipelinePromise = (async () => {
+        try {
+            const { pipeline } = await import('@huggingface/transformers');
+            console.log(`Loading image-to-text model: ${modelName}`);
+            imageToTextPipeline = await pipeline('image-to-text', modelName);
+            console.log(`Successfully loaded image-to-text model: ${modelName}`);
+            return imageToTextPipeline;
+        }
+        catch (error) {
+            console.error(`Failed to load image-to-text model ${modelName}:`, error);
+            imageToTextPipelinePromise = null; // Reset on error so it can be retried
+            throw new Error(`Failed to initialize image-to-text pipeline: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    })();
+    return imageToTextPipelinePromise;
 }
 /**
  * Parse PNG image dimensions from file buffer
@@ -545,8 +556,11 @@ async function extractImageMetadata(imagePath) {
 async function generateImageDescription(imagePath, options = DEFAULT_IMAGE_TO_TEXT_OPTIONS) {
     try {
         const pipeline = await initializeImageToTextPipeline(options.model);
-        // Generate description
-        const result = await pipeline(imagePath, {
+        // Load image using RawImage.fromURL which works with local file paths
+        const { RawImage } = await import('@huggingface/transformers');
+        const image = await RawImage.fromURL(imagePath);
+        // Generate description with loaded image
+        const result = await pipeline(image, {
             max_length: options.maxLength || 50,
             num_beams: 4,
             early_stopping: true
@@ -597,93 +611,6 @@ async function generateImageDescriptionsBatch(imagePaths, options = DEFAULT_IMAG
     }
     return results;
 }
-/**
- * Generate text descriptions for multiple images using optimized batch processing
- * Uses BatchProcessingOptimizer for memory-efficient processing of large image collections
- */
-async function generateImageDescriptionsBatchOptimized(imagePaths, options = DEFAULT_IMAGE_TO_TEXT_OPTIONS) {
-    // For small batches, use the existing implementation
-    if (imagePaths.length <= 10) {
-        return generateImageDescriptionsBatch(imagePaths, options);
-    }
-    try {
-        // Import batch processing optimizer
-        const { createImageBatchProcessor } = await import('./core/batch-processing-optimizer.js');
-        const batchProcessor = createImageBatchProcessor();
-        // Convert image paths to batch items
-        const batchItems = imagePaths.map(path => ({
-            content: path,
-            contentType: 'image',
-            metadata: { originalPath: path }
-        }));
-        // Create image description function
-        const imageDescriptionFunction = async (item) => {
-            try {
-                const result = await generateImageDescription(item.content, options);
-                return {
-                    embedding_id: `img_desc_${Date.now()}_${Math.random()}`,
-                    vector: new Float32Array([0]), // Placeholder vector
-                    contentType: 'image',
-                    metadata: {
-                        path: item.content,
-                        description: result.description,
-                        confidence: result.confidence,
-                        model: result.model
-                    }
-                };
-            }
-            catch (error) {
-                throw new Error(`Failed to generate description for ${item.content}: ${error instanceof Error ? error.message : String(error)}`);
-            }
-        };
-        // Process with optimization and progress reporting
-        const batchResult = await batchProcessor.processBatch(batchItems, imageDescriptionFunction, (stats) => {
-            console.log(`Image description progress: ${stats.processedItems}/${stats.totalItems} (${Math.round((stats.processedItems / stats.totalItems) * 100)}%)`);
-            console.log(`  Memory usage: ${stats.memoryUsageMB}MB (peak: ${stats.peakMemoryUsageMB}MB)`);
-            if (stats.failedItems > 0) {
-                console.log(`  Failed items: ${stats.failedItems}`);
-            }
-        });
-        // Log final statistics
-        console.log(`✓ Image description generation complete:`);
-        console.log(`  Processed: ${batchResult.stats.processedItems}/${batchResult.stats.totalItems}`);
-        console.log(`  Failed: ${batchResult.stats.failedItems}`);
-        console.log(`  Processing time: ${Math.round(batchResult.stats.processingTimeMs / 1000)}s`);
-        console.log(`  Rate: ${Math.round(batchResult.stats.itemsPerSecond)} images/sec`);
-        console.log(`  Peak memory usage: ${batchResult.stats.peakMemoryUsageMB}MB`);
-        if (batchResult.stats.retryCount > 0) {
-            console.log(`  Retries: ${batchResult.stats.retryCount}`);
-        }
-        // Convert results back to expected format
-        const results = [];
-        // Add successful results
-        for (const result of batchResult.results) {
-            if (result.metadata?.description) {
-                results.push({
-                    path: result.metadata.path,
-                    result: {
-                        description: result.metadata.description,
-                        confidence: result.metadata.confidence,
-                        model: result.metadata.model
-                    }
-                });
-            }
-        }
-        // Add failed results
-        for (const error of batchResult.errors) {
-            results.push({
-                path: error.item.content,
-                error: error.error
-            });
-        }
-        return results;
-    }
-    catch (error) {
-        console.warn(`Optimized batch processing failed, falling back to standard batch processing: ${error instanceof Error ? error.message : String(error)}`);
-        // Fall back to existing implementation
-        return generateImageDescriptionsBatch(imagePaths, options);
-    }
-}
 /**
  * Process image file to extract text description and metadata
  */
@@ -834,8 +761,8 @@ export async function processFiles(filePaths, pathManager, imageToTextOptions) {
     if (imageFiles.length > 0) {
         console.log(`Processing ${imageFiles.length} image files with optimized batch processing`);
         try {
-            // Use optimized batch processing for image descriptions
-            const batchResults = await generateImageDescriptionsBatchOptimized(imageFiles, imageToTextOptions);
+            // Use batch processing for image descriptions
+            const batchResults = await generateImageDescriptionsBatch(imageFiles, imageToTextOptions);
             // Convert batch results to documents with metadata extraction
             for (const batchResult of batchResults) {
                 try {
@@ -961,6 +888,7 @@ export async function cleanupImageProcessingResources() {
                 await imageToTextPipeline.dispose();
             }
             imageToTextPipeline = null;
+            imageToTextPipelinePromise = null;
             console.log('Image-to-text pipeline cleaned up');
         }
         catch (error) {

package/dist/index.d.ts

@@ -41,8 +41,31 @@
  * ```
  */
 export { TextSearchFactory, TextIngestionFactory, TextRAGFactory, TextFactoryHelpers } from './factories/index.js';
+/**
+ * @deprecated PolymorphicSearchFactory is no longer needed - SearchEngine now automatically
+ * detects mode from database and adapts accordingly (Chameleon Architecture).
+ *
+ * Migration Guide:
+ * ```typescript
+ * // Old way (deprecated):
+ * const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
+ *
+ * // New way (recommended):
+ * const search = new SearchEngine('./index.bin', './db.sqlite');
+ * await search.search('query'); // Mode automatically detected
+ * ```
+ *
+ * The SearchEngine constructor now uses the polymorphic factory internally,
+ * providing the same automatic mode detection without requiring explicit factory usage.
+ */
+export { PolymorphicSearchFactory } from './factories/index.js';
 export { TextSearchFactory as SearchFactory, TextIngestionFactory as IngestionFactory, TextRAGFactory as RAGFactory } from './factories/index.js';
 export type { TextSearchOptions, TextIngestionOptions } from './factories/index.js';
+/**
+ * @deprecated PolymorphicSearchOptions is no longer needed - use SearchEngineOptions instead.
+ * SearchEngine now automatically detects mode and adapts (Chameleon Architecture).
+ */
+export type { PolymorphicSearchOptions } from './factories/index.js';
 export type { TextSearchOptions as SearchEngineOptions, TextIngestionOptions as IngestionPipelineOptions } from './factories/index.js';
 export { SearchEngine as CoreSearchEngine } from './core/search.js';
 export { IngestionPipeline as CoreIngestionPipeline } from './core/ingestion.js';

package/dist/index.js

@@ -45,6 +45,24 @@
 // =============================================================================
 // Main factory classes for simple usage
 export { TextSearchFactory, TextIngestionFactory, TextRAGFactory, TextFactoryHelpers } from './factories/index.js';
+/**
+ * @deprecated PolymorphicSearchFactory is no longer needed - SearchEngine now automatically
+ * detects mode from database and adapts accordingly (Chameleon Architecture).
+ *
+ * Migration Guide:
+ * ```typescript
+ * // Old way (deprecated):
+ * const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
+ *
+ * // New way (recommended):
+ * const search = new SearchEngine('./index.bin', './db.sqlite');
+ * await search.search('query'); // Mode automatically detected
+ * ```
+ *
+ * The SearchEngine constructor now uses the polymorphic factory internally,
+ * providing the same automatic mode detection without requiring explicit factory usage.
+ */
+export { PolymorphicSearchFactory } from './factories/index.js';
 // Convenience aliases for common usage
 export { TextSearchFactory as SearchFactory, TextIngestionFactory as IngestionFactory, TextRAGFactory as RAGFactory } from './factories/index.js';
 // =============================================================================

package/dist/ingestion.js

@@ -64,7 +64,12 @@ export class IngestionPipeline {
         if (!this.corePipeline) {
             throw new Error('IngestionPipeline failed to initialize');
         }
-        return this.corePipeline.ingestFile(filePath, options);
+        // Merge mode from constructor options with runtime options
+        const mergedOptions = {
+            ...options,
+            mode: options?.mode || this.options.mode
+        };
+        return this.corePipeline.ingestFile(filePath, mergedOptions);
     }
     /**
      * Ingest all documents in a directory
@@ -74,7 +79,12 @@ export class IngestionPipeline {
         if (!this.corePipeline) {
             throw new Error('IngestionPipeline failed to initialize');
         }
-        return this.corePipeline.ingestDirectory(directoryPath, options);
+        // Merge mode from constructor options with runtime options
+        const mergedOptions = {
+            ...options,
+            mode: options?.mode || this.options.mode
+        };
+        return this.corePipeline.ingestDirectory(directoryPath, mergedOptions);
     }
     /**
      * Ingest content from memory buffer
@@ -95,7 +105,12 @@ export class IngestionPipeline {
         if (!this.corePipeline) {
             throw new Error('IngestionPipeline failed to initialize');
         }
-        return this.corePipeline.ingestFromMemory(content, metadata, options);
+        // Merge mode from constructor options with runtime options
+        const mergedOptions = {
+            ...options,
+            mode: options?.mode || this.options.mode
+        };
+        return this.corePipeline.ingestFromMemory(content, metadata, mergedOptions);
     }
     /**
      * Clean up resources

package/dist/multimodal/clip-embedder.d.ts

@@ -84,6 +84,19 @@ export declare class CLIPEmbedder extends BaseUniversalEmbedder {
      * during cleanup - errors are logged but don't prevent cleanup completion.
      */
     cleanup(): Promise<void>;
+    /**
+     * Apply L2-normalization to an embedding vector
+     *
+     * L2-normalization ensures that all embeddings have unit length (magnitude = 1),
+     * which is essential for CLIP models as they were trained with normalized embeddings.
+     * This normalization makes cosine similarity calculations more reliable and ensures
+     * that vector magnitudes don't affect similarity scores.
+     *
+     * @param embedding - The embedding vector to normalize (modified in-place)
+     * @returns The normalized embedding vector (same reference as input)
+     * @private
+     */
+    private normalizeEmbedding;
     /**
      * Embed text using CLIP text encoder
      *
@@ -91,11 +104,11 @@ export declare class CLIPEmbedder extends BaseUniversalEmbedder {
      * pixel_values errors. Text is tokenized with CLIP's 77 token limit and
      * automatically truncated if necessary.
      *
-     * Returns a 512-dimensional embedding vector in the unified CLIP embedding space,
-     * which is directly comparable to image embeddings for cross-modal search.
+     * Returns a 512-dimensional L2-normalized embedding vector in the unified CLIP
+     * embedding space, which is directly comparable to image embeddings for cross-modal search.
      *
      * @param text - The text to embed (will be trimmed and validated)
-     * @returns EmbeddingResult with 512-dimensional vector and metadata
+     * @returns EmbeddingResult with 512-dimensional normalized vector and metadata
      * @throws {Error} If text is empty, model not loaded, or embedding fails
      *
      * @example
@@ -117,10 +130,10 @@ export declare class CLIPEmbedder extends BaseUniversalEmbedder {
      * - Converted to proper pixel_values format using AutoProcessor
      * - Normalized for CLIP vision model
      *
-     * Returns a 512-dimensional embedding vector directly comparable to text embeddings.
+     * Returns a 512-dimensional L2-normalized embedding vector directly comparable to text embeddings.
      *
      * @param imagePath - Local file path or URL to the image
-     * @returns EmbeddingResult with 512-dimensional vector and metadata
+     * @returns EmbeddingResult with 512-dimensional normalized vector and metadata
      * @throws {Error} If image not found, unsupported format, or embedding fails
      *
      * @example