rag-lite-ts 1.0.2 → 2.0.1

package/dist/factories/polymorphic-factory.js

@@ -0,0 +1,159 @@
+/**
+ * Polymorphic factory for creating mode-aware search engines
+ * Automatically detects mode from database and uses appropriate embedder
+ *
+ * This factory implements the Chameleon Architecture principle:
+ * - Detects mode (text/multimodal) from database configuration
+ * - Uses appropriate embedder based on detected mode
+ * - Provides seamless polymorphic behavior without user intervention
+ *
+ * @example
+ * ```typescript
+ * // Automatically detects mode and creates appropriate search engine
+ * const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
+ *
+ * // Works for both text and multimodal modes
+ * const results = await search.search('query');
+ * ```
+ */
+import { SearchEngine } from '../core/search.js';
+import { IndexManager } from '../index-manager.js';
+import { openDatabase, getSystemInfo } from '../core/db.js';
+import { createTextEmbedFunction } from '../text/embedder.js';
+import { createTextRerankFunction } from '../text/reranker.js';
+import { config, getModelDefaults } from '../core/config.js';
+import { existsSync } from 'fs';
+import { createMissingFileError, createInvalidPathError, createFactoryCreationError } from '../core/actionable-error-messages.js';
+/**
+ * Factory for creating mode-aware search engines
+ * Automatically detects mode from database and uses appropriate embedder
+ */
+export class PolymorphicSearchFactory {
+    /**
+     * Create a SearchEngine that automatically adapts to the mode stored in the database
+     *
+     * This method:
+     * 1. Validates that required files exist
+     * 2. Opens database and reads system configuration
+     * 3. Detects mode (text/multimodal) from database
+     * 4. Creates appropriate embedder based on mode
+     * 5. Optionally creates reranker based on configuration
+     * 6. Returns fully configured SearchEngine
+     *
+     * @param indexPath - Path to the vector index file (must exist)
+     * @param dbPath - Path to the SQLite database file (must exist)
+     * @param options - Optional configuration overrides
+     * @returns Promise resolving to configured SearchEngine
+     * @throws {Error} If required files don't exist or initialization fails
+     */
+    static async create(indexPath, dbPath, options = {}) {
+        try {
+            console.log('🏭 PolymorphicSearchFactory: Initializing mode-aware search engine...');
+            // Validate input paths
+            if (!indexPath || !dbPath) {
+                throw createInvalidPathError([
+                    { name: 'indexPath', value: indexPath },
+                    { name: 'dbPath', value: dbPath }
+                ], { operationContext: 'PolymorphicSearchFactory.create' });
+            }
+            // Check if required files exist
+            if (!existsSync(indexPath)) {
+                throw createMissingFileError(indexPath, 'index', {
+                    operationContext: 'PolymorphicSearchFactory.create'
+                });
+            }
+            if (!existsSync(dbPath)) {
+                throw createMissingFileError(dbPath, 'database', {
+                    operationContext: 'PolymorphicSearchFactory.create'
+                });
+            }
+            // Step 1: Open database and detect mode
+            console.log('💾 Opening database and detecting mode...');
+            const db = await openDatabase(dbPath);
+            let mode = 'text';
+            let embeddingModel;
+            let modelDimensions;
+            try {
+                const systemInfo = await getSystemInfo(db);
+                if (systemInfo) {
+                    mode = systemInfo.mode;
+                    embeddingModel = systemInfo.modelName;
+                    modelDimensions = systemInfo.modelDimensions;
+                    console.log(`📊 Detected mode: ${mode}`);
+                    console.log(`📊 Detected model: ${embeddingModel} (${modelDimensions} dimensions)`);
+                }
+                else {
+                    // Fallback to default if no system info
+                    embeddingModel = config.embedding_model;
+                    const modelDefaults = getModelDefaults(embeddingModel);
+                    modelDimensions = modelDefaults.dimensions;
+                    console.log(`📊 No system info found, using default: ${embeddingModel} (${modelDimensions} dimensions)`);
+                }
+            }
+            catch (error) {
+                // If getSystemInfo fails, use defaults
+                embeddingModel = config.embedding_model;
+                const modelDefaults = getModelDefaults(embeddingModel);
+                modelDimensions = modelDefaults.dimensions;
+                console.log(`📊 Using default configuration: ${embeddingModel} (${modelDimensions} dimensions)`);
+            }
+            // Step 2: Create appropriate embedder based on mode
+            let embedFn;
+            if (mode === 'multimodal') {
+                console.log('📊 Loading CLIP embedder for multimodal mode...');
+                const { createEmbedder } = await import('../core/embedder-factory.js');
+                const clipEmbedder = await createEmbedder(embeddingModel);
+                // Wrap CLIP embedder to match EmbedFunction signature
+                embedFn = async (content, contentType) => {
+                    if (contentType === 'image') {
+                        return await clipEmbedder.embedImage(content);
+                    }
+                    return await clipEmbedder.embedText(content);
+                };
+                console.log('✓ CLIP embedder loaded for multimodal mode');
+            }
+            else {
+                console.log('📊 Loading text embedder for text mode...');
+                embedFn = createTextEmbedFunction(embeddingModel);
+                console.log('✓ Text embedder loaded');
+            }
+            // Step 3: Initialize reranking function (optional)
+            let rerankFn;
+            if (options.enableReranking === true) {
+                console.log('🔄 Loading reranking model...');
+                rerankFn = createTextRerankFunction();
+                await rerankFn('test query', []);
+                console.log('✓ Reranking model loaded successfully');
+            }
+            else {
+                console.log('🔄 Reranking disabled (local-first, fast mode)');
+            }
+            // Step 4: Initialize database schema
+            const { initializeSchema } = await import('../core/db.js');
+            await initializeSchema(db);
+            console.log('✓ Database connection established');
+            // Step 5: Initialize index manager
+            console.log('📇 Loading vector index...');
+            const indexManager = new IndexManager(indexPath, dbPath, modelDimensions, embeddingModel);
+            await indexManager.initialize();
+            console.log('✓ Vector index loaded successfully');
+            // Step 6: Create ContentResolver
+            console.log('📁 Initializing content resolver...');
+            const { ContentResolver } = await import('../core/content-resolver.js');
+            const contentResolver = new ContentResolver(db);
+            console.log('✓ Content resolver ready');
+            // Step 7: Create SearchEngine with dependency injection
+            const searchEngine = new SearchEngine(embedFn, indexManager, db, rerankFn, contentResolver);
+            // Step 8: Validate the setup
+            const stats = await searchEngine.getStats();
+            console.log(`✓ Search engine ready: ${stats.totalChunks} chunks indexed, mode: ${mode}, reranking ${stats.rerankingEnabled ? 'enabled' : 'disabled'}`);
+            console.log('🎉 PolymorphicSearchFactory: Mode-aware search engine initialized successfully');
+            return searchEngine;
+        }
+        catch (error) {
+            console.error('❌ PolymorphicSearchFactory: Failed to create search engine');
+            throw createFactoryCreationError('PolymorphicSearchFactory', error instanceof Error ? error.message : 'Unknown error', { operationContext: 'polymorphic search engine creation' });
+        }
+    }
+}
+//# sourceMappingURL=polymorphic-factory.js.map

package/dist/factories/text-factory.d.ts

@@ -5,10 +5,21 @@
  * FACTORY PATTERN BENEFITS:
  * - Abstracts complex initialization (model loading, database setup, index initialization)
  * - Provides simple API for common use cases while preserving access to dependency injection
- * - Handles error recovery and validation
+ * - Clear validation and error handling without fallback mechanisms
  * - Supports different embedding models and configurations
  * - Enables clean separation between simple usage and advanced customization
  *
+ * MODE SELECTION GUIDE:
+ * - Text Mode (default): Optimized for text-only content
+ *   - Uses sentence-transformer models (fast, accurate for text)
+ *   - Images converted to text descriptions
+ *   - Best for: document search, text clustering, semantic similarity
+ *
+ * - Multimodal Mode: Optimized for mixed text/image content
+ *   - Uses CLIP models (unified embedding space)
+ *   - True cross-modal search (text finds images, images find text)
+ *   - Best for: image search, visual QA, multimodal retrieval
+ *
  * USAGE PATTERNS:
  *
  * 1. Simple Search Setup:
@@ -43,15 +54,31 @@
  * const results = await searchEngine.search('query');
  * ```
  *
- * 4. Error Recovery:
+ * 4. Clear Error Handling:
  * ```typescript
- * // Create with automatic fallback options
- * const search = await TextFactoryHelpers.createSearchWithFallback(
+ * // Create with clear validation and error reporting
+ * const search = await TextFactoryHelpers.createSearchWithValidation(
  *   './index.bin',
  *   './db.sqlite',
- *   { enableReranking: true } // Will fallback to disabled if reranking fails
+ *   { enableReranking: true } // Clear errors if issues occur
  * );
  * ```
+ *
+ * 5. Mode Selection:
+ * ```typescript
+ * // Text mode (default) - optimized for text-only content
+ * const textIngestion = await TextIngestionFactory.create('./db.sqlite', './index.bin', {
+ *   mode: 'text',
+ *   embeddingModel: 'sentence-transformers/all-MiniLM-L6-v2'
+ * });
+ *
+ * // Multimodal mode - enables cross-modal search
+ * const multimodalIngestion = await TextIngestionFactory.create('./db.sqlite', './index.bin', {
+ *   mode: 'multimodal',
+ *   embeddingModel: 'Xenova/clip-vit-base-patch32',
+ *   rerankingStrategy: 'text-derived'
+ * });
+ * ```
  */
 import { SearchEngine } from '../core/search.js';
 import { IngestionPipeline } from '../core/ingestion.js';
@@ -70,6 +97,21 @@ export interface TextSearchOptions {
     /** Top-k results to return (default: from config) */
     topK?: number;
 }
+/**
+ * Content system configuration options
+ */
+export interface ContentSystemConfig {
+    /** Content directory path (default: '.raglite/content') */
+    contentDir?: string;
+    /** Maximum file size in bytes (default: 50MB) */
+    maxFileSize?: number;
+    /** Maximum content directory size in bytes (default: 2GB) */
+    maxContentDirSize?: number;
+    /** Enable content deduplication (default: true) */
+    enableDeduplication?: boolean;
+    /** Enable storage tracking (default: true) */
+    enableStorageTracking?: boolean;
+}
 /**
  * Options for text ingestion factory
  */
@@ -84,17 +126,30 @@ export interface TextIngestionOptions {
     chunkOverlap?: number;
     /** Whether to force rebuild the index */
     forceRebuild?: boolean;
+    /** Mode for the ingestion pipeline (text or multimodal) */
+    mode?: 'text' | 'multimodal';
+    /** Reranking strategy for multimodal mode */
+    rerankingStrategy?: 'cross-encoder' | 'text-derived' | 'metadata' | 'hybrid' | 'disabled';
+    /** Content system configuration */
+    contentSystemConfig?: ContentSystemConfig;
 }
 /**
  * Factory for creating text-based SearchEngine instances
  * Handles model loading, database initialization, and index setup
  *
  * This factory abstracts the complex initialization process required for text search:
- * 1. Loads and validates text embedding models
- * 2. Optionally loads reranking models with fallback handling
- * 3. Establishes database connections and initializes schema
- * 4. Loads vector indexes with proper model compatibility checking
- * 5. Creates SearchEngine with proper dependency injection
+ * 1. Auto-detects embedding model from database configuration
+ * 2. Validates mode-model compatibility (no fallback mechanisms)
+ * 3. Loads embedding models with clear error reporting
+ * 4. Optionally loads reranking models based on configuration
+ * 5. Establishes database connections and initializes schema
+ * 6. Loads vector indexes with proper model compatibility checking
+ * 7. Creates SearchEngine with proper dependency injection
+ *
+ * Mode Support:
+ * - Automatically detects mode from database (text or multimodal)
+ * - Each mode uses its optimal implementation without fallbacks
+ * - Clear validation ensures mode-model compatibility
  *
  * @example
  * ```typescript
@@ -122,7 +177,7 @@ export declare class TextSearchFactory {
      * This method handles the complete initialization process:
      * - Validates that required files exist
      * - Loads text embedding model (with lazy initialization)
-     * - Optionally loads reranking model (with graceful fallback)
+     * - Optionally loads reranking model (with clear error reporting)
      * - Opens database connection and initializes schema
      * - Loads vector index with compatibility validation
      * - Creates SearchEngine with dependency injection
@@ -187,10 +242,18 @@ export declare class TextSearchFactory {
  *
  * This factory abstracts the complex initialization process required for text ingestion:
  * 1. Creates necessary directories if they don't exist
- * 2. Loads and validates text embedding models
- * 3. Establishes database connections and initializes schema
- * 4. Creates or loads vector indexes with proper configuration
- * 5. Creates IngestionPipeline with proper dependency injection
+ * 2. Validates mode-model compatibility (no fallback mechanisms)
+ * 3. Loads and validates embedding models with clear error reporting
+ * 4. Establishes database connections and initializes schema
+ * 5. Stores mode configuration in database for automatic detection
+ * 6. Creates or loads vector indexes with proper configuration
+ * 7. Creates IngestionPipeline with proper dependency injection
+ *
+ * Mode Configuration:
+ * - Text Mode (default): Uses sentence-transformer models for text-only content
+ * - Multimodal Mode: Uses CLIP models for mixed text/image content
+ * - Mode is stored in database and auto-detected during search
+ * - Clear validation prevents mode-model mismatches
  *
  * @example
  * ```typescript
@@ -232,20 +295,39 @@ export declare class TextIngestionFactory {
      * @param options.chunkSize - Override chunk size (default: from config)
      * @param options.chunkOverlap - Override chunk overlap (default: from config)
      * @param options.forceRebuild - Force rebuild of existing index (default: false)
+     * @param options.contentSystemConfig - Content system configuration options
+     * @param options.contentSystemConfig.contentDir - Content directory path (default: '.raglite/content')
+     * @param options.contentSystemConfig.maxFileSize - Maximum file size in bytes (default: 50MB)
+     * @param options.contentSystemConfig.maxContentDirSize - Maximum content directory size (default: 2GB)
+     * @param options.contentSystemConfig.enableDeduplication - Enable content deduplication (default: true)
+     * @param options.contentSystemConfig.enableStorageTracking - Enable storage tracking (default: true)
      * @returns Promise resolving to configured IngestionPipeline
      * @throws {Error} If initialization fails
      *
      * @example
      * ```typescript
-     * // Create ingestion pipeline
+     * // Create ingestion pipeline with default content system
      * const ingestion = await TextIngestionFactory.create('./my-db.sqlite', './my-index.bin');
      *
+     * // Create with custom content system configuration
+     * const ingestion = await TextIngestionFactory.create('./my-db.sqlite', './my-index.bin', {
+     *   contentSystemConfig: {
+     *     contentDir: './custom-content',
+     *     maxFileSize: 100 * 1024 * 1024, // 100MB
+     *     maxContentDirSize: 5 * 1024 * 1024 * 1024, // 5GB
+     *     enableDeduplication: true
+     *   }
+     * });
+     *
      * // Ingest documents from directory
      * const result = await ingestion.ingestDirectory('./documents');
      * console.log(`Processed ${result.documentsProcessed} documents`);
      *
-     * // Ingest single file
-     * await ingestion.ingestFile('./document.pdf');
+     * // Ingest content from memory (MCP integration)
+     * const contentId = await ingestion.ingestFromMemory(buffer, {
+     *   displayName: 'uploaded-file.pdf',
+     *   contentType: 'application/pdf'
+     * });
      *
      * // Clean up when done
      * await ingestion.cleanup();
@@ -259,6 +341,22 @@ export declare class TextIngestionFactory {
      * @returns Promise resolving to configured IngestionPipeline
      */
     static createWithDefaults(options?: TextIngestionOptions): Promise<IngestionPipeline>;
+    /**
+     * Handles mode storage during ingestion
+     * Creates or validates system info based on the provided mode and options
+     * @private
+     */
+    private static handleModeStorage;
+    /**
+     * Updates system info in the database
+     * @private
+     */
+    private static updateSystemInfo;
+    /**
+     * Validates and prepares content system configuration
+     * @private
+     */
+    private static validateAndPrepareContentSystemConfig;
 }
 /**
  * Convenience factory to create both search and ingestion instances
@@ -365,9 +463,9 @@ export declare class TextRAGFactory {
  * const { searchOptions, ingestionOptions } = TextFactoryHelpers.getRecommendedConfig('quality');
  * const search = await TextSearchFactory.create('./index.bin', './db.sqlite', searchOptions);
  *
- * // Create with automatic error recovery
- * const search = await TextFactoryHelpers.createSearchWithFallback('./index.bin', './db.sqlite', {
- *   enableReranking: true // Will fallback to disabled if reranking fails
+ * // Create with clear validation and error reporting
+ * const search = await TextFactoryHelpers.createSearchWithValidation('./index.bin', './db.sqlite', {
+ *   enableReranking: true // Will fail clearly if reranking has issues
  * });
  * ```
  */
@@ -433,34 +531,30 @@ export declare class TextFactoryHelpers {
         ingestionOptions: TextIngestionOptions;
     };
     /**
-     * Create a search engine with automatic error recovery
+     * Create a search engine with clear error reporting
      *
-     * This method attempts to create a search engine with the provided options,
-     * and if that fails, it tries again with fallback options (primarily
-     * disabling reranking, which is a common source of initialization failures).
-     * This provides a more robust way to create search engines in environments
-     * where reranking models might not be available or might fail to load.
+     * This method creates a search engine with the provided options and fails
+     * clearly if there are any issues, providing actionable error messages.
      *
      * @param indexPath - Path to vector index file
      * @param dbPath - Path to database file
-     * @param options - Initial options to try
-     * @returns Promise resolving to SearchEngine (possibly with fallback options)
-     * @throws {Error} If both original and fallback creation attempts fail
+     * @param options - Configuration options
+     * @returns Promise resolving to SearchEngine
+     * @throws {Error} If creation fails with clear error message
      *
      * @example
      * ```typescript
-     * // Try to create with reranking, fallback to without if it fails
-     * const search = await TextFactoryHelpers.createSearchWithFallback(
+     * // Create search engine with clear error handling
+     * const search = await TextFactoryHelpers.createSearchWithValidation(
      *   './index.bin',
      *   './db.sqlite',
      *   { enableReranking: true, topK: 20 }
      * );
      *
-     * // The search engine will work even if reranking model fails to load
      * const results = await search.search('query');
      * console.log(`Search created successfully with ${results.length} results`);
      * ```
      */
-    static createSearchWithFallback(indexPath: string, dbPath: string, options?: TextSearchOptions): Promise<SearchEngine>;
+    static createSearchWithValidation(indexPath: string, dbPath: string, options?: TextSearchOptions): Promise<SearchEngine>;
 }
 //# sourceMappingURL=text-factory.d.ts.map