rag-lite-ts 1.0.1 → 2.0.0

package/dist/cli/indexer.js

@@ -1,7 +1,110 @@
 import { existsSync, statSync } from 'fs';
 import { resolve } from 'path';
-import { IngestionPipeline, rebuildIndex } from '../ingestion.js';
-import { EXIT_CODES, ConfigurationError } from '../config.js';
+import { TextIngestionFactory } from '../factories/text-factory.js';
+import { withCLIDatabaseAccess, setupCLICleanup, isDatabaseBusy } from '../core/cli-database-utils.js';
+import { EXIT_CODES, ConfigurationError } from '../core/config.js';
+/**
+ * Validate mode-specific model and strategy combinations
+ * Ensures that the selected model is compatible with the chosen mode
+ * and that reranking strategies are valid for the mode
+ */
+async function validateModeConfiguration(options) {
+    const mode = options.mode || 'text';
+    const model = options.embeddingModel;
+    const rerankingStrategy = options.rerankingStrategy;
+    // Define supported models for each mode
+    const textModels = [
+        'sentence-transformers/all-MiniLM-L6-v2',
+        'Xenova/all-mpnet-base-v2'
+    ];
+    const multimodalModels = [
+        'Xenova/clip-vit-base-patch32'
+    ];
+    // Validate model compatibility with mode
+    if (model) {
+        if (mode === 'text' && !textModels.includes(model)) {
+            if (multimodalModels.includes(model)) {
+                throw new ConfigurationError(`Model '${model}' is a multimodal model but text mode was selected.\n` +
+                    `\n` +
+                    `To use this model, specify multimodal mode:\n` +
+                    `  raglite ingest <path> --mode multimodal --model ${model}\n` +
+                    `\n` +
+                    `Or choose a text model for text mode:\n` +
+                    `  ${textModels.map(m => `raglite ingest <path> --model ${m}`).join('\n  ')}\n`, EXIT_CODES.INVALID_ARGUMENTS);
+            }
+            else {
+                throw new ConfigurationError(`Model '${model}' is not supported for text mode.\n` +
+                    `\n` +
+                    `Supported models for text mode:\n` +
+                    `  ${textModels.join('\n  ')}\n` +
+                    `\n` +
+                    `Examples:\n` +
+                    `  raglite ingest <path> --model sentence-transformers/all-MiniLM-L6-v2\n` +
+                    `  raglite ingest <path> --model Xenova/all-mpnet-base-v2\n`, EXIT_CODES.INVALID_ARGUMENTS);
+            }
+        }
+        if (mode === 'multimodal' && !multimodalModels.includes(model)) {
+            if (textModels.includes(model)) {
+                throw new ConfigurationError(`Model '${model}' is a text-only model but multimodal mode was selected.\n` +
+                    `\n` +
+                    `To use this model, specify text mode:\n` +
+                    `  raglite ingest <path> --mode text --model ${model}\n` +
+                    `\n` +
+                    `Or choose a multimodal model for multimodal mode:\n` +
+                    `  ${multimodalModels.map(m => `raglite ingest <path> --mode multimodal --model ${m}`).join('\n  ')}\n`, EXIT_CODES.INVALID_ARGUMENTS);
+            }
+            else {
+                throw new ConfigurationError(`Model '${model}' is not supported for multimodal mode.\n` +
+                    `\n` +
+                    `Supported models for multimodal mode:\n` +
+                    `  ${multimodalModels.join('\n  ')}\n` +
+                    `\n` +
+                    `Example:\n` +
+                    `  raglite ingest <path> --mode multimodal --model Xenova/clip-vit-base-patch32\n`, EXIT_CODES.INVALID_ARGUMENTS);
+            }
+        }
+    }
+    // Validate reranking strategy compatibility with mode
+    if (rerankingStrategy) {
+        const textStrategies = ['cross-encoder', 'disabled'];
+        const multimodalStrategies = ['text-derived', 'metadata', 'disabled'];
+        if (mode === 'text' && !textStrategies.includes(rerankingStrategy)) {
+            throw new ConfigurationError(`Reranking strategy '${rerankingStrategy}' is not supported for text mode.\n` +
+                `\n` +
+                `Supported strategies for text mode:\n` +
+                `  cross-encoder  Use cross-encoder model for reranking (default)\n` +
+                `  disabled       No reranking, use vector similarity only\n` +
+                `\n` +
+                `Examples:\n` +
+                `  raglite ingest <path> --mode text --rerank-strategy cross-encoder\n` +
+                `  raglite ingest <path> --mode text --rerank-strategy disabled\n`, EXIT_CODES.INVALID_ARGUMENTS);
+        }
+        if (mode === 'multimodal' && !multimodalStrategies.includes(rerankingStrategy)) {
+            throw new ConfigurationError(`Reranking strategy '${rerankingStrategy}' is not supported for multimodal mode.\n` +
+                `\n` +
+                `Supported strategies for multimodal mode:\n` +
+                `  text-derived  Convert images to text, then use cross-encoder (default)\n` +
+                `  metadata      Use filename and metadata-based scoring\n` +
+                `  disabled      No reranking, use vector similarity only\n` +
+                `\n` +
+                `Examples:\n` +
+                `  raglite ingest <path> --mode multimodal --rerank-strategy text-derived\n` +
+                `  raglite ingest <path> --mode multimodal --rerank-strategy metadata\n` +
+                `  raglite ingest <path> --mode multimodal --rerank-strategy disabled\n`, EXIT_CODES.INVALID_ARGUMENTS);
+        }
+    }
+    // Log the final configuration
+    console.log('✅ Mode configuration validated successfully');
+    if (mode !== 'text') {
+        console.log(`   Mode: ${mode}`);
+    }
+    if (model) {
+        console.log(`   Model: ${model}`);
+    }
+    if (rerankingStrategy) {
+        console.log(`   Reranking: ${rerankingStrategy}`);
+    }
+}
 /**
  * Run document ingestion from CLI
  * @param path - File or directory path to ingest
@@ -9,6 +112,7 @@ import { EXIT_CODES, ConfigurationError } from '../config.js';
  */
 export async function runIngest(path, options = {}) {
     try {
+        // Handle --rebuild-if-needed flag immediately to prevent dimension mismatch error
         // Validate path exists
         const resolvedPath = resolve(path);
         if (!existsSync(resolvedPath)) {
@@ -58,50 +162,59 @@ export async function runIngest(path, options = {}) {
         console.log(`Starting ingestion of ${pathType}: ${resolvedPath}`);
         console.log('This may take a while for large document collections...');
         console.log('');
-        // Create and run ingestion pipeline
-        const configOverrides = {};
+        // Prepare factory options
+        const factoryOptions = {};
         if (options.model) {
-            configOverrides.embedding_model = options.model;
+            factoryOptions.embeddingModel = options.model;
             console.log(`Using embedding model: ${options.model}`);
         }
-        const pipeline = new IngestionPipeline();
-        pipeline.setConfigOverrides(configOverrides);
-        // Set path storage strategy if specified
-        if (options['path-strategy'] || options['path-base']) {
-            const strategy = options['path-strategy'] || 'relative';
-            const basePath = options['path-base'] || process.cwd();
-            pipeline.setPathStorageStrategy(strategy, basePath);
-            console.log(`Using path storage strategy: ${strategy} (base: ${basePath})`);
+        if (options.mode) {
+            factoryOptions.mode = options.mode;
+            console.log(`Using processing mode: ${options.mode}`);
         }
-        try {
-            // Handle automatic rebuild if needed
-            if (options.rebuildIfNeeded) {
+        if (options['rerank-strategy']) {
+            factoryOptions.rerankingStrategy = options['rerank-strategy'];
+            console.log(`Using reranking strategy: ${options['rerank-strategy']}`);
+        }
+        if (options.rebuildIfNeeded) {
+            factoryOptions.forceRebuild = true;
+            console.log('Force rebuild enabled due to rebuildIfNeeded option');
+            // Delete old index file immediately to prevent dimension mismatch errors
+            const indexPath = process.env.RAG_INDEX_FILE || './vector-index.bin';
+            const { existsSync, unlinkSync } = await import('fs');
+            if (existsSync(indexPath)) {
                 try {
-                    await pipeline.initialize();
+                    unlinkSync(indexPath);
+                    console.log('🗑️ Removed old index file to prevent dimension mismatch');
                 }
                 catch (error) {
-                    if (error instanceof Error && error.message.includes('Model mismatch detected')) {
-                        console.log('⚠️  Model mismatch detected. Rebuilding index automatically...');
-                        console.log('⚠️  WARNING: This will regenerate ALL embeddings and may take a while.');
-                        console.log('');
-                        // Create a new pipeline with the same config overrides for rebuild
-                        const rebuildPipeline = new IngestionPipeline();
-                        rebuildPipeline.setConfigOverrides(configOverrides);
-                        try {
-                            await rebuildPipeline.initialize();
-                            await rebuildPipeline.rebuildIndex();
-                        }
-                        finally {
-                            await rebuildPipeline.cleanup();
-                        }
-                        console.log('✅ Rebuild completed. Continuing with ingestion...');
-                        console.log('');
-                    }
-                    else {
-                        throw error;
-                    }
+                    console.warn(`⚠️ Could not remove old index file: ${error}`);
                 }
             }
+        }
+        // Validate mode-specific model and strategy combinations
+        await validateModeConfiguration(factoryOptions);
+        const dbPath = process.env.RAG_DB_FILE || './db.sqlite';
+        const indexPath = process.env.RAG_INDEX_FILE || './vector-index.bin';
+        // Setup graceful cleanup
+        setupCLICleanup(dbPath);
+        // Check if database is busy before starting
+        const busyStatus = await isDatabaseBusy(dbPath);
+        if (busyStatus.isBusy) {
+            console.log('⚠️  Database appears to be in use by another process');
+            console.log(`   Reason: ${busyStatus.reason}`);
+            console.log('   Attempting to proceed anyway...');
+            console.log('');
+        }
+        // Create ingestion pipeline using factory
+        let pipeline;
+        try {
+            // Create ingestion pipeline using TextIngestionFactory with database protection
+            pipeline = await withCLIDatabaseAccess(dbPath, () => TextIngestionFactory.create(dbPath, indexPath, factoryOptions), {
+                commandName: 'Ingestion command',
+                showProgress: true,
+                maxWaitMs: 15000 // Longer timeout for ingestion
+            });
             const result = await pipeline.ingestPath(resolvedPath);
             // Display final results
             console.log('\n' + '='.repeat(50));
@@ -122,10 +235,26 @@ export async function runIngest(path, options = {}) {
                 console.log(`Processing rate: ${chunksPerSecond} chunks/second`);
             }
             console.log('\nIngestion completed successfully!');
+            // Display mode-specific information
+            const mode = options.mode || 'text';
+            if (mode === 'multimodal') {
+                console.log('✨ Multimodal mode enabled - you can now search across text and image content');
+            }
             console.log('You can now search your documents using: raglite search "your query"');
+            console.log('');
+            console.log('💡 The search command will automatically detect and use the ingestion mode.');
         }
         finally {
-            await pipeline.cleanup();
+            if (pipeline) {
+                await pipeline.cleanup();
+            }
+            // Ensure clean exit for CLI commands
+            const { DatabaseConnectionManager } = await import('../core/database-connection-manager.js');
+            await DatabaseConnectionManager.closeAllConnections();
+            // Force exit for CLI commands to prevent hanging
+            setTimeout(() => {
+                process.exit(0);
+            }, 100);
         }
     }
     catch (error) {
@@ -205,14 +334,101 @@ export async function runRebuild() {
         console.log('');
         console.log('Progress will be shown below...');
         console.log('');
-        await rebuildIndex();
-        console.log('\n' + '='.repeat(50));
-        console.log('REBUILD COMPLETE');
-        console.log('='.repeat(50));
-        console.log('The vector index has been successfully rebuilt.');
-        console.log('All embeddings have been regenerated with the current model.');
-        console.log('');
-        console.log('You can now search your documents using: raglite search "your query"');
+        // Detect mode from existing database for rebuild
+        const dbPath = process.env.RAG_DB_FILE || './db.sqlite';
+        const indexPath = process.env.RAG_INDEX_FILE || './vector-index.bin';
+        let rebuildOptions = { forceRebuild: true };
+        if (existsSync(dbPath)) {
+            try {
+                // Import mode detection service
+                const { ModeDetectionService } = await import('../core/mode-detection-service.js');
+                const modeService = new ModeDetectionService(dbPath);
+                const systemInfo = await modeService.detectMode();
+                console.log(`🎯 Detected existing configuration:`);
+                console.log(`   Mode: ${systemInfo.mode}`);
+                console.log(`   Model: ${systemInfo.modelName}`);
+                console.log(`   Reranking: ${systemInfo.rerankingStrategy}`);
+                console.log('');
+                // Use the detected configuration for rebuild
+                rebuildOptions.mode = systemInfo.mode;
+                rebuildOptions.embeddingModel = systemInfo.modelName;
+                rebuildOptions.rerankingStrategy = systemInfo.rerankingStrategy;
+            }
+            catch (error) {
+                console.warn('⚠️  Could not detect existing mode configuration, using defaults');
+                console.warn(`   Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            }
+        }
+        // Create ingestion pipeline with force rebuild using factory
+        const pipeline = await TextIngestionFactory.create(dbPath, indexPath, rebuildOptions);
+        try {
+            // Get all documents from database and re-ingest them
+            const { openDatabase } = await import('../core/db.js');
+            const db = await openDatabase(dbPath);
+            try {
+                const documents = await db.all('SELECT DISTINCT source FROM documents ORDER BY source');
+                if (documents.length === 0) {
+                    throw new Error('No documents found in database. Nothing to rebuild.');
+                }
+                console.log(`Found ${documents.length} document${documents.length === 1 ? '' : 's'} to rebuild`);
+                let totalResult = {
+                    documentsProcessed: 0,
+                    chunksCreated: 0,
+                    embeddingsGenerated: 0,
+                    documentErrors: 0,
+                    embeddingErrors: 0,
+                    processingTimeMs: 0
+                };
+                // Re-ingest each document
+                for (const doc of documents) {
+                    if (existsSync(doc.source)) {
+                        console.log(`Rebuilding: ${doc.source}`);
+                        const result = await pipeline.ingestFile(doc.source);
+                        totalResult.documentsProcessed += result.documentsProcessed;
+                        totalResult.chunksCreated += result.chunksCreated;
+                        totalResult.embeddingsGenerated += result.embeddingsGenerated;
+                        totalResult.documentErrors += result.documentErrors;
+                        totalResult.embeddingErrors += result.embeddingErrors;
+                        totalResult.processingTimeMs += result.processingTimeMs;
+                    }
+                    else {
+                        console.warn(`⚠️  Document not found, skipping: ${doc.source}`);
+                        totalResult.documentErrors++;
+                    }
+                }
+                console.log('\n' + '='.repeat(50));
+                console.log('REBUILD COMPLETE');
+                console.log('='.repeat(50));
+                console.log(`Documents processed: ${totalResult.documentsProcessed}`);
+                console.log(`Chunks created: ${totalResult.chunksCreated}`);
+                console.log(`Embeddings generated: ${totalResult.embeddingsGenerated}`);
+                if (totalResult.documentErrors > 0) {
+                    console.log(`Document errors: ${totalResult.documentErrors}`);
+                }
+                if (totalResult.embeddingErrors > 0) {
+                    console.log(`Embedding errors: ${totalResult.embeddingErrors}`);
+                }
+                console.log(`Total processing time: ${(totalResult.processingTimeMs / 1000).toFixed(2)} seconds`);
+                console.log('');
+                console.log('The vector index has been successfully rebuilt.');
+                console.log('All embeddings have been regenerated with the current model.');
+                console.log('');
+                console.log('You can now search your documents using: raglite search "your query"');
+            }
+            finally {
+                await db.close();
+            }
+        }
+        finally {
+            await pipeline.cleanup();
+            // Ensure clean exit for CLI commands
+            const { DatabaseConnectionManager } = await import('../core/database-connection-manager.js');
+            await DatabaseConnectionManager.closeAllConnections();
+            // Force exit for CLI commands to prevent hanging
+            setTimeout(() => {
+                process.exit(0);
+            }, 100);
+        }
     }
     catch (error) {
         console.error('\n' + '='.repeat(50));

package/dist/cli/search.js

@@ -1,6 +1,7 @@
 import { existsSync } from 'fs';
-import { openDatabase } from '../db.js';
-import { config, EXIT_CODES, ConfigurationError } from '../config.js';
+import { PolymorphicSearchFactory } from '../core/polymorphic-search-factory.js';
+import { withCLIDatabaseAccess, setupCLICleanup } from '../core/cli-database-utils.js';
+import { config, EXIT_CODES, ConfigurationError } from '../core/config.js';
 /**
  * Run search from CLI
  * @param query - Search query string
@@ -53,33 +54,17 @@ export async function runSearch(query, options = {}) {
             process.exit(EXIT_CODES.INDEX_ERROR);
         }
         console.log(`Searching for: "${query}"`);
-        console.log('Loading search engine...');
         console.log('');
-        // Initialize components
-        let db, embedder, indexManager, searchEngine;
+        // Setup graceful cleanup
+        setupCLICleanup(effectiveConfig.db_file);
+        // Initialize search engine using polymorphic factory with database protection
+        let searchEngine;
         try {
-            // Open database connection
-            db = await openDatabase(effectiveConfig.db_file);
-            // Read the model that was used during ingestion from the database
-            const { getStoredModelInfo } = await import('../db.js');
-            const storedModel = await getStoredModelInfo(db);
-            const modelToUse = storedModel ? storedModel.modelName : effectiveConfig.embedding_model;
-            console.log(`Using model from ingestion: ${modelToUse}`);
-            // Initialize embedding engine with the correct model
-            const { EmbeddingEngine } = await import('../embedder.js');
-            embedder = new EmbeddingEngine(modelToUse);
-            await embedder.loadModel();
-            // Initialize index manager (which handles the vector index and hash mapping)
-            const { IndexManager } = await import('../index-manager.js');
-            const { getModelDefaults } = await import('../config.js');
-            const modelDefaults = getModelDefaults(modelToUse);
-            indexManager = new IndexManager(effectiveConfig.index_file, effectiveConfig.db_file, modelDefaults.dimensions, modelToUse);
-            await indexManager.initialize();
-            // Create search engine with index manager
-            const enableReranking = options.rerank !== undefined ? options.rerank : effectiveConfig.rerank_enabled;
-            const { SearchEngine } = await import('../search.js');
-            searchEngine = SearchEngine.createWithComponents(embedder, indexManager, db, enableReranking);
-            await searchEngine.initialize();
+            // Create search engine using PolymorphicSearchFactory (auto-detects mode)
+            searchEngine = await withCLIDatabaseAccess(effectiveConfig.db_file, () => PolymorphicSearchFactory.create(effectiveConfig.index_file, effectiveConfig.db_file), {
+                commandName: 'Search command',
+                showProgress: true
+            });
             // Prepare search options
             const searchOptions = {};
             if (options['top-k'] !== undefined) {
@@ -90,8 +75,18 @@ export async function runSearch(query, options = {}) {
             }
             // Perform search
             const startTime = Date.now();
-            const results = await searchEngine.search(query, searchOptions);
+            let results = await searchEngine.search(query, searchOptions);
             const searchTime = Date.now() - startTime;
+            // Apply content type filter if specified
+            const contentTypeFilter = options['content-type'];
+            if (contentTypeFilter && contentTypeFilter !== 'all') {
+                const originalCount = results.length;
+                results = results.filter(r => r.contentType === contentTypeFilter);
+                if (results.length < originalCount) {
+                    console.log(`Filtered to ${results.length} ${contentTypeFilter} result${results.length === 1 ? '' : 's'} (from ${originalCount} total)`);
+                    console.log('');
+                }
+            }
             // Display results
             if (results.length === 0) {
                 console.log('No results found.');
@@ -103,10 +98,30 @@ export async function runSearch(query, options = {}) {
             else {
                 console.log(`Found ${results.length} result${results.length === 1 ? '' : 's'} in ${searchTime}ms:\n`);
                 results.forEach((result, index) => {
-                    console.log(`${index + 1}. ${result.document.title}`);
+                    // Add content type icon for visual distinction
+                    const contentTypeIcon = result.contentType === 'image' ? '🖼️ ' : '📄 ';
+                    const contentTypeLabel = result.contentType === 'image' ? '[IMAGE]' : '[TEXT]';
+                    console.log(`${index + 1}. ${contentTypeIcon}${result.document.title}`);
                     console.log(`   Source: ${result.document.source}`);
+                    console.log(`   Type: ${contentTypeLabel}`);
                     console.log(`   Score: ${(result.score * 100).toFixed(1)}%`);
-                    console.log(`   Text: ${truncateText(result.text, 200)}`);
+                    // Display content differently based on type
+                    if (result.contentType === 'image') {
+                        // For images, show metadata if available
+                        if (result.metadata?.description) {
+                            console.log(`   Description: ${truncateText(result.metadata.description, 200)}`);
+                        }
+                        if (result.metadata?.dimensions) {
+                            console.log(`   Dimensions: ${result.metadata.dimensions}`);
+                        }
+                        if (result.metadata?.format) {
+                            console.log(`   Format: ${result.metadata.format}`);
+                        }
+                    }
+                    else {
+                        // For text, show content preview
+                        console.log(`   Text: ${truncateText(result.content, 200)}`);
+                    }
                     console.log('');
                 });
                 // Show search statistics
@@ -121,9 +136,16 @@ export async function runSearch(query, options = {}) {
         }
         finally {
             // Cleanup resources
-            if (db) {
-                await db.close();
+            if (searchEngine) {
+                await searchEngine.cleanup();
             }
+            // Ensure clean exit for CLI commands
+            const { DatabaseConnectionManager } = await import('../core/database-connection-manager.js');
+            await DatabaseConnectionManager.closeAllConnections();
+            // Force exit for CLI commands to prevent hanging
+            setTimeout(() => {
+                process.exit(0);
+            }, 100);
         }
     }
     catch (error) {