rag-lite-ts 1.0.2 → 2.0.0

package/dist/core/ingestion.js

@@ -8,6 +8,7 @@ import { insertChunk, upsertDocument } from './db.js';
 import { config } from './config.js';
 import { DocumentPathManager } from './path-manager.js';
 import { existsSync } from 'fs';
+import { ContentManager } from './content-manager.js';
 /**
  * Main ingestion pipeline class
  * Coordinates the entire process from file discovery to vector storage
@@ -19,8 +20,10 @@ export class IngestionPipeline {
     db;
     defaultChunkConfig;
     pathManager;
+    contentManager;
     /**
      * Creates a new IngestionPipeline with explicit dependency injection
+     * Enhanced with ContentManager integration for unified content system
      *
      * DEPENDENCY INJECTION PATTERN:
      * This constructor requires all dependencies to be explicitly provided, enabling:
@@ -28,6 +31,7 @@ export class IngestionPipeline {
      * - Support for different embedding models and content types
      * - Testability through mock injection
      * - Future extensibility for multimodal content processing
+     * - Unified content management for both filesystem and memory-based ingestion
      *
      * @param embedFn - Function to embed document chunks into vectors
      *   - Signature: (query: string, contentType?: string) => Promise<EmbeddingResult>
@@ -47,32 +51,33 @@ export class IngestionPipeline {
      *   - Supports different content types through metadata fields
      *   - Example: await openDatabase('./db.sqlite')
      *
+     * @param contentManager - Optional ContentManager for unified content system
+     *   - Handles content storage routing and deduplication
+     *   - If not provided, creates default instance with standard configuration
+     *   - Example: new ContentManager(db, { contentDir: '.raglite/content' })
+     *
      * USAGE EXAMPLES:
      * ```typescript
-     * // Text-only ingestion pipeline
+     * // Text-only ingestion pipeline with unified content system
      * const textEmbedFn = await createTextEmbedder();
      * const indexManager = new IndexManager('./index.bin');
      * const db = await openDatabase('./db.sqlite');
+     * const contentManager = new ContentManager(db);
+     * const ingestion = new IngestionPipeline(textEmbedFn, indexManager, db, undefined, contentManager);
+     *
+     * // Simple usage (ContentManager created automatically)
      * const ingestion = new IngestionPipeline(textEmbedFn, indexManager, db);
      *
-     * // Custom embedding implementation
+     * // Custom embedding implementation with memory ingestion
      * const customEmbedFn = async (text) => ({
      *   embedding_id: generateId(),
      *   vector: await myCustomModel.embed(text)
      * });
      * const ingestion = new IngestionPipeline(customEmbedFn, indexManager, db);
-     *
-     * // Multimodal ingestion (future)
-     * const multimodalEmbedFn = async (content, contentType) => {
-     *   if (contentType === 'image') {
-     *     return { embedding_id: generateId(), vector: await clipModel.embedImage(content) };
-     *   }
-     *   return { embedding_id: generateId(), vector: await clipModel.embedText(content) };
-     * };
-     * const ingestion = new IngestionPipeline(multimodalEmbedFn, indexManager, db);
+     * await ingestion.ingestFromMemory(buffer, { displayName: 'file.txt' });
      * ```
      */
-    constructor(embedFn, indexManager, db, defaultChunkConfig) {
+    constructor(embedFn, indexManager, db, defaultChunkConfig, contentManager) {
         this.embedFn = embedFn;
         this.indexManager = indexManager;
         this.db = db;
@@ -89,6 +94,8 @@ export class IngestionPipeline {
         }
         // Initialize path manager with default configuration
         this.pathManager = new DocumentPathManager(config.path_storage_strategy, process.cwd());
+        // Initialize ContentManager (create default if not provided)
+        this.contentManager = contentManager || new ContentManager(this.db);
     }
     /**
      * Ingest documents from a directory
@@ -114,15 +121,152 @@ export class IngestionPipeline {
         }
         return this.ingestPath(filePath, options);
     }
+    /**
+     * Ingest content from memory buffer
+     * Enables MCP integration and real-time content processing
+     * @param content - Buffer containing the content to ingest
+     * @param metadata - Memory content metadata including display name and content type
+     * @param options - Optional ingestion configuration
+     * @returns Promise resolving to content ID for the ingested content
+     */
+    async ingestFromMemory(content, metadata, options = {}) {
+        const startTime = Date.now();
+        console.log(`\n=== Starting memory ingestion: ${metadata.displayName} ===`);
+        try {
+            // Phase 1: Content Storage via ContentManager
+            console.log('\n--- Phase 1: Content Storage ---');
+            const contentResult = await this.contentManager.ingestFromMemory(content, metadata);
+            if (contentResult.wasDeduped) {
+                console.log(`✓ Content deduplicated: ${metadata.displayName} (ID: ${contentResult.contentId})`);
+                return contentResult.contentId;
+            }
+            console.log(`✓ Content stored: ${metadata.displayName} (ID: ${contentResult.contentId})`);
+            // Phase 2: Document Processing
+            console.log('\n--- Phase 2: Document Processing ---');
+            // Determine content type for processing
+            const detectedContentType = metadata.contentType || 'text/plain';
+            const isImageContent = detectedContentType.startsWith('image/');
+            let document;
+            if (isImageContent) {
+                // Process image content using the existing image processing pipeline
+                console.log(`Processing image content: ${metadata.displayName} (${detectedContentType})`);
+                document = await this.processImageFromMemory(content, contentResult, metadata, options);
+            }
+            else if (detectedContentType === 'application/pdf') {
+                // Process PDF content
+                console.log(`Processing PDF content: ${metadata.displayName}`);
+                document = await this.processPDFFromMemory(content, contentResult, metadata, options);
+            }
+            else if (detectedContentType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') {
+                // Process DOCX content
+                console.log(`Processing DOCX content: ${metadata.displayName}`);
+                document = await this.processDOCXFromMemory(content, contentResult, metadata, options);
+            }
+            else {
+                // Process as text content
+                console.log(`Processing text content: ${metadata.displayName} (${detectedContentType})`);
+                document = {
+                    source: metadata.displayName,
+                    title: metadata.displayName,
+                    content: content.toString('utf8'), // Convert buffer to string for processing
+                    metadata: {
+                        contentType: detectedContentType,
+                        contentId: contentResult.contentId,
+                        storageType: contentResult.storageType,
+                        originalPath: metadata.originalPath
+                    }
+                };
+            }
+            // Phase 3: Document Chunking
+            console.log('\n--- Phase 3: Document Chunking ---');
+            const effectiveChunkConfig = options.chunkConfig || this.defaultChunkConfig || {
+                chunkSize: config.chunk_size,
+                chunkOverlap: config.chunk_overlap
+            };
+            const chunks = await chunkDocument(document, effectiveChunkConfig);
+            console.log(`✓ Created ${chunks.length} chunks from memory content`);
+            if (chunks.length === 0) {
+                console.log('No chunks created from memory content');
+                return contentResult.contentId;
+            }
+            // Phase 4: Embedding Generation
+            console.log('\n--- Phase 4: Embedding Generation ---');
+            const embeddings = [];
+            let embeddingErrors = 0;
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
+                try {
+                    // Convert MIME type to simple content type for embedding function
+                    const contentTypeForEmbedding = this.getContentTypeForEmbedding(document.metadata?.contentType);
+                    const embedding = await this.embedFn(chunk.text, contentTypeForEmbedding);
+                    // Enhance embedding result with content type metadata
+                    if (!embedding.contentType) {
+                        embedding.contentType = contentTypeForEmbedding;
+                    }
+                    if (!embedding.metadata) {
+                        embedding.metadata = document.metadata;
+                    }
+                    embeddings.push(embedding);
+                }
+                catch (error) {
+                    console.warn(`Failed to embed chunk ${i + 1}:`, error instanceof Error ? error.message : String(error));
+                    embeddingErrors++;
+                }
+            }
+            console.log(`✓ Generated ${embeddings.length} embeddings for memory content`);
+            if (embeddings.length === 0) {
+                console.log('No embeddings generated from memory content');
+                return contentResult.contentId;
+            }
+            // Phase 5: Database Storage
+            console.log('\n--- Phase 5: Database Storage ---');
+            // Insert document with content_id reference
+            const documentContentType = this.getContentTypeForEmbedding(document.metadata?.contentType);
+            const documentId = await upsertDocument(this.db, document.source, document.title, documentContentType, document.metadata, contentResult.contentId);
+            // Insert chunks with embeddings
+            let chunksStored = 0;
+            for (let i = 0; i < chunks.length && i < embeddings.length; i++) {
+                const chunk = chunks[i];
+                const embedding = embeddings[i];
+                try {
+                    await insertChunk(this.db, embedding.embedding_id, documentId, chunk.text, chunk.chunkIndex, documentContentType, document.metadata);
+                    chunksStored++;
+                }
+                catch (error) {
+                    console.error(`Failed to store chunk ${i + 1}:`, error instanceof Error ? error.message : String(error));
+                }
+            }
+            console.log(`✓ Stored document and ${chunksStored} chunks in database`);
+            // Phase 6: Vector Index Updates
+            console.log('\n--- Phase 6: Vector Index Updates ---');
+            await this.updateVectorIndex(embeddings);
+            const endTime = Date.now();
+            const processingTimeMs = endTime - startTime;
+            console.log('\n=== Memory Ingestion Complete ===');
+            console.log(`Content ID: ${contentResult.contentId}`);
+            console.log(`Chunks created: ${chunks.length}`);
+            console.log(`Embeddings generated: ${embeddings.length}`);
+            console.log(`Chunks stored: ${chunksStored}`);
+            console.log(`Embedding errors: ${embeddingErrors}`);
+            console.log(`Total time: ${(processingTimeMs / 1000).toFixed(2)}s`);
+            return contentResult.contentId;
+        }
+        catch (error) {
+            console.error('\n=== Memory Ingestion Failed ===');
+            console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            throw new Error(`Memory ingestion failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
     /**
      * Ingest documents from a path (file or directory)
      * Implements the complete pipeline: file processing → chunking → embedding → storage
+     * Enhanced to handle mixed content types (text and images) in multimodal mode
      */
     async ingestPath(path, options = {}) {
         const startTime = Date.now();
         console.log(`\n=== Starting ingestion from: ${path} ===`);
         try {
-            // Phase 1: File Discovery and Processing
+            // Phase 1: File Discovery and Processing with Content-Type Detection
             console.log('\n--- Phase 1: File Discovery and Processing ---');
             const fileResult = await discoverAndProcessFiles(path, options.fileOptions, this.pathManager);
             if (fileResult.documents.length === 0) {
@@ -133,16 +277,20 @@ export class IngestionPipeline {
                     embeddingsGenerated: 0,
                     documentErrors: fileResult.processingResult.errors.length,
                     embeddingErrors: 0,
-                    processingTimeMs: Date.now() - startTime
+                    processingTimeMs: Date.now() - startTime,
+                    contentIds: []
                 };
             }
-            // Phase 2: Document Chunking
+            // Content-type detection and routing
+            const contentTypeStats = this.analyzeContentTypes(fileResult.documents);
+            console.log(`📊 Content analysis: ${contentTypeStats.text} text, ${contentTypeStats.image} image, ${contentTypeStats.other} other files`);
+            // Phase 2: Document Chunking with Content-Type Awareness
             console.log('\n--- Phase 2: Document Chunking ---');
             const effectiveChunkConfig = options.chunkConfig || this.defaultChunkConfig || {
                 chunkSize: config.chunk_size,
                 chunkOverlap: config.chunk_overlap
             };
-            const chunkingResult = await this.chunkDocuments(fileResult.documents, effectiveChunkConfig);
+            const chunkingResult = await this.chunkDocumentsWithContentTypes(fileResult.documents, effectiveChunkConfig);
             if (chunkingResult.totalChunks === 0) {
                 console.log('No chunks created from documents');
                 return {
@@ -151,15 +299,16 @@ export class IngestionPipeline {
                     embeddingsGenerated: 0,
                     documentErrors: fileResult.processingResult.errors.length,
                     embeddingErrors: 0,
-                    processingTimeMs: Date.now() - startTime
+                    processingTimeMs: Date.now() - startTime,
+                    contentIds: []
                 };
             }
-            // Phase 3: Embedding Generation
+            // Phase 3: Embedding Generation with Content-Type Support
             console.log('\n--- Phase 3: Embedding Generation ---');
-            const embeddingResult = await this.generateEmbeddings(chunkingResult.allChunks);
-            // Phase 4: Database and Index Storage
+            const embeddingResult = await this.generateEmbeddingsWithContentTypes(chunkingResult.allChunks);
+            // Phase 4: Database and Index Storage with Content-Type Metadata
             console.log('\n--- Phase 4: Storage Operations ---');
-            await this.storeDocumentsAndChunks(chunkingResult.documentChunks, embeddingResult.embeddings);
+            const contentIds = await this.storeDocumentsAndChunksWithContentTypes(chunkingResult.documentChunks, embeddingResult.embeddings);
             // Phase 5: Vector Index Updates
             console.log('\n--- Phase 5: Vector Index Updates ---');
             await this.updateVectorIndex(embeddingResult.embeddings);
@@ -171,7 +320,8 @@ export class IngestionPipeline {
                 embeddingsGenerated: embeddingResult.embeddings.length,
                 documentErrors: fileResult.processingResult.errors.length,
                 embeddingErrors: embeddingResult.errors,
-                processingTimeMs
+                processingTimeMs,
+                contentIds
             };
             console.log('\n=== Ingestion Complete ===');
             console.log(`Documents processed: ${result.documentsProcessed}`);
@@ -189,9 +339,32 @@ export class IngestionPipeline {
         }
     }
     /**
-     * Chunk all documents and organize results
+     * Analyze content types in the document collection
+     * @private
      */
-    async chunkDocuments(documents, chunkConfig) {
+    analyzeContentTypes(documents) {
+        const stats = { text: 0, image: 0, other: 0 };
+        for (const document of documents) {
+            const contentType = document.metadata?.contentType || 'text';
+            switch (contentType) {
+                case 'text':
+                    stats.text++;
+                    break;
+                case 'image':
+                    stats.image++;
+                    break;
+                default:
+                    stats.other++;
+                    break;
+            }
+        }
+        return stats;
+    }
+    /**
+     * Chunk all documents and organize results with content-type awareness
+     * Enhanced to handle different content types appropriately
+     */
+    async chunkDocumentsWithContentTypes(documents, chunkConfig) {
         const documentChunks = [];
         const allChunks = [];
         let totalChunks = 0;
@@ -199,11 +372,36 @@ export class IngestionPipeline {
         for (let i = 0; i < documents.length; i++) {
             const document = documents[i];
             try {
-                const chunks = await chunkDocument(document, chunkConfig);
+                const contentType = document.metadata?.contentType || 'text';
+                // Handle different content types appropriately
+                let chunks;
+                if (contentType === 'image') {
+                    // For images, create a single chunk with the full content (description + metadata)
+                    chunks = [{
+                            text: document.content,
+                            chunkIndex: 0,
+                            contentType: 'image',
+                            metadata: document.metadata
+                        }];
+                }
+                else {
+                    // For text documents, use normal chunking
+                    const textChunks = await chunkDocument(document, chunkConfig);
+                    chunks = textChunks.map(chunk => ({
+                        ...chunk,
+                        contentType: 'text',
+                        metadata: document.metadata
+                    }));
+                }
                 documentChunks.push({ document, chunks });
-                // Collect all chunk texts for embedding
-                const chunkTexts = chunks.map(chunk => chunk.text);
-                allChunks.push(...chunkTexts);
+                // Collect all chunks with their content type information
+                for (const chunk of chunks) {
+                    allChunks.push({
+                        text: chunk.text,
+                        contentType: chunk.contentType,
+                        metadata: chunk.metadata
+                    });
+                }
                 totalChunks += chunks.length;
                 // Progress logging - more frequent for better user experience
                 if (documents.length <= 10 || (i + 1) % Math.max(1, Math.floor(documents.length / 10)) === 0 || i === documents.length - 1) {
@@ -221,28 +419,52 @@ export class IngestionPipeline {
         return { documentChunks, allChunks, totalChunks };
     }
     /**
-     * Generate embeddings for all chunks with error handling
+     * Chunk all documents and organize results (legacy method for backward compatibility)
+     * @deprecated Use chunkDocumentsWithContentTypes for multimodal support
      */
-    async generateEmbeddings(chunkTexts) {
-        console.log(`Generating embeddings for ${chunkTexts.length} chunk${chunkTexts.length === 1 ? '' : 's'}...`);
+    async chunkDocuments(documents, chunkConfig) {
+        const result = await this.chunkDocumentsWithContentTypes(documents, chunkConfig);
+        // Convert to legacy format for backward compatibility
+        return {
+            documentChunks: result.documentChunks,
+            allChunks: result.allChunks.map(chunk => chunk.text),
+            totalChunks: result.totalChunks
+        };
+    }
+    /**
+     * Generate embeddings for all chunks with content-type support
+     * Enhanced to handle different content types and pass metadata to embedding function
+     */
+    async generateEmbeddingsWithContentTypes(chunks) {
+        console.log(`Generating embeddings for ${chunks.length} chunk${chunks.length === 1 ? '' : 's'}...`);
         console.log('This may take a few minutes depending on the number of chunks...');
         try {
-            // Generate embeddings using injected embed function
+            // Generate embeddings using injected embed function with content type support
             const embeddings = [];
             let errors = 0;
-            for (let i = 0; i < chunkTexts.length; i++) {
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
                 try {
-                    const embedding = await this.embedFn(chunkTexts[i]);
+                    // Convert MIME type to simple content type for embedding function
+                    const contentTypeForEmbedding = this.getContentTypeForEmbedding(chunk.contentType);
+                    const embedding = await this.embedFn(chunk.text, contentTypeForEmbedding);
+                    // Enhance embedding result with content type metadata if not already present
+                    if (!embedding.contentType) {
+                        embedding.contentType = contentTypeForEmbedding;
+                    }
+                    if (!embedding.metadata && chunk.metadata) {
+                        embedding.metadata = chunk.metadata;
+                    }
                     embeddings.push(embedding);
                 }
                 catch (error) {
-                    console.warn(`Failed to embed chunk ${i + 1}:`, error instanceof Error ? error.message : String(error));
+                    console.warn(`Failed to embed ${chunk.contentType} chunk ${i + 1}:`, error instanceof Error ? error.message : String(error));
                     errors++;
                 }
                 // Progress logging
-                if (chunkTexts.length > 10 && (i + 1) % Math.max(1, Math.floor(chunkTexts.length / 10)) === 0) {
-                    const percentage = Math.round(((i + 1) / chunkTexts.length) * 100);
-                    console.log(`Generated ${i + 1} of ${chunkTexts.length} embeddings (${percentage}%)`);
+                if (chunks.length > 10 && (i + 1) % Math.max(1, Math.floor(chunks.length / 10)) === 0) {
+                    const percentage = Math.round(((i + 1) / chunks.length) * 100);
+                    console.log(`Generated ${i + 1} of ${chunks.length} embeddings (${percentage}%)`);
                 }
             }
             if (errors > 0) {
@@ -257,9 +479,20 @@ export class IngestionPipeline {
         }
     }
     /**
-     * Store documents and chunks in database
+     * Generate embeddings for all chunks with error handling (legacy method for backward compatibility)
+     * @deprecated Use generateEmbeddingsWithContentTypes for multimodal support
      */
-    async storeDocumentsAndChunks(documentChunks, embeddings) {
+    async generateEmbeddings(chunkTexts) {
+        // Convert to new format for backward compatibility
+        const chunks = chunkTexts.map(text => ({ text, contentType: 'text' }));
+        return this.generateEmbeddingsWithContentTypes(chunks);
+    }
+    /**
+     * Store documents and chunks in database with content-type support
+     * Enhanced to handle content type metadata and multimodal content
+     * @returns Array of content IDs for successfully stored documents
+     */
+    async storeDocumentsAndChunksWithContentTypes(documentChunks, embeddings) {
         console.log(`Storing ${documentChunks.length} document${documentChunks.length === 1 ? '' : 's'} and chunks in database...`);
         // Create a mapping of chunk text to embedding for efficient lookup
         const embeddingMap = new Map();
@@ -275,24 +508,51 @@ export class IngestionPipeline {
         }
         let totalChunksStored = 0;
         let documentsStored = 0;
+        const contentIds = [];
         // Process each document sequentially
         for (const { document, chunks } of documentChunks) {
             try {
-                // Insert or get existing document
-                const documentId = await upsertDocument(this.db, document.source, document.title);
+                // Generate content ID for filesystem content using ContentManager
+                let contentId = document.metadata?.contentId;
+                if (!contentId) {
+                    try {
+                        // Use ContentManager to create filesystem reference and get content ID
+                        const contentResult = await this.contentManager.ingestFromFilesystem(document.source);
+                        contentId = contentResult.contentId;
+                        // Update document metadata with content ID
+                        if (!document.metadata) {
+                            document.metadata = {};
+                        }
+                        document.metadata.contentId = contentId;
+                        document.metadata.storageType = contentResult.storageType;
+                    }
+                    catch (contentError) {
+                        console.warn(`Failed to create content reference for ${document.source}:`, contentError instanceof Error ? contentError.message : String(contentError));
+                        // Continue without content ID - fallback to legacy behavior
+                    }
+                }
+                // Insert or get existing document with content type support and content_id reference
+                const documentContentType = document.metadata?.contentType || 'text';
+                const documentId = await upsertDocument(this.db, document.source, document.title, documentContentType, document.metadata, contentId);
                 documentsStored++;
-                // Insert all chunks for this document
+                // Add content ID to results if available
+                if (contentId) {
+                    contentIds.push(contentId);
+                }
+                // Insert all chunks for this document with content type support
                 let chunksStoredForDoc = 0;
                 for (const chunk of chunks) {
                     const embedding = embeddingMap.get(chunk.text);
                     if (embedding) {
                         try {
-                            await insertChunk(this.db, embedding.embedding_id, documentId, chunk.text, chunk.chunkIndex);
+                            const chunkContentType = chunk.contentType || documentContentType;
+                            const chunkMetadata = chunk.metadata || document.metadata;
+                            await insertChunk(this.db, embedding.embedding_id, documentId, chunk.text, chunk.chunkIndex, chunkContentType, chunkMetadata);
                             chunksStoredForDoc++;
                             totalChunksStored++;
                         }
                         catch (chunkError) {
-                            console.error(`Failed to store chunk ${chunk.chunkIndex} for document ${document.source}:`, chunkError instanceof Error ? chunkError.message : String(chunkError));
+                            console.error(`Failed to store ${chunk.contentType || 'text'} chunk ${chunk.chunkIndex} for document ${document.source}:`, chunkError instanceof Error ? chunkError.message : String(chunkError));
                             // Continue with other chunks
                         }
                     }
@@ -312,6 +572,14 @@ export class IngestionPipeline {
             }
         }
         console.log(`✓ Storage complete: ${documentsStored} documents, ${totalChunksStored} chunks saved to database`);
+        return contentIds;
+    }
+    /**
+     * Store documents and chunks in database (legacy method for backward compatibility)
+     * @deprecated Use storeDocumentsAndChunksWithContentTypes for multimodal support
+     */
+    async storeDocumentsAndChunks(documentChunks, embeddings) {
+        await this.storeDocumentsAndChunksWithContentTypes(documentChunks, embeddings);
     }
     /**
      * Update vector index with new embeddings
@@ -331,11 +599,208 @@ export class IngestionPipeline {
             throw error;
         }
     }
+    /**
+     * Converts MIME type to simple content type for embedding function
+     * @param mimeType - MIME type string (e.g., 'text/plain', 'image/jpeg')
+     * @returns Simple content type ('text', 'image', etc.)
+     */
+    getContentTypeForEmbedding(mimeType) {
+        if (!mimeType) {
+            return 'text';
+        }
+        // Convert MIME types to simple content types
+        if (mimeType.startsWith('text/')) {
+            return 'text';
+        }
+        else if (mimeType.startsWith('image/')) {
+            return 'image';
+        }
+        else if (mimeType === 'application/pdf') {
+            return 'text'; // PDFs are processed as text
+        }
+        else if (mimeType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') {
+            return 'text'; // DOCX files are processed as text
+        }
+        else {
+            return 'text'; // Default to text for unknown types
+        }
+    }
+    /**
+     * Save the vector index to disk
+     */
+    async saveIndex() {
+        await this.indexManager.saveIndex();
+    }
+    /**
+     * Process image content from memory using the existing image processing pipeline
+     * @private
+     */
+    async processImageFromMemory(content, contentResult, metadata, options) {
+        try {
+            // Import image processing functions
+            const { generateImageDescriptionForFile, extractImageMetadataForFile } = await import('../file-processor.js');
+            // Use the content path from the content manager (where the image is stored)
+            const imagePath = contentResult.contentPath;
+            // Extract image metadata
+            let imageMetadata = {};
+            try {
+                imageMetadata = await extractImageMetadataForFile(imagePath);
+            }
+            catch (error) {
+                console.warn(`Failed to extract image metadata for ${metadata.displayName}:`, error instanceof Error ? error.message : String(error));
+                // Continue with empty metadata
+            }
+            // Generate text description for the image
+            let descriptionResult = { description: 'Image content', model: 'none', confidence: 0 };
+            try {
+                const imageToTextOptions = {}; // Use default options for now
+                descriptionResult = await generateImageDescriptionForFile(imagePath, imageToTextOptions);
+                console.log(`✓ Generated image description: "${descriptionResult.description}"`);
+            }
+            catch (error) {
+                console.warn(`Failed to generate image description for ${metadata.displayName}:`, error instanceof Error ? error.message : String(error));
+                // Continue with fallback description
+            }
+            // Update metadata with description information
+            imageMetadata.description = descriptionResult.description;
+            imageMetadata.descriptionModel = descriptionResult.model;
+            imageMetadata.descriptionConfidence = descriptionResult.confidence;
+            // Create document with image description as content
+            const title = metadata.displayName;
+            // Create content that includes description and key metadata
+            const contentParts = [
+                `Image: ${title}`,
+                `Description: ${descriptionResult.description}`
+            ];
+            if (imageMetadata.dimensions) {
+                contentParts.push(`Dimensions: ${imageMetadata.dimensions.width}x${imageMetadata.dimensions.height}`);
+            }
+            if (imageMetadata.format) {
+                contentParts.push(`Format: ${imageMetadata.format}`);
+            }
+            const documentContent = contentParts.join('\n');
+            return {
+                source: metadata.displayName,
+                title,
+                content: documentContent.trim(),
+                metadata: {
+                    contentType: 'image',
+                    contentId: contentResult.contentId,
+                    storageType: contentResult.storageType,
+                    originalPath: metadata.originalPath,
+                    ...imageMetadata // Spread all image metadata fields
+                }
+            };
+        }
+        catch (error) {
+            console.warn(`Failed to process image from memory, falling back to basic processing:`, error instanceof Error ? error.message : String(error));
+            // Fallback to basic document creation
+            return {
+                source: metadata.displayName,
+                title: metadata.displayName,
+                content: `Image: ${metadata.displayName}\nPath: ${contentResult.contentPath}`,
+                metadata: {
+                    contentType: 'image',
+                    contentId: contentResult.contentId,
+                    storageType: contentResult.storageType,
+                    originalPath: metadata.originalPath,
+                    processingError: error instanceof Error ? error.message : String(error)
+                }
+            };
+        }
+    }
+    /**
+     * Process PDF content from memory using the existing PDF processing pipeline
+     * @private
+     */
+    async processPDFFromMemory(content, contentResult, metadata, options) {
+        try {
+            // Import PDF processing
+            const pdfParse = require('pdf-parse');
+            // Parse PDF content directly from buffer
+            const pdfData = await pdfParse(content);
+            console.log(`✓ Extracted ${pdfData.text.length} characters from PDF`);
+            return {
+                source: metadata.displayName,
+                title: metadata.displayName,
+                content: pdfData.text.trim(),
+                metadata: {
+                    contentType: 'application/pdf',
+                    contentId: contentResult.contentId,
+                    storageType: contentResult.storageType,
+                    originalPath: metadata.originalPath,
+                    pages: pdfData.numpages,
+                    pdfInfo: pdfData.info
+                }
+            };
+        }
+        catch (error) {
+            console.warn(`Failed to process PDF from memory, falling back to basic processing:`, error instanceof Error ? error.message : String(error));
+            // Fallback to basic document creation
+            return {
+                source: metadata.displayName,
+                title: metadata.displayName,
+                content: `PDF Document: ${metadata.displayName}\nPath: ${contentResult.contentPath}`,
+                metadata: {
+                    contentType: 'application/pdf',
+                    contentId: contentResult.contentId,
+                    storageType: contentResult.storageType,
+                    originalPath: metadata.originalPath,
+                    processingError: error instanceof Error ? error.message : String(error)
+                }
+            };
+        }
+    }
+    /**
+     * Process DOCX content from memory using the existing DOCX processing pipeline
+     * @private
+     */
+    async processDOCXFromMemory(content, contentResult, metadata, options) {
+        try {
+            // Import DOCX processing
+            const mammoth = await import('mammoth');
+            // Parse DOCX content directly from buffer
+            const docxResult = await mammoth.extractRawText({ buffer: content });
+            console.log(`✓ Extracted ${docxResult.value.length} characters from DOCX`);
+            return {
+                source: metadata.displayName,
+                title: metadata.displayName,
+                content: docxResult.value.trim(),
+                metadata: {
+                    contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+                    contentId: contentResult.contentId,
+                    storageType: contentResult.storageType,
+                    originalPath: metadata.originalPath,
+                    messages: docxResult.messages
+                }
+            };
+        }
+        catch (error) {
+            console.warn(`Failed to process DOCX from memory, falling back to basic processing:`, error instanceof Error ? error.message : String(error));
+            // Fallback to basic document creation
+            return {
+                source: metadata.displayName,
+                title: metadata.displayName,
+                content: `DOCX Document: ${metadata.displayName}\nPath: ${contentResult.contentPath}`,
+                metadata: {
+                    contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+                    contentId: contentResult.contentId,
+                    storageType: contentResult.storageType,
+                    originalPath: metadata.originalPath,
+                    processingError: error instanceof Error ? error.message : String(error)
+                }
+            };
+        }
+    }
     /**
      * Clean up resources - explicit cleanup method
      */
     async cleanup() {
         try {
+            // Clean up ContentManager to prevent resource leaks
+            if (this.contentManager && typeof this.contentManager.cleanup === 'function') {
+                this.contentManager.cleanup();
+            }
             await this.db.close();
             await this.indexManager.close();
         }