rag-lite-ts 1.0.2 → 2.0.0

package/dist/core/ingestion.d.ts

@@ -7,6 +7,7 @@ import { type ChunkConfig } from './chunker.js';
 import { IndexManager } from '../index-manager.js';
 import { type DatabaseConnection } from './db.js';
 import type { EmbedFunction } from './interfaces.js';
+import { ContentManager, type MemoryContentMetadata } from './content-manager.js';
 /**
  * Options for the ingestion pipeline
  */
@@ -17,6 +18,10 @@ export interface IngestionOptions {
     chunkConfig?: ChunkConfig;
     /** Whether to force rebuild the index */
     forceRebuild?: boolean;
+    /** Mode for the ingestion pipeline (text or multimodal) */
+    mode?: 'text' | 'multimodal';
+    /** Content type for the ingested content */
+    contentType?: string;
 }
 /**
  * Result of the ingestion process
@@ -34,6 +39,8 @@ export interface IngestionResult {
     embeddingErrors: number;
     /** Processing time in milliseconds */
     processingTimeMs: number;
+    /** Content IDs of successfully ingested documents */
+    contentIds: string[];
 }
 /**
  * Main ingestion pipeline class
@@ -46,8 +53,10 @@ export declare class IngestionPipeline {
     private db;
     private defaultChunkConfig?;
     private pathManager;
+    private contentManager;
     /**
      * Creates a new IngestionPipeline with explicit dependency injection
+     * Enhanced with ContentManager integration for unified content system
      *
      * DEPENDENCY INJECTION PATTERN:
      * This constructor requires all dependencies to be explicitly provided, enabling:
@@ -55,6 +64,7 @@ export declare class IngestionPipeline {
      * - Support for different embedding models and content types
      * - Testability through mock injection
      * - Future extensibility for multimodal content processing
+     * - Unified content management for both filesystem and memory-based ingestion
      *
      * @param embedFn - Function to embed document chunks into vectors
      *   - Signature: (query: string, contentType?: string) => Promise<EmbeddingResult>
@@ -74,32 +84,33 @@ export declare class IngestionPipeline {
      *   - Supports different content types through metadata fields
      *   - Example: await openDatabase('./db.sqlite')
      *
+     * @param contentManager - Optional ContentManager for unified content system
+     *   - Handles content storage routing and deduplication
+     *   - If not provided, creates default instance with standard configuration
+     *   - Example: new ContentManager(db, { contentDir: '.raglite/content' })
+     *
      * USAGE EXAMPLES:
      * ```typescript
-     * // Text-only ingestion pipeline
+     * // Text-only ingestion pipeline with unified content system
      * const textEmbedFn = await createTextEmbedder();
      * const indexManager = new IndexManager('./index.bin');
      * const db = await openDatabase('./db.sqlite');
+     * const contentManager = new ContentManager(db);
+     * const ingestion = new IngestionPipeline(textEmbedFn, indexManager, db, undefined, contentManager);
+     *
+     * // Simple usage (ContentManager created automatically)
      * const ingestion = new IngestionPipeline(textEmbedFn, indexManager, db);
      *
-     * // Custom embedding implementation
+     * // Custom embedding implementation with memory ingestion
      * const customEmbedFn = async (text) => ({
      *   embedding_id: generateId(),
      *   vector: await myCustomModel.embed(text)
      * });
      * const ingestion = new IngestionPipeline(customEmbedFn, indexManager, db);
-     *
-     * // Multimodal ingestion (future)
-     * const multimodalEmbedFn = async (content, contentType) => {
-     *   if (contentType === 'image') {
-     *     return { embedding_id: generateId(), vector: await clipModel.embedImage(content) };
-     *   }
-     *   return { embedding_id: generateId(), vector: await clipModel.embedText(content) };
-     * };
-     * const ingestion = new IngestionPipeline(multimodalEmbedFn, indexManager, db);
+     * await ingestion.ingestFromMemory(buffer, { displayName: 'file.txt' });
      * ```
      */
-    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, defaultChunkConfig?: ChunkConfig | undefined);
+    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, defaultChunkConfig?: ChunkConfig | undefined, contentManager?: ContentManager);
     /**
      * Ingest documents from a directory
      * @param directoryPath - Path to directory containing documents
@@ -114,27 +125,86 @@ export declare class IngestionPipeline {
      * @returns Promise resolving to ingestion results
      */
     ingestFile(filePath: string, options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Ingest content from memory buffer
+     * Enables MCP integration and real-time content processing
+     * @param content - Buffer containing the content to ingest
+     * @param metadata - Memory content metadata including display name and content type
+     * @param options - Optional ingestion configuration
+     * @returns Promise resolving to content ID for the ingested content
+     */
+    ingestFromMemory(content: Buffer, metadata: MemoryContentMetadata, options?: IngestionOptions): Promise<string>;
     /**
      * Ingest documents from a path (file or directory)
      * Implements the complete pipeline: file processing → chunking → embedding → storage
+     * Enhanced to handle mixed content types (text and images) in multimodal mode
      */
     ingestPath(path: string, options?: IngestionOptions): Promise<IngestionResult>;
     /**
-     * Chunk all documents and organize results
+     * Analyze content types in the document collection
+     * @private
+     */
+    private analyzeContentTypes;
+    /**
+     * Chunk all documents and organize results with content-type awareness
+     * Enhanced to handle different content types appropriately
+     */
+    private chunkDocumentsWithContentTypes;
+    /**
+     * Chunk all documents and organize results (legacy method for backward compatibility)
+     * @deprecated Use chunkDocumentsWithContentTypes for multimodal support
      */
     private chunkDocuments;
     /**
-     * Generate embeddings for all chunks with error handling
+     * Generate embeddings for all chunks with content-type support
+     * Enhanced to handle different content types and pass metadata to embedding function
+     */
+    private generateEmbeddingsWithContentTypes;
+    /**
+     * Generate embeddings for all chunks with error handling (legacy method for backward compatibility)
+     * @deprecated Use generateEmbeddingsWithContentTypes for multimodal support
      */
     private generateEmbeddings;
     /**
-     * Store documents and chunks in database
+     * Store documents and chunks in database with content-type support
+     * Enhanced to handle content type metadata and multimodal content
+     * @returns Array of content IDs for successfully stored documents
+     */
+    private storeDocumentsAndChunksWithContentTypes;
+    /**
+     * Store documents and chunks in database (legacy method for backward compatibility)
+     * @deprecated Use storeDocumentsAndChunksWithContentTypes for multimodal support
      */
     private storeDocumentsAndChunks;
     /**
      * Update vector index with new embeddings
      */
     private updateVectorIndex;
+    /**
+     * Converts MIME type to simple content type for embedding function
+     * @param mimeType - MIME type string (e.g., 'text/plain', 'image/jpeg')
+     * @returns Simple content type ('text', 'image', etc.)
+     */
+    private getContentTypeForEmbedding;
+    /**
+     * Save the vector index to disk
+     */
+    saveIndex(): Promise<void>;
+    /**
+     * Process image content from memory using the existing image processing pipeline
+     * @private
+     */
+    private processImageFromMemory;
+    /**
+     * Process PDF content from memory using the existing PDF processing pipeline
+     * @private
+     */
+    private processPDFFromMemory;
+    /**
+     * Process DOCX content from memory using the existing DOCX processing pipeline
+     * @private
+     */
+    private processDOCXFromMemory;
     /**
      * Clean up resources - explicit cleanup method
      */