rag-lite-ts 1.0.1 → 2.0.0

package/dist/core/index.d.ts

@@ -0,0 +1,59 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ *
+ * This module provides the clean re-export surface for the core layer, enabling
+ * dependency injection patterns for different implementations (text-only, multimodal, etc.).
+ *
+ * DEPENDENCY INJECTION ARCHITECTURE:
+ *
+ * The core layer uses explicit dependency injection to maintain clean separation between
+ * model-agnostic logic and implementation-specific components:
+ *
+ * 1. Core Classes (SearchEngine, IngestionPipeline):
+ *    - Accept injected functions (EmbedFunction, RerankFunction) in constructors
+ *    - Coordinate model-agnostic operations (database, vector index, search pipeline)
+ *    - No knowledge of specific embedding models or transformers
+ *
+ * 2. Dependency Injection Interfaces:
+ *    - EmbedFunction: (query: string, contentType?: string) => Promise<EmbeddingResult>
+ *    - RerankFunction: (query: string, results: SearchResult[], contentType?: string) => Promise<SearchResult[]>
+ *    - Support different content types (text, image, etc.) and embedding dimensions
+ *
+ * 3. Usage Patterns:
+ *
+ *    // Direct dependency injection (advanced users)
+ *    const embedFn = await createTextEmbedder();
+ *    const rerankFn = await createTextReranker();
+ *    const indexManager = new IndexManager('./index.bin');
+ *    const db = await openDatabase('./db.sqlite');
+ *    const search = new SearchEngine(embedFn, indexManager, db, rerankFn);
+ *
+ *    // Factory pattern (recommended for common use cases)
+ *    const search = await TextSearchFactory.create('./index.bin', './db.sqlite');
+ *
+ * 4. Extension Points:
+ *    - New implementations (multimodal, custom models) implement the same interfaces
+ *    - Core classes remain unchanged when adding new modalities
+ *    - Plugin architecture enabled through interface-based design
+ *
+ * 5. Benefits:
+ *    - Clean separation of concerns
+ *    - Testability through mock injection
+ *    - Future extensibility without core changes
+ *    - Support for different embedding dimensions and content types
+ */
+export { type ContentDocument, type ContentChunk, type Document, type Chunk, type EmbeddingResult, type SearchResult, type SearchOptions, } from './types.js';
+export { type EmbedFunction, type RerankFunction, type EmbeddingQueryInterface, type RerankingInterface, type SearchEngineConfig, type ContentTypeStrategy, type ModelAgnosticInterface, type ExtendedEmbeddingInterface, type ExtendedRerankingInterface, type SearchPipelineInterface, type SearchDependencyFactory, InterfaceValidator } from './interfaces.js';
+export * from './adapters.js';
+export * from './config.js';
+export { type DatabaseConnection, type ContentMetadata, openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, getModelVersion, setModelVersion, getStoredModelInfo, setStoredModelInfo, insertContentMetadata, getContentMetadata, getContentMetadataByHash, getContentMetadataByStorageType, deleteContentMetadata, getStorageStats, updateStorageStats } from './db.js';
+export { type VectorIndexOptions, VectorIndex } from './vector-index.js';
+export { type ChunkConfig, type GenericDocument, type GenericChunk, type ChunkingStrategy, ChunkingStrategyRegistry, DEFAULT_CHUNK_CONFIG, chunkingRegistry, chunkGenericDocument, registerTextChunkingStrategy } from './chunker.js';
+export * from './search.js';
+export * from './ingestion.js';
+export * from './path-manager.js';
+export { ContentManager, type MemoryContentMetadata, type ContentIngestionResult, type ContentManagerConfig } from './content-manager.js';
+export { ContentResolver, type ContentRequest, type ContentResult } from './content-resolver.js';
+export * from './error-handler.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/core/index.js

@@ -0,0 +1,69 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ *
+ * This module provides the clean re-export surface for the core layer, enabling
+ * dependency injection patterns for different implementations (text-only, multimodal, etc.).
+ *
+ * DEPENDENCY INJECTION ARCHITECTURE:
+ *
+ * The core layer uses explicit dependency injection to maintain clean separation between
+ * model-agnostic logic and implementation-specific components:
+ *
+ * 1. Core Classes (SearchEngine, IngestionPipeline):
+ *    - Accept injected functions (EmbedFunction, RerankFunction) in constructors
+ *    - Coordinate model-agnostic operations (database, vector index, search pipeline)
+ *    - No knowledge of specific embedding models or transformers
+ *
+ * 2. Dependency Injection Interfaces:
+ *    - EmbedFunction: (query: string, contentType?: string) => Promise<EmbeddingResult>
+ *    - RerankFunction: (query: string, results: SearchResult[], contentType?: string) => Promise<SearchResult[]>
+ *    - Support different content types (text, image, etc.) and embedding dimensions
+ *
+ * 3. Usage Patterns:
+ *
+ *    // Direct dependency injection (advanced users)
+ *    const embedFn = await createTextEmbedder();
+ *    const rerankFn = await createTextReranker();
+ *    const indexManager = new IndexManager('./index.bin');
+ *    const db = await openDatabase('./db.sqlite');
+ *    const search = new SearchEngine(embedFn, indexManager, db, rerankFn);
+ *
+ *    // Factory pattern (recommended for common use cases)
+ *    const search = await TextSearchFactory.create('./index.bin', './db.sqlite');
+ *
+ * 4. Extension Points:
+ *    - New implementations (multimodal, custom models) implement the same interfaces
+ *    - Core classes remain unchanged when adding new modalities
+ *    - Plugin architecture enabled through interface-based design
+ *
+ * 5. Benefits:
+ *    - Clean separation of concerns
+ *    - Testability through mock injection
+ *    - Future extensibility without core changes
+ *    - Support for different embedding dimensions and content types
+ */
+// Dependency injection interfaces and utilities
+export { InterfaceValidator } from './interfaces.js';
+// Adapter utilities for converting implementations to dependency injection
+export * from './adapters.js';
+// Core configuration management - model-agnostic settings
+export * from './config.js';
+// Database operations - supports different content types through metadata
+export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, getModelVersion, setModelVersion, getStoredModelInfo, setStoredModelInfo, insertContentMetadata, getContentMetadata, getContentMetadataByHash, getContentMetadataByStorageType, deleteContentMetadata, getStorageStats, updateStorageStats } from './db.js';
+// Vector index operations - works with any embedding dimensions
+export { VectorIndex } from './vector-index.js';
+// Generic chunking interfaces and strategies - supports text, image metadata, etc.
+export { ChunkingStrategyRegistry, DEFAULT_CHUNK_CONFIG, chunkingRegistry, chunkGenericDocument, registerTextChunkingStrategy } from './chunker.js';
+// Core search engine - uses dependency injection for embedding and reranking
+export * from './search.js';
+// Core ingestion pipeline - uses dependency injection for embedding
+export * from './ingestion.js';
+// Path management utilities - content-type agnostic
+export * from './path-manager.js';
+// Unified content system - handles both filesystem and memory content
+export { ContentManager } from './content-manager.js';
+export { ContentResolver } from './content-resolver.js';
+// Error handling framework - supports implementation-specific error contexts
+export * from './error-handler.js';
+//# sourceMappingURL=index.js.map

package/dist/core/ingestion.d.ts

@@ -0,0 +1,213 @@
+/**
+ * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
+ * Model-agnostic. No transformer or modality-specific logic.
+ */
+import { type FileProcessorOptions } from '../file-processor.js';
+import { type ChunkConfig } from './chunker.js';
+import { IndexManager } from '../index-manager.js';
+import { type DatabaseConnection } from './db.js';
+import type { EmbedFunction } from './interfaces.js';
+import { ContentManager, type MemoryContentMetadata } from './content-manager.js';
+/**
+ * Options for the ingestion pipeline
+ */
+export interface IngestionOptions {
+    /** File processing options */
+    fileOptions?: FileProcessorOptions;
+    /** Chunking configuration */
+    chunkConfig?: ChunkConfig;
+    /** Whether to force rebuild the index */
+    forceRebuild?: boolean;
+    /** Mode for the ingestion pipeline (text or multimodal) */
+    mode?: 'text' | 'multimodal';
+    /** Content type for the ingested content */
+    contentType?: string;
+}
+/**
+ * Result of the ingestion process
+ */
+export interface IngestionResult {
+    /** Total documents processed */
+    documentsProcessed: number;
+    /** Total chunks created */
+    chunksCreated: number;
+    /** Total embeddings generated */
+    embeddingsGenerated: number;
+    /** Number of documents that failed processing */
+    documentErrors: number;
+    /** Number of chunks that failed embedding */
+    embeddingErrors: number;
+    /** Processing time in milliseconds */
+    processingTimeMs: number;
+    /** Content IDs of successfully ingested documents */
+    contentIds: string[];
+}
+/**
+ * Main ingestion pipeline class
+ * Coordinates the entire process from file discovery to vector storage
+ * Uses explicit dependency injection for clean architecture
+ */
+export declare class IngestionPipeline {
+    private embedFn;
+    private indexManager;
+    private db;
+    private defaultChunkConfig?;
+    private pathManager;
+    private contentManager;
+    /**
+     * Creates a new IngestionPipeline with explicit dependency injection
+     * Enhanced with ContentManager integration for unified content system
+     *
+     * DEPENDENCY INJECTION PATTERN:
+     * This constructor requires all dependencies to be explicitly provided, enabling:
+     * - Clean separation between core ingestion logic and implementation-specific components
+     * - Support for different embedding models and content types
+     * - Testability through mock injection
+     * - Future extensibility for multimodal content processing
+     * - Unified content management for both filesystem and memory-based ingestion
+     *
+     * @param embedFn - Function to embed document chunks into vectors
+     *   - Signature: (query: string, contentType?: string) => Promise<EmbeddingResult>
+     *   - Must handle chunk text and return consistent embedding format
+     *   - Examples:
+     *     - Text: const embedFn = (text) => textEmbedder.embedSingle(text)
+     *     - Multimodal: const embedFn = (content, type) => type === 'image' ? clipEmbedder.embedImage(content) : clipEmbedder.embedText(content)
+     *     - Custom: const embedFn = (text) => customModel.embed(text)
+     *
+     * @param indexManager - Vector index manager for storing embeddings
+     *   - Handles vector storage and indexing operations
+     *   - Must support the embedding dimensions produced by embedFn
+     *   - Example: new IndexManager('./index.bin')
+     *
+     * @param db - Database connection for metadata storage
+     *   - Stores document and chunk metadata with content type support
+     *   - Supports different content types through metadata fields
+     *   - Example: await openDatabase('./db.sqlite')
+     *
+     * @param contentManager - Optional ContentManager for unified content system
+     *   - Handles content storage routing and deduplication
+     *   - If not provided, creates default instance with standard configuration
+     *   - Example: new ContentManager(db, { contentDir: '.raglite/content' })
+     *
+     * USAGE EXAMPLES:
+     * ```typescript
+     * // Text-only ingestion pipeline with unified content system
+     * const textEmbedFn = await createTextEmbedder();
+     * const indexManager = new IndexManager('./index.bin');
+     * const db = await openDatabase('./db.sqlite');
+     * const contentManager = new ContentManager(db);
+     * const ingestion = new IngestionPipeline(textEmbedFn, indexManager, db, undefined, contentManager);
+     *
+     * // Simple usage (ContentManager created automatically)
+     * const ingestion = new IngestionPipeline(textEmbedFn, indexManager, db);
+     *
+     * // Custom embedding implementation with memory ingestion
+     * const customEmbedFn = async (text) => ({
+     *   embedding_id: generateId(),
+     *   vector: await myCustomModel.embed(text)
+     * });
+     * const ingestion = new IngestionPipeline(customEmbedFn, indexManager, db);
+     * await ingestion.ingestFromMemory(buffer, { displayName: 'file.txt' });
+     * ```
+     */
+    constructor(embedFn: EmbedFunction, indexManager: IndexManager, db: DatabaseConnection, defaultChunkConfig?: ChunkConfig | undefined, contentManager?: ContentManager);
+    /**
+     * Ingest documents from a directory
+     * @param directoryPath - Path to directory containing documents
+     * @param options - Optional ingestion configuration
+     * @returns Promise resolving to ingestion results
+     */
+    ingestDirectory(directoryPath: string, options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Ingest a single file
+     * @param filePath - Path to the file to ingest
+     * @param options - Optional ingestion configuration
+     * @returns Promise resolving to ingestion results
+     */
+    ingestFile(filePath: string, options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Ingest content from memory buffer
+     * Enables MCP integration and real-time content processing
+     * @param content - Buffer containing the content to ingest
+     * @param metadata - Memory content metadata including display name and content type
+     * @param options - Optional ingestion configuration
+     * @returns Promise resolving to content ID for the ingested content
+     */
+    ingestFromMemory(content: Buffer, metadata: MemoryContentMetadata, options?: IngestionOptions): Promise<string>;
+    /**
+     * Ingest documents from a path (file or directory)
+     * Implements the complete pipeline: file processing → chunking → embedding → storage
+     * Enhanced to handle mixed content types (text and images) in multimodal mode
+     */
+    ingestPath(path: string, options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Analyze content types in the document collection
+     * @private
+     */
+    private analyzeContentTypes;
+    /**
+     * Chunk all documents and organize results with content-type awareness
+     * Enhanced to handle different content types appropriately
+     */
+    private chunkDocumentsWithContentTypes;
+    /**
+     * Chunk all documents and organize results (legacy method for backward compatibility)
+     * @deprecated Use chunkDocumentsWithContentTypes for multimodal support
+     */
+    private chunkDocuments;
+    /**
+     * Generate embeddings for all chunks with content-type support
+     * Enhanced to handle different content types and pass metadata to embedding function
+     */
+    private generateEmbeddingsWithContentTypes;
+    /**
+     * Generate embeddings for all chunks with error handling (legacy method for backward compatibility)
+     * @deprecated Use generateEmbeddingsWithContentTypes for multimodal support
+     */
+    private generateEmbeddings;
+    /**
+     * Store documents and chunks in database with content-type support
+     * Enhanced to handle content type metadata and multimodal content
+     * @returns Array of content IDs for successfully stored documents
+     */
+    private storeDocumentsAndChunksWithContentTypes;
+    /**
+     * Store documents and chunks in database (legacy method for backward compatibility)
+     * @deprecated Use storeDocumentsAndChunksWithContentTypes for multimodal support
+     */
+    private storeDocumentsAndChunks;
+    /**
+     * Update vector index with new embeddings
+     */
+    private updateVectorIndex;
+    /**
+     * Converts MIME type to simple content type for embedding function
+     * @param mimeType - MIME type string (e.g., 'text/plain', 'image/jpeg')
+     * @returns Simple content type ('text', 'image', etc.)
+     */
+    private getContentTypeForEmbedding;
+    /**
+     * Save the vector index to disk
+     */
+    saveIndex(): Promise<void>;
+    /**
+     * Process image content from memory using the existing image processing pipeline
+     * @private
+     */
+    private processImageFromMemory;
+    /**
+     * Process PDF content from memory using the existing PDF processing pipeline
+     * @private
+     */
+    private processPDFFromMemory;
+    /**
+     * Process DOCX content from memory using the existing DOCX processing pipeline
+     * @private
+     */
+    private processDOCXFromMemory;
+    /**
+     * Clean up resources - explicit cleanup method
+     */
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=ingestion.d.ts.map