rag-lite-ts 2.1.1 → 2.3.0

package/dist/esm/index.d.ts

@@ -0,0 +1,76 @@
+/**
+ * rag-lite-ts - Clean Architecture with Factory Pattern
+ *
+ * Quick Start (Recommended):
+ * ```typescript
+ * import { SearchFactory, IngestionFactory } from 'rag-lite-ts';
+ *
+ * // Simple search - just works!
+ * const search = await SearchFactory.create('./index.bin', './db.sqlite');
+ * const results = await search.search('your query');
+ *
+ * // Simple ingestion - just works!
+ * const ingestion = await IngestionFactory.create('./db.sqlite', './index.bin');
+ * await ingestion.ingestDirectory('./documents');
+ * ```
+ *
+ * With Configuration:
+ * ```typescript
+ * const search = await SearchFactory.create('./index.bin', './db.sqlite', {
+ *   embeddingModel: 'Xenova/all-mpnet-base-v2',
+ *   enableReranking: true
+ * });
+ * ```
+ *
+ * Complete RAG System:
+ * ```typescript
+ * import { RAGFactory } from 'rag-lite-ts';
+ *
+ * const { searchEngine, ingestionPipeline } = await RAGFactory.createBoth(
+ *   './index.bin',
+ *   './db.sqlite'
+ * );
+ * ```
+ *
+ * Advanced Usage (Direct Dependency Injection):
+ * ```typescript
+ * import { CoreSearchEngine, createTextEmbedFunction } from 'rag-lite-ts';
+ *
+ * const embedFn = await createTextEmbedFunction();
+ * const search = new CoreSearchEngine(embedFn, indexManager, db);
+ * ```
+ */
+export { IngestionFactory, SearchFactory } from './factories/index.js';
+export type { IngestionFactoryOptions } from './factories/index.js';
+export { SearchEngine as CoreSearchEngine } from './core/search.js';
+export { IngestionPipeline as CoreIngestionPipeline } from './core/ingestion.js';
+export { SearchEngine } from './search.js';
+export { IngestionPipeline } from './ingestion.js';
+export { LazyEmbedderLoader, LazyRerankerLoader, LazyMultimodalLoader, LazyDependencyManager } from './core/lazy-dependency-loader.js';
+export type { EmbedFunction, RerankFunction, EmbeddingQueryInterface, RerankingInterface, SearchEngineConfig, ContentTypeStrategy, ModelAgnosticInterface, ExtendedEmbeddingInterface, ExtendedRerankingInterface, SearchPipelineInterface, SearchDependencyFactory } from './core/interfaces.js';
+export { InterfaceValidator } from './core/interfaces.js';
+export { validateModeModelCompatibility, validateModeModelCompatibilityOrThrow, getRecommendedModelsForMode, isModeModelCompatible, getCompatibleModelsForMode, type ModeModelValidationResult } from './core/mode-model-validator.js';
+export { createMissingFileError, createInvalidPathError, createModelLoadingError, createDimensionMismatchError, createModeMismatchError, createInvalidContentError, createMissingDependencyError, createFactoryCreationError, enhanceError, createContextualError, type ActionableErrorConfig } from './core/actionable-error-messages.js';
+export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction } from './text/embedder.js';
+export type { UniversalEmbedder } from './core/universal-embedder.js';
+export { CLIPEmbedder } from './multimodal/clip-embedder.js';
+export { createEmbedder } from './core/embedder-factory.js';
+export { CrossEncoderReranker, createTextRerankFunction } from './text/reranker.js';
+export { countTokens } from './text/tokenizer.js';
+export type { RerankingStrategyType, RerankingConfig } from './core/reranking-config.js';
+export { validateRerankingStrategy, validateRerankingConfig, getDefaultRerankingConfig, isStrategySupported, getSupportedStrategies, RerankingConfigBuilder, DEFAULT_TEXT_RERANKING_CONFIG, DEFAULT_MULTIMODAL_RERANKING_CONFIG } from './core/reranking-config.js';
+export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, resetDatabase, hasDatabaseData, type DatabaseConnection, type DatabaseResetOptions, type DatabaseResetResult } from './core/db.js';
+export { KnowledgeBaseManager, type KnowledgeBaseResetOptions, type KnowledgeBaseResetResult } from './core/knowledge-base-manager.js';
+export { IndexManager } from './index-manager.js';
+export { VectorIndex } from './core/vector-index.js';
+export { config, getModelDefaults, type CoreConfig, type ExtensibleConfig, type ModelDefaults, EXIT_CODES, ConfigurationError, getDefaultModelCachePath, handleUnrecoverableError, logError } from './core/config.js';
+export { discoverFiles, processFiles, discoverAndProcessFiles, DEFAULT_FILE_PROCESSOR_OPTIONS, type FileProcessorOptions, type FileDiscoveryResult, type DocumentProcessingResult } from './file-processor.js';
+export { chunkDocument, type ChunkConfig } from './core/chunker.js';
+export { DocumentPathManager } from './core/path-manager.js';
+export { resolveRagLitePaths, ensureRagLiteStructure, migrateToRagLiteStructure, getStandardRagLitePaths, type RagLiteConfig, type RagLitePaths } from './core/raglite-paths.js';
+export type { SearchResult, SearchOptions, Document, EmbeddingResult, ContentDocument, ContentChunk } from './core/types.js';
+export type { Chunk, Preprocessor, PreprocessorOptions, PreprocessingConfig } from './types.js';
+export type { IngestionOptions, IngestionResult } from './core/ingestion.js';
+export { handleError, safeExecute, ErrorCategory, ErrorSeverity, createError, type ErrorContext } from './core/error-handler.js';
+export { APIError, IngestionError, SearchError, ResourceError, ModelCompatibilityError, ErrorFactory, CommonErrors, handleAPIError } from './api-errors.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/index.js

@@ -0,0 +1,112 @@
+/**
+ * rag-lite-ts - Clean Architecture with Factory Pattern
+ *
+ * Quick Start (Recommended):
+ * ```typescript
+ * import { SearchFactory, IngestionFactory } from 'rag-lite-ts';
+ *
+ * // Simple search - just works!
+ * const search = await SearchFactory.create('./index.bin', './db.sqlite');
+ * const results = await search.search('your query');
+ *
+ * // Simple ingestion - just works!
+ * const ingestion = await IngestionFactory.create('./db.sqlite', './index.bin');
+ * await ingestion.ingestDirectory('./documents');
+ * ```
+ *
+ * With Configuration:
+ * ```typescript
+ * const search = await SearchFactory.create('./index.bin', './db.sqlite', {
+ *   embeddingModel: 'Xenova/all-mpnet-base-v2',
+ *   enableReranking: true
+ * });
+ * ```
+ *
+ * Complete RAG System:
+ * ```typescript
+ * import { RAGFactory } from 'rag-lite-ts';
+ *
+ * const { searchEngine, ingestionPipeline } = await RAGFactory.createBoth(
+ *   './index.bin',
+ *   './db.sqlite'
+ * );
+ * ```
+ *
+ * Advanced Usage (Direct Dependency Injection):
+ * ```typescript
+ * import { CoreSearchEngine, createTextEmbedFunction } from 'rag-lite-ts';
+ *
+ * const embedFn = await createTextEmbedFunction();
+ * const search = new CoreSearchEngine(embedFn, indexManager, db);
+ * ```
+ */
+// =============================================================================
+// PRIMARY API (FACTORY PATTERN)
+// =============================================================================
+// Main factory classes for simple usage
+export { IngestionFactory, SearchFactory } from './factories/index.js';
+// =============================================================================
+// REMOVED IN v3.0.0: Type aliases
+// =============================================================================
+// IngestionPipelineOptions has been removed. Use IngestionFactoryOptions directly.
+// Migration: Replace IngestionPipelineOptions with IngestionFactoryOptions
+// =============================================================================
+// CORE ARCHITECTURE (FOR LIBRARY AUTHORS)
+// =============================================================================
+// Core classes for direct dependency injection (advanced)
+export { SearchEngine as CoreSearchEngine } from './core/search.js';
+export { IngestionPipeline as CoreIngestionPipeline } from './core/ingestion.js';
+// Public API classes
+export { SearchEngine } from './search.js';
+export { IngestionPipeline } from './ingestion.js';
+// Lazy loading system for performance optimization
+export { LazyEmbedderLoader, LazyRerankerLoader, LazyMultimodalLoader, LazyDependencyManager } from './core/lazy-dependency-loader.js';
+// Interface validation utilities
+export { InterfaceValidator } from './core/interfaces.js';
+// Mode-model compatibility validation
+export { validateModeModelCompatibility, validateModeModelCompatibilityOrThrow, getRecommendedModelsForMode, isModeModelCompatible, getCompatibleModelsForMode } from './core/mode-model-validator.js';
+// Actionable error messages
+export { createMissingFileError, createInvalidPathError, createModelLoadingError, createDimensionMismatchError, createModeMismatchError, createInvalidContentError, createMissingDependencyError, createFactoryCreationError, enhanceError, createContextualError } from './core/actionable-error-messages.js';
+// =============================================================================
+// TEXT IMPLEMENTATIONS (FOR CUSTOM DEPENDENCY INJECTION)
+// =============================================================================
+// Text-specific embedding implementations
+export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction } from './text/embedder.js';
+export { CLIPEmbedder } from './multimodal/clip-embedder.js';
+export { createEmbedder } from './core/embedder-factory.js';
+// Text-specific reranking implementations
+export { CrossEncoderReranker, createTextRerankFunction } from './text/reranker.js';
+// Text tokenization utilities
+export { countTokens } from './text/tokenizer.js';
+export { validateRerankingStrategy, validateRerankingConfig, getDefaultRerankingConfig, isStrategySupported, getSupportedStrategies, RerankingConfigBuilder, DEFAULT_TEXT_RERANKING_CONFIG, DEFAULT_MULTIMODAL_RERANKING_CONFIG } from './core/reranking-config.js';
+// =============================================================================
+// CORE INFRASTRUCTURE (FOR ADVANCED USERS)
+// =============================================================================
+// Database operations
+export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, resetDatabase, hasDatabaseData } from './core/db.js';
+// Knowledge Base Manager (for reset operations)
+export { KnowledgeBaseManager } from './core/knowledge-base-manager.js';
+// Vector index management
+export { IndexManager } from './index-manager.js';
+export { VectorIndex } from './core/vector-index.js';
+// Configuration and utilities
+export { config, getModelDefaults, EXIT_CODES, ConfigurationError, getDefaultModelCachePath, handleUnrecoverableError, logError } from './core/config.js';
+// =============================================================================
+// FILE PROCESSING AND UTILITIES
+// =============================================================================
+// File processing operations
+export { discoverFiles, processFiles, discoverAndProcessFiles, DEFAULT_FILE_PROCESSOR_OPTIONS } from './file-processor.js';
+// Document chunking
+export { chunkDocument } from './core/chunker.js';
+// Path management
+export { DocumentPathManager } from './core/path-manager.js';
+// RAG-lite directory structure management
+export { resolveRagLitePaths, ensureRagLiteStructure, migrateToRagLiteStructure, getStandardRagLitePaths } from './core/raglite-paths.js';
+// =============================================================================
+// ERROR HANDLING
+// =============================================================================
+// Core error handling
+export { handleError, safeExecute, ErrorCategory, ErrorSeverity, createError } from './core/error-handler.js';
+// API-specific errors
+export { APIError, IngestionError, SearchError, ResourceError, ModelCompatibilityError, ErrorFactory, CommonErrors, handleAPIError } from './api-errors.js';
+//# sourceMappingURL=index.js.map

package/dist/esm/indexer.d.ts

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+/**
+ * Standalone indexer script for direct node execution
+ * Usage: node indexer.js <path>
+ */
+export {};
+//# sourceMappingURL=indexer.d.ts.map

package/dist/esm/indexer.js

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+/**
+ * Standalone indexer script for direct node execution
+ * Usage: node indexer.js <path>
+ */
+import { runIngest } from './cli/indexer.js';
+import { EXIT_CODES, ConfigurationError } from './core/config.js';
+async function main() {
+    const args = process.argv.slice(2);
+    if (args.length === 0) {
+        console.error('RAG-lite TS Document Indexer');
+        console.error('');
+        console.error('Usage: node indexer.js <path>');
+        console.error('');
+        console.error('Arguments:');
+        console.error('  <path>    File or directory path to ingest (.md and .txt files)');
+        console.error('');
+        console.error('Examples:');
+        console.error('  node indexer.js ./docs/           # Ingest all .md/.txt/.pdf/.docs files in docs/');
+        console.error('  node indexer.js ./readme.md       # Ingest single file');
+        console.error('  node indexer.js ../project/docs/  # Ingest from parent directory');
+        console.error('');
+        console.error('Supported file types:');
+        console.error('  Text: .md, .txt, .mdx');
+        console.error('  Documents: .pdf, .docx');
+        console.error('  Images (multimodal mode): .jpg, .jpeg, .png, .gif, .webp, .bmp');
+        console.error('');
+        console.error('After ingestion, use: node search.js "your query"');
+        process.exit(EXIT_CODES.INVALID_ARGUMENTS);
+    }
+    if (args.length > 1) {
+        console.error('Error: Too many arguments provided');
+        console.error('');
+        console.error('Usage: node indexer.js <path>');
+        console.error('');
+        console.error('If your path contains spaces, wrap it in quotes:');
+        console.error('  node indexer.js "my documents folder"');
+        process.exit(EXIT_CODES.INVALID_ARGUMENTS);
+    }
+    const path = args[0];
+    await runIngest(path);
+}
+main().catch((error) => {
+    if (error instanceof ConfigurationError) {
+        console.error('Configuration Error:');
+        console.error(error.message);
+        process.exit(error.exitCode);
+    }
+    else {
+        console.error('Fatal Error:', error instanceof Error ? error.message : String(error));
+        process.exit(EXIT_CODES.GENERAL_ERROR);
+    }
+});
+//# sourceMappingURL=indexer.js.map

package/dist/esm/ingestion.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Public API IngestionPipeline - Simple constructor interface with internal factory usage
+ *
+ * This class provides a clean, simple API while using the new core architecture
+ * internally. It handles dependency injection automatically.
+ *
+ * @example
+ * ```typescript
+ * // Simple usage
+ * const pipeline = new IngestionPipeline('./db.sqlite', './index.bin');
+ * await pipeline.ingestDirectory('./documents');
+ *
+ * // With options
+ * const pipeline = new IngestionPipeline('./db.sqlite', './index.bin', {
+ *   embeddingModel: 'all-MiniLM-L6-v2',
+ *   chunkSize: 512
+ * });
+ * ```
+ */
+import { type IngestionFactoryOptions } from './factories/index.js';
+import type { IngestionOptions, IngestionResult } from './core/ingestion.js';
+import type { MemoryContentMetadata } from './core/content-manager.js';
+export declare class IngestionPipeline {
+    private dbPath;
+    private indexPath;
+    private options;
+    private corePipeline;
+    private initPromise;
+    private defaultChunkConfig;
+    constructor(dbPath: string, indexPath: string, options?: IngestionFactoryOptions);
+    /**
+     * Initialize the ingestion pipeline using the factory
+     */
+    private initialize;
+    /**
+     * Ingest a single document
+     */
+    ingestDocument(filePath: string, options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Ingest all documents in a directory
+     */
+    ingestDirectory(directoryPath: string, options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Ingest content from memory buffer
+     * Enables MCP integration and real-time content processing
+     *
+     * @example
+     * ```typescript
+     * const pipeline = new IngestionPipeline('./db.sqlite', './index.bin');
+     * const contentId = await pipeline.ingestFromMemory(buffer, {
+     *   displayName: 'uploaded-file.txt',
+     *   contentType: 'text/plain'
+     * });
+     * console.log('Content ingested with ID:', contentId);
+     * ```
+     */
+    ingestFromMemory(content: Buffer, metadata: MemoryContentMetadata, options?: IngestionOptions): Promise<string>;
+    /**
+     * Clean up resources
+     */
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=ingestion.d.ts.map

package/dist/esm/ingestion.js

@@ -0,0 +1,124 @@
+/**
+ * Public API IngestionPipeline - Simple constructor interface with internal factory usage
+ *
+ * This class provides a clean, simple API while using the new core architecture
+ * internally. It handles dependency injection automatically.
+ *
+ * @example
+ * ```typescript
+ * // Simple usage
+ * const pipeline = new IngestionPipeline('./db.sqlite', './index.bin');
+ * await pipeline.ingestDirectory('./documents');
+ *
+ * // With options
+ * const pipeline = new IngestionPipeline('./db.sqlite', './index.bin', {
+ *   embeddingModel: 'all-MiniLM-L6-v2',
+ *   chunkSize: 512
+ * });
+ * ```
+ */
+import { IngestionFactory } from './factories/index.js';
+export class IngestionPipeline {
+    dbPath;
+    indexPath;
+    options;
+    corePipeline = null;
+    initPromise = null;
+    defaultChunkConfig = null;
+    constructor(dbPath, indexPath, options = {}) {
+        this.dbPath = dbPath;
+        this.indexPath = indexPath;
+        this.options = options;
+        // Validate required parameters
+        if (!dbPath || typeof dbPath !== 'string' || dbPath.trim() === '') {
+            throw new Error('Both dbPath and indexPath are required.\n' +
+                'Example: const ingestion = new IngestionPipeline("./db.sqlite", "./index.bin");\n' +
+                'Or use: const ingestion = await IngestionFactory.create("./db.sqlite", "./index.bin");');
+        }
+        if (!indexPath || typeof indexPath !== 'string' || indexPath.trim() === '') {
+            throw new Error('Both dbPath and indexPath are required.\n' +
+                'Example: const ingestion = new IngestionPipeline("./db.sqlite", "./index.bin");\n' +
+                'Or use: const ingestion = await IngestionFactory.create("./db.sqlite", "./index.bin");');
+        }
+    }
+    /**
+     * Initialize the ingestion pipeline using the factory
+     */
+    async initialize() {
+        if (this.corePipeline) {
+            return; // Already initialized
+        }
+        if (this.initPromise) {
+            return this.initPromise; // Initialization in progress
+        }
+        this.initPromise = (async () => {
+            this.corePipeline = await IngestionFactory.create(this.dbPath, this.indexPath, this.options);
+        })();
+        return this.initPromise;
+    }
+    /**
+     * Ingest a single document
+     */
+    async ingestDocument(filePath, options) {
+        await this.initialize();
+        if (!this.corePipeline) {
+            throw new Error('IngestionPipeline failed to initialize');
+        }
+        // Merge mode from constructor options with runtime options
+        const mergedOptions = {
+            ...options,
+            mode: options?.mode || this.options.mode
+        };
+        return this.corePipeline.ingestFile(filePath, mergedOptions);
+    }
+    /**
+     * Ingest all documents in a directory
+     */
+    async ingestDirectory(directoryPath, options) {
+        await this.initialize();
+        if (!this.corePipeline) {
+            throw new Error('IngestionPipeline failed to initialize');
+        }
+        // Merge mode from constructor options with runtime options
+        const mergedOptions = {
+            ...options,
+            mode: options?.mode || this.options.mode
+        };
+        return this.corePipeline.ingestDirectory(directoryPath, mergedOptions);
+    }
+    /**
+     * Ingest content from memory buffer
+     * Enables MCP integration and real-time content processing
+     *
+     * @example
+     * ```typescript
+     * const pipeline = new IngestionPipeline('./db.sqlite', './index.bin');
+     * const contentId = await pipeline.ingestFromMemory(buffer, {
+     *   displayName: 'uploaded-file.txt',
+     *   contentType: 'text/plain'
+     * });
+     * console.log('Content ingested with ID:', contentId);
+     * ```
+     */
+    async ingestFromMemory(content, metadata, options) {
+        await this.initialize();
+        if (!this.corePipeline) {
+            throw new Error('IngestionPipeline failed to initialize');
+        }
+        // Merge mode from constructor options with runtime options
+        const mergedOptions = {
+            ...options,
+            mode: options?.mode || this.options.mode
+        };
+        return this.corePipeline.ingestFromMemory(content, metadata, mergedOptions);
+    }
+    /**
+     * Clean up resources
+     */
+    async cleanup() {
+        if (this.corePipeline) {
+            await this.corePipeline.cleanup();
+        }
+    }
+}
+//# sourceMappingURL=ingestion.js.map

package/dist/esm/mcp-server.d.ts

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+/**
+ * MCP server entry point for rag-lite-ts with Chameleon Multimodal Architecture
+ *
+ * This is a thin wrapper around the polymorphic search and ingestion functions
+ * that exposes them as MCP tools without creating REST/GraphQL endpoints.
+ *
+ * The MCP server supports both text-only and multimodal modes:
+ * - Text mode: Optimized for text documents using sentence-transformer models
+ * - Multimodal mode: Supports mixed text and image content using CLIP models
+ *
+ * Key Features:
+ * - Automatic mode detection from database configuration
+ * - Polymorphic runtime that adapts to stored mode settings
+ * - Support for multiple embedding models and reranking strategies
+ * - Content type filtering and multimodal search capabilities
+ * - Comprehensive model and strategy information tools
+ *
+ * The MCP server lives in the same package as CLI with dual entry points
+ * and provides proper MCP tool definitions for search and indexing capabilities.
+ *
+ * Requirements addressed: 6.2, 6.4, 6.5, 6.6, 9.1, 9.2, 9.3
+ */
+export {};
+/**
+ * MCP Server Multimodal Integration Complete
+ *
+ * This implementation addresses task 9.3 requirements:
+ * ✅ Updated MCP server configuration to support multimodal parameters
+ * ✅ Added new MCP tools for mode configuration and multimodal search
+ * ✅ Integrated with polymorphic runtime system and mode detection
+ * ✅ Enhanced error handling for multimodal-specific errors
+ * ✅ Created comprehensive documentation and examples
+ * ✅ Added support for content type filtering and model selection
+ * ✅ Implemented reranking strategy configuration
+ * ✅ Provided detailed system information and statistics tools
+ *
+ * Key Features Added:
+ * - Multimodal ingestion with mode and model parameters
+ * - Content type filtering in search operations
+ * - Comprehensive model and strategy information tools
+ * - Enhanced error handling with recovery guidance
+ * - Automatic mode detection and polymorphic behavior
+ * - Detailed documentation and configuration examples
+ */ 
+//# sourceMappingURL=mcp-server.d.ts.map