rag-lite-ts 1.0.2 → 2.0.1

package/dist/core/validation-messages.js

@@ -0,0 +1,334 @@
+/**
+ * CORE MODULE — Validation Messages and Error Descriptions
+ * Comprehensive error messages and user guidance for model validation
+ * Provides helpful, actionable error messages with troubleshooting steps
+ */
+import { ModelRegistry } from './model-registry.js';
+// =============================================================================
+// ERROR MESSAGE TEMPLATES
+// =============================================================================
+/**
+ * Error message templates for different validation scenarios
+ */
+export const ERROR_MESSAGES = {
+    MODEL_NOT_FOUND: (modelName, suggestions) => ({
+        title: `Model '${modelName}' not found`,
+        description: `The specified model is not supported by the Chameleon architecture.`,
+        details: [
+            `Model '${modelName}' is not in the supported models registry.`,
+            `This could be due to a typo in the model name or the model not being compatible with transformers.js.`
+        ],
+        suggestions: suggestions.length > 0 ? [
+            `Did you mean one of these models?`,
+            ...suggestions.map(s => `  • ${s}`)
+        ] : [
+            `Available models:`,
+            ...ModelRegistry.getSupportedModels().map(s => `  • ${s}`)
+        ],
+        actions: [
+            `Check the model name for typos`,
+            `Use 'ModelRegistry.getSupportedModels()' to see all available models`,
+            `Visit the documentation for the latest supported models list`
+        ]
+    }),
+    TRANSFORMERS_VERSION_INCOMPATIBLE: (modelName, required, current) => ({
+        title: `Transformers.js version incompatible`,
+        description: `Model '${modelName}' requires a newer version of transformers.js.`,
+        details: [
+            `Required version: ${required}`,
+            `Current version: ${current}`,
+            `The model uses features not available in the current transformers.js version.`
+        ],
+        suggestions: [
+            `Upgrade transformers.js to the latest version:`,
+            `  npm install @huggingface/transformers@latest`,
+            ``,
+            `Or install a specific compatible version:`,
+            `  npm install @huggingface/transformers@${required.replace(/[>=<~^]/g, '')}`
+        ],
+        actions: [
+            `Update your package.json dependencies`,
+            `Run npm install to update transformers.js`,
+            `Restart your application after updating`
+        ]
+    }),
+    INSUFFICIENT_MEMORY: (modelName, required, available) => ({
+        title: `Insufficient memory for model`,
+        description: `Model '${modelName}' requires more memory than available.`,
+        details: [
+            `Required memory: ${required}MB`,
+            `Available memory: ${available}MB`,
+            `Shortfall: ${required - available}MB`
+        ],
+        suggestions: [
+            `Consider using a smaller model variant:`,
+            ...ModelRegistry.getSupportedModels().filter(name => {
+                const info = ModelRegistry.getModelInfo(name);
+                return info &&
+                    info.requirements.minimumMemory &&
+                    info.requirements.minimumMemory <= available;
+            }).map(name => `  • ${name}`),
+            ``,
+            `Or increase available memory by:`,
+            `  • Closing other applications`,
+            `  • Increasing Node.js memory limit: --max-old-space-size=${required + 512}`,
+            `  • Using a machine with more RAM`
+        ],
+        actions: [
+            `Free up system memory`,
+            `Choose a more memory-efficient model`,
+            `Consider using model quantization if available`
+        ]
+    }),
+    PLATFORM_UNSUPPORTED: (modelName, currentPlatform, supportedPlatforms) => ({
+        title: `Platform not supported`,
+        description: `Model '${modelName}' is not supported on ${currentPlatform}.`,
+        details: [
+            `Current platform: ${currentPlatform}`,
+            `Supported platforms: ${supportedPlatforms.join(', ')}`,
+            `The model may use platform-specific features or optimizations.`
+        ],
+        suggestions: [
+            `Try running on a supported platform:`,
+            ...supportedPlatforms.map(platform => `  • ${platform}`),
+            ``,
+            `Or use a platform-agnostic model:`,
+            ...ModelRegistry.getSupportedModels().filter(name => {
+                const info = ModelRegistry.getModelInfo(name);
+                return info &&
+                    info.requirements.platformSupport &&
+                    info.requirements.platformSupport.includes(currentPlatform);
+            }).slice(0, 3).map(name => `  • ${name}`)
+        ],
+        actions: [
+            `Switch to a supported platform`,
+            `Use a different model that supports your platform`,
+            `Check if there are platform-specific installation instructions`
+        ]
+    }),
+    FEATURES_MISSING: (modelName, missingFeatures) => ({
+        title: `Required features not available`,
+        description: `Model '${modelName}' requires features not available in current transformers.js version.`,
+        details: [
+            `Missing features: ${missingFeatures.join(', ')}`,
+            `These features are required for the model to function properly.`
+        ],
+        suggestions: [
+            `Upgrade transformers.js to get missing features:`,
+            `  npm install @huggingface/transformers@latest`,
+            ``,
+            `Or use a model that doesn't require these features:`,
+            ...ModelRegistry.getSupportedModels().filter(name => {
+                const info = ModelRegistry.getModelInfo(name);
+                return info &&
+                    (!info.requirements.requiredFeatures ||
+                        info.requirements.requiredFeatures.every(f => !missingFeatures.includes(f)));
+            }).slice(0, 3).map(name => `  • ${name}`)
+        ],
+        actions: [
+            `Update transformers.js to the latest version`,
+            `Check the transformers.js changelog for feature availability`,
+            `Use an alternative model with fewer feature requirements`
+        ]
+    }),
+    CONTENT_TYPE_UNSUPPORTED: (contentType, modelName, supportedTypes) => ({
+        title: `Content type not supported`,
+        description: `Model '${modelName}' does not support '${contentType}' content.`,
+        details: [
+            `Requested content type: ${contentType}`,
+            `Supported content types: ${supportedTypes.join(', ')}`,
+            `The model was not trained to handle this type of content.`
+        ],
+        suggestions: [
+            `Use a model that supports '${contentType}' content:`,
+            ...ModelRegistry.getModelsByContentType(contentType).slice(0, 3).map(name => `  • ${name}`),
+            ``,
+            `Or convert your content to a supported type:`,
+            ...supportedTypes.map(type => `  • Convert to ${type}`)
+        ],
+        actions: [
+            `Choose a multimodal model for mixed content types`,
+            `Preprocess your content to match supported types`,
+            `Use separate models for different content types`
+        ]
+    })
+};
+// =============================================================================
+// WARNING MESSAGE TEMPLATES
+// =============================================================================
+/**
+ * Warning message templates for non-critical issues
+ */
+export const WARNING_MESSAGES = {
+    HIGH_MEMORY_USAGE: (modelName, memoryMB) => ({
+        title: `High memory usage`,
+        message: `Model '${modelName}' requires ${memoryMB}MB of memory, which may impact performance.`,
+        suggestions: [
+            `Monitor system memory usage during operation`,
+            `Consider using a smaller model variant if performance is affected`,
+            `Ensure sufficient swap space is available`
+        ]
+    }),
+    LIMITED_BATCH_SIZE: (modelName, maxBatchSize) => ({
+        title: `Limited batch processing`,
+        message: `Model '${modelName}' supports maximum batch size of ${maxBatchSize}.`,
+        suggestions: [
+            `Use smaller batch sizes for optimal performance`,
+            `Process large datasets in smaller chunks`,
+            `Consider parallel processing with multiple model instances`
+        ]
+    }),
+    EXPERIMENTAL_FEATURES: (modelName, features) => ({
+        title: `Experimental features in use`,
+        message: `Model '${modelName}' uses experimental features: ${features.join(', ')}.`,
+        suggestions: [
+            `Test thoroughly before using in production`,
+            `Monitor for unexpected behavior or errors`,
+            `Have fallback options ready`,
+            `Check for updates that may stabilize these features`
+        ]
+    }),
+    PERFORMANCE_IMPACT: (modelName, reason) => ({
+        title: `Potential performance impact`,
+        message: `Model '${modelName}' may have reduced performance: ${reason}.`,
+        suggestions: [
+            `Monitor processing times and resource usage`,
+            `Consider using GPU acceleration if available`,
+            `Optimize batch sizes for your use case`,
+            `Profile your application to identify bottlenecks`
+        ]
+    })
+};
+// =============================================================================
+// MESSAGE FORMATTING UTILITIES
+// =============================================================================
+/**
+ * Format an error message for console output
+ */
+export function formatErrorMessage(error) {
+    const lines = [];
+    lines.push(`❌ ${error.title}`);
+    lines.push('');
+    lines.push(error.description);
+    if (error.details.length > 0) {
+        lines.push('');
+        lines.push('Details:');
+        error.details.forEach(detail => lines.push(`  ${detail}`));
+    }
+    if (error.suggestions.length > 0) {
+        lines.push('');
+        lines.push('Suggestions:');
+        error.suggestions.forEach(suggestion => lines.push(`  ${suggestion}`));
+    }
+    if (error.actions.length > 0) {
+        lines.push('');
+        lines.push('Actions:');
+        error.actions.forEach((action, index) => lines.push(`  ${index + 1}. ${action}`));
+    }
+    return lines.join('\n');
+}
+/**
+ * Format a warning message for console output
+ */
+export function formatWarningMessage(warning) {
+    const lines = [];
+    lines.push(`⚠️  ${warning.title}`);
+    lines.push('');
+    lines.push(warning.message);
+    if (warning.suggestions.length > 0) {
+        lines.push('');
+        lines.push('Suggestions:');
+        warning.suggestions.forEach(suggestion => lines.push(`  • ${suggestion}`));
+    }
+    return lines.join('\n');
+}
+/**
+ * Create a comprehensive error message for model validation failure
+ */
+export function createValidationErrorMessage(modelName, errorType, context = {}) {
+    switch (errorType) {
+        case 'not_found':
+            return formatErrorMessage(ERROR_MESSAGES.MODEL_NOT_FOUND(modelName, context.suggestions || []));
+        case 'version_incompatible':
+            return formatErrorMessage(ERROR_MESSAGES.TRANSFORMERS_VERSION_INCOMPATIBLE(modelName, context.required || 'unknown', context.current || 'unknown'));
+        case 'insufficient_memory':
+            return formatErrorMessage(ERROR_MESSAGES.INSUFFICIENT_MEMORY(modelName, context.required || 0, context.available || 0));
+        case 'platform_unsupported':
+            return formatErrorMessage(ERROR_MESSAGES.PLATFORM_UNSUPPORTED(modelName, context.currentPlatform || 'unknown', context.supportedPlatforms || []));
+        case 'features_missing':
+            return formatErrorMessage(ERROR_MESSAGES.FEATURES_MISSING(modelName, context.missingFeatures || []));
+        case 'content_type_unsupported':
+            return formatErrorMessage(ERROR_MESSAGES.CONTENT_TYPE_UNSUPPORTED(context.contentType || 'unknown', modelName, context.supportedTypes || []));
+        default:
+            return `❌ Validation failed for model '${modelName}': ${errorType}`;
+    }
+}
+/**
+ * Create helpful suggestions based on model type and use case
+ */
+export function createModelSuggestions(modelType, contentTypes, memoryLimit) {
+    const suggestions = [];
+    if (modelType === 'sentence-transformer') {
+        suggestions.push('For text-only tasks, sentence-transformers are most efficient');
+        suggestions.push('all-MiniLM-L6-v2 offers the best balance of speed and accuracy');
+        suggestions.push('all-mpnet-base-v2 provides higher accuracy but uses more memory');
+    }
+    if (modelType === 'clip') {
+        suggestions.push('CLIP models support both text and image content');
+        suggestions.push('clip-vit-base-patch32 is recommended for most use cases');
+        suggestions.push('patch16 variants are more accurate but slower');
+    }
+    if (contentTypes?.includes('image')) {
+        suggestions.push('Multimodal content requires CLIP models');
+        suggestions.push('Ensure images are in supported formats (jpg, png, webp)');
+        suggestions.push('Consider image preprocessing for better results');
+    }
+    if (memoryLimit && memoryLimit < 1024) {
+        suggestions.push('Low memory environments should use smaller models');
+        suggestions.push('Consider model quantization to reduce memory usage');
+        suggestions.push('Process content in smaller batches');
+    }
+    return suggestions;
+}
+/**
+ * Get troubleshooting steps for common issues
+ */
+export function getTroubleshootingSteps(issue) {
+    const steps = {
+        'model_loading_failed': [
+            'Check internet connection for model download',
+            'Verify model name spelling and availability',
+            'Ensure sufficient disk space for model cache',
+            'Try clearing the model cache and re-downloading',
+            'Check transformers.js version compatibility'
+        ],
+        'out_of_memory': [
+            'Reduce batch size for processing',
+            'Use a smaller model variant',
+            'Increase Node.js memory limit with --max-old-space-size',
+            'Close other memory-intensive applications',
+            'Consider using model quantization'
+        ],
+        'slow_performance': [
+            'Use GPU acceleration if available',
+            'Optimize batch sizes for your hardware',
+            'Consider using a smaller, faster model',
+            'Profile your code to identify bottlenecks',
+            'Use appropriate hardware for your model size'
+        ],
+        'compatibility_issues': [
+            'Update transformers.js to the latest version',
+            'Check model requirements against your environment',
+            'Verify platform compatibility (Node.js vs browser)',
+            'Test with a known working model first',
+            'Check for conflicting dependencies'
+        ]
+    };
+    return steps[issue] || [
+        'Check the documentation for your specific issue',
+        'Search for similar issues in the project repository',
+        'Ensure all dependencies are up to date',
+        'Try with a minimal test case to isolate the problem'
+    ];
+}
+//# sourceMappingURL=validation-messages.js.map

package/dist/core/vector-index.d.ts

@@ -30,7 +30,7 @@ export declare class VectorIndex {
      */
     loadIndex(): Promise<void>;
     /**
-     * Save index to file using JSON format (since IDBFS doesn't work in Node.js)
+     * Save index to binary format
      */
     saveIndex(): Promise<void>;
     /**

package/dist/core/vector-index.js

@@ -2,9 +2,11 @@
  * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
  * Model-agnostic. No transformer or modality-specific logic.
  */
-import { readFileSync, writeFileSync, existsSync } from 'fs';
+import { existsSync } from 'fs';
 import { JSDOM } from 'jsdom';
 import { ErrorCategory, ErrorSeverity, safeExecute } from './error-handler.js';
+import { createMissingFileError, createDimensionMismatchError } from './actionable-error-messages.js';
+import { BinaryIndexFormat } from './binary-index-format.js';
 // Set up browser-like environment for hnswlib-wasm
 if (typeof window === 'undefined') {
     const dom = new JSDOM('<!DOCTYPE html><html><body></body></html>', {
@@ -103,7 +105,9 @@ export class VectorIndex {
      */
     async loadIndex() {
         if (!existsSync(this.indexPath)) {
-            throw new Error(`Index file not found: ${this.indexPath}`);
+            throw createMissingFileError(this.indexPath, 'index', {
+                operationContext: 'VectorIndex.loadIndex'
+            });
         }
         try {
             // Load the hnswlib module
@@ -150,70 +154,64 @@ export class VectorIndex {
             }
             // Create new HNSW index (third parameter is autoSaveFilename, but we'll handle persistence manually)
             this.index = new this.hnswlib.HierarchicalNSW('cosine', this.options.dimensions, '');
-            // Load from JSON format since IDBFS doesn't work in Node.js
-            const data = readFileSync(this.indexPath, 'utf-8');
-            const stored = JSON.parse(data);
-            // Check dimension compatibility and log details
-            if (stored.dimensions && stored.dimensions !== this.options.dimensions) {
+            // Load from binary format
+            const data = await BinaryIndexFormat.load(this.indexPath);
+            // Validate dimensions
+            if (data.dimensions !== this.options.dimensions) {
                 console.log(`⚠️  Dimension mismatch detected:`);
-                console.log(`   Stored dimensions: ${stored.dimensions}`);
+                console.log(`   Stored dimensions: ${data.dimensions}`);
                 console.log(`   Expected dimensions: ${this.options.dimensions}`);
-                console.log(`   Number of vectors: ${stored.vectors?.length || 0}`);
-                if (stored.vectors && stored.vectors.length > 0) {
-                    console.log(`   Actual vector length: ${stored.vectors[0].vector.length}`);
+                console.log(`   Number of vectors: ${data.vectors.length}`);
+                if (data.vectors.length > 0) {
+                    console.log(`   Actual vector length: ${data.vectors[0].vector.length}`);
                 }
-                throw new Error(`Vector dimension mismatch!\n` +
-                    `Index was created with ${stored.dimensions} dimensions\n` +
-                    `Current model expects ${this.options.dimensions} dimensions\n` +
-                    `This indicates the embedding model has changed.\n` +
-                    `Please rebuild the index with the current model.`);
+                throw createDimensionMismatchError(this.options.dimensions, data.dimensions, 'vector index loading', { operationContext: 'VectorIndex.loadIndex' });
             }
             // Update options from stored data
-            this.options.maxElements = stored.maxElements || this.options.maxElements;
-            this.options.M = stored.M || this.options.M;
-            this.options.efConstruction = stored.efConstruction || this.options.efConstruction;
-            this.options.seed = stored.seed || this.options.seed;
-            // Recreate the index from stored data
-            this.index.initIndex(this.options.maxElements, this.options.M || 16, this.options.efConstruction || 200, this.options.seed || 100);
+            this.options.maxElements = data.maxElements;
+            this.options.M = data.M;
+            this.options.efConstruction = data.efConstruction;
+            this.options.seed = data.seed;
+            // Initialize HNSW index
+            this.index.initIndex(this.options.maxElements, this.options.M, this.options.efConstruction, this.options.seed);
             // Clear and repopulate vector storage
             this.vectorStorage.clear();
-            // Add all stored vectors back
-            for (const item of stored.vectors || []) {
-                const vector = new Float32Array(item.vector);
-                this.index.addPoint(vector, item.id, false);
-                this.vectorStorage.set(item.id, vector);
+            // Add all stored vectors to HNSW index
+            for (const item of data.vectors) {
+                this.index.addPoint(item.vector, item.id, false);
+                this.vectorStorage.set(item.id, item.vector);
             }
-            this.currentSize = stored.vectors?.length || 0;
-            console.log(`Loaded HNSW index with ${this.currentSize} vectors from ${this.indexPath}`);
+            this.currentSize = data.currentSize;
+            console.log(`✓ Loaded HNSW index with ${this.currentSize} vectors from ${this.indexPath}`);
         }
         catch (error) {
             throw new Error(`Failed to load index from ${this.indexPath}: ${error}`);
         }
     }
     /**
-     * Save index to file using JSON format (since IDBFS doesn't work in Node.js)
+     * Save index to binary format
      */
     async saveIndex() {
         if (!this.index) {
             throw new Error('Index not initialized');
         }
         try {
-            // Convert stored vectors to serializable format
+            // Collect all vectors from storage
             const vectors = Array.from(this.vectorStorage.entries()).map(([id, vector]) => ({
                 id,
-                vector: Array.from(vector)
+                vector
             }));
-            const stored = {
+            // Save to binary format
+            await BinaryIndexFormat.save(this.indexPath, {
                 dimensions: this.options.dimensions,
                 maxElements: this.options.maxElements,
                 M: this.options.M || 16,
                 efConstruction: this.options.efConstruction || 200,
                 seed: this.options.seed || 100,
                 currentSize: this.currentSize,
-                vectors: vectors
-            };
-            writeFileSync(this.indexPath, JSON.stringify(stored, null, 2));
-            console.log(`Saved HNSW index with ${this.currentSize} vectors to ${this.indexPath}`);
+                vectors
+            });
+            console.log(`✓ Saved HNSW index with ${this.currentSize} vectors to ${this.indexPath}`);
         }
         catch (error) {
             throw new Error(`Failed to save index to ${this.indexPath}: ${error}`);
@@ -227,7 +225,7 @@ export class VectorIndex {
             throw new Error('Index not initialized');
         }
         if (vector.length !== this.options.dimensions) {
-            throw new Error(`Vector dimension mismatch: expected ${this.options.dimensions}, got ${vector.length}`);
+            throw createDimensionMismatchError(this.options.dimensions, vector.length, 'vector addition', { operationContext: 'VectorIndex.addVector' });
         }
         try {
             this.index.addPoint(vector, embeddingId, false);
@@ -255,7 +253,7 @@ export class VectorIndex {
             throw new Error('Index not initialized');
         }
         if (queryVector.length !== this.options.dimensions) {
-            throw new Error(`Query vector dimension mismatch: expected ${this.options.dimensions}, got ${queryVector.length}`);
+            throw createDimensionMismatchError(this.options.dimensions, queryVector.length, 'vector search', { operationContext: 'VectorIndex.search' });
         }
         if (this.currentSize === 0) {
             return { neighbors: [], distances: [] };

package/dist/factories/index.d.ts

@@ -36,7 +36,9 @@
  * ```
  */
 export { TextSearchFactory, TextIngestionFactory, TextRAGFactory, TextFactoryHelpers } from './text-factory.js';
-export type { TextSearchOptions, TextIngestionOptions } from './text-factory.js';
+export { PolymorphicSearchFactory } from './polymorphic-factory.js';
+export type { PolymorphicSearchOptions } from './polymorphic-factory.js';
+export type { TextSearchOptions, TextIngestionOptions, ContentSystemConfig } from './text-factory.js';
 export { TextSearchFactory as SearchFactory } from './text-factory.js';
 export { TextIngestionFactory as IngestionFactory } from './text-factory.js';
 export { TextRAGFactory as RAGFactory } from './text-factory.js';

package/dist/factories/index.js

@@ -37,6 +37,8 @@
  */
 // Main factory classes
 export { TextSearchFactory, TextIngestionFactory, TextRAGFactory, TextFactoryHelpers } from './text-factory.js';
+// Polymorphic factory for mode-aware search
+export { PolymorphicSearchFactory } from './polymorphic-factory.js';
 // Convenience re-exports for common patterns
 export { TextSearchFactory as SearchFactory } from './text-factory.js';
 export { TextIngestionFactory as IngestionFactory } from './text-factory.js';

package/dist/factories/polymorphic-factory.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Polymorphic factory for creating mode-aware search engines
+ * Automatically detects mode from database and uses appropriate embedder
+ *
+ * This factory implements the Chameleon Architecture principle:
+ * - Detects mode (text/multimodal) from database configuration
+ * - Uses appropriate embedder based on detected mode
+ * - Provides seamless polymorphic behavior without user intervention
+ *
+ * @example
+ * ```typescript
+ * // Automatically detects mode and creates appropriate search engine
+ * const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
+ *
+ * // Works for both text and multimodal modes
+ * const results = await search.search('query');
+ * ```
+ */
+import { SearchEngine } from '../core/search.js';
+export interface PolymorphicSearchOptions {
+    /** Whether to enable reranking (default: true) */
+    enableReranking?: boolean;
+    /** Top-k results to return (default: from config) */
+    topK?: number;
+}
+/**
+ * Factory for creating mode-aware search engines
+ * Automatically detects mode from database and uses appropriate embedder
+ */
+export declare class PolymorphicSearchFactory {
+    /**
+     * Create a SearchEngine that automatically adapts to the mode stored in the database
+     *
+     * This method:
+     * 1. Validates that required files exist
+     * 2. Opens database and reads system configuration
+     * 3. Detects mode (text/multimodal) from database
+     * 4. Creates appropriate embedder based on mode
+     * 5. Optionally creates reranker based on configuration
+     * 6. Returns fully configured SearchEngine
+     *
+     * @param indexPath - Path to the vector index file (must exist)
+     * @param dbPath - Path to the SQLite database file (must exist)
+     * @param options - Optional configuration overrides
+     * @returns Promise resolving to configured SearchEngine
+     * @throws {Error} If required files don't exist or initialization fails
+     */
+    static create(indexPath: string, dbPath: string, options?: PolymorphicSearchOptions): Promise<SearchEngine>;
+}
+//# sourceMappingURL=polymorphic-factory.d.ts.map