rag-lite-ts 2.1.1 → 2.3.0

package/dist/esm/core/universal-embedder.js

@@ -0,0 +1,139 @@
+/**
+ * CORE MODULE — Universal Embedder Interface for Chameleon Architecture
+ *
+ * Model-agnostic interfaces supporting both text and multimodal models.
+ * Designed for runtime polymorphism and extensibility.
+ *
+ * ARCHITECTURAL ROLE:
+ * This file contains ONLY interfaces, types, and utility functions - no implementation logic.
+ * It defines the contract that all embedder implementations must follow.
+ *
+ * CONTENTS:
+ * - UniversalEmbedder interface: Core contract for all embedders
+ * - Type definitions: ModelType, ContentType, etc.
+ * - Error classes: ModelValidationError, ContentTypeError, etc.
+ * - Utility functions: Type guards and validation helpers
+ * - Constants: Default capabilities and content types
+ *
+ * USAGE:
+ * - Implementation layers (text, multimodal) implement these interfaces
+ * - Core layer uses these types for dependency injection
+ * - Public API exports these types for external use
+ */
+// =============================================================================
+// ERROR TYPES
+// =============================================================================
+/**
+ * Model validation error for unsupported or incompatible models
+ */
+export class ModelValidationError extends Error {
+    modelName;
+    availableModels;
+    constructor(modelName, availableModels, message) {
+        super(message);
+        this.modelName = modelName;
+        this.availableModels = availableModels;
+        this.name = 'ModelValidationError';
+    }
+}
+/**
+ * Transformers.js compatibility error for version mismatches
+ */
+export class TransformersCompatibilityError extends Error {
+    modelName;
+    requiredVersion;
+    currentVersion;
+    constructor(modelName, requiredVersion, currentVersion, message) {
+        super(message);
+        this.modelName = modelName;
+        this.requiredVersion = requiredVersion;
+        this.currentVersion = currentVersion;
+        this.name = 'TransformersCompatibilityError';
+    }
+}
+/**
+ * Content type error for unsupported content types
+ */
+export class ContentTypeError extends Error {
+    contentType;
+    supportedTypes;
+    constructor(contentType, supportedTypes, message) {
+        super(message);
+        this.contentType = contentType;
+        this.supportedTypes = supportedTypes;
+        this.name = 'ContentTypeError';
+    }
+}
+// =============================================================================
+// UTILITY TYPES
+// =============================================================================
+/**
+ * Type guard for checking if an embedder supports images
+ */
+export function supportsImages(embedder) {
+    return embedder.supportedContentTypes.includes('image') &&
+        typeof embedder.embedImage === 'function';
+}
+/**
+ * Type guard for checking if an embedder supports a specific content type
+ */
+export function supportsContentType(embedder, contentType) {
+    return embedder.supportedContentTypes.includes(contentType);
+}
+/**
+ * Utility function to create enhanced embedding results
+ */
+export function createEnhancedEmbeddingResult(embeddingId, vector, contentType, metadata) {
+    return {
+        embedding_id: embeddingId,
+        vector,
+        contentType,
+        metadata
+    };
+}
+/**
+ * Utility function to validate content type against supported types
+ */
+export function validateContentType(contentType, supportedTypes) {
+    if (!supportedTypes.includes(contentType)) {
+        throw new ContentTypeError(contentType, supportedTypes, `Content type '${contentType}' is not supported. Supported types: ${supportedTypes.join(', ')}`);
+    }
+}
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+/**
+ * Default supported content types for different model types
+ */
+export const DEFAULT_CONTENT_TYPES = {
+    'sentence-transformer': ['text'],
+    'clip': ['text', 'image']
+};
+/**
+ * Default model capabilities for different model types
+ */
+export const DEFAULT_CAPABILITIES = {
+    'sentence-transformer': {
+        supportsText: true,
+        supportsImages: false,
+        supportsBatchProcessing: true,
+        supportsMetadata: true,
+        maxBatchSize: 32,
+        maxTextLength: 512
+    },
+    'clip': {
+        supportsText: true,
+        supportsImages: true,
+        supportsBatchProcessing: true,
+        supportsMetadata: true,
+        maxBatchSize: 16,
+        maxTextLength: 77,
+        supportedImageFormats: ['jpg', 'jpeg', 'png', 'webp', 'gif'],
+        // Enhanced capabilities for fixed CLIP implementation
+        supportsMultimodal: true,
+        supportsCrossModalSearch: true,
+        unifiedEmbeddingSpace: true,
+        reliableImplementation: true
+    }
+};
+//# sourceMappingURL=universal-embedder.js.map

package/dist/esm/core/validation-messages.d.ts

@@ -0,0 +1,99 @@
+/**
+ * CORE MODULE — Validation Messages and Error Descriptions
+ * Comprehensive error messages and user guidance for model validation
+ * Provides helpful, actionable error messages with troubleshooting steps
+ */
+import type { ModelType } from './universal-embedder.js';
+/**
+ * Error message templates for different validation scenarios
+ */
+export declare const ERROR_MESSAGES: {
+    readonly MODEL_NOT_FOUND: (modelName: string, suggestions: string[]) => {
+        title: string;
+        description: string;
+        details: string[];
+        suggestions: string[];
+        actions: string[];
+    };
+    readonly TRANSFORMERS_VERSION_INCOMPATIBLE: (modelName: string, required: string, current: string) => {
+        title: string;
+        description: string;
+        details: string[];
+        suggestions: string[];
+        actions: string[];
+    };
+    readonly INSUFFICIENT_MEMORY: (modelName: string, required: number, available: number) => {
+        title: string;
+        description: string;
+        details: string[];
+        suggestions: string[];
+        actions: string[];
+    };
+    readonly PLATFORM_UNSUPPORTED: (modelName: string, currentPlatform: string, supportedPlatforms: string[]) => {
+        title: string;
+        description: string;
+        details: string[];
+        suggestions: string[];
+        actions: string[];
+    };
+    readonly FEATURES_MISSING: (modelName: string, missingFeatures: string[]) => {
+        title: string;
+        description: string;
+        details: string[];
+        suggestions: string[];
+        actions: string[];
+    };
+    readonly CONTENT_TYPE_UNSUPPORTED: (contentType: string, modelName: string, supportedTypes: string[]) => {
+        title: string;
+        description: string;
+        details: string[];
+        suggestions: string[];
+        actions: string[];
+    };
+};
+/**
+ * Warning message templates for non-critical issues
+ */
+export declare const WARNING_MESSAGES: {
+    readonly HIGH_MEMORY_USAGE: (modelName: string, memoryMB: number) => {
+        title: string;
+        message: string;
+        suggestions: string[];
+    };
+    readonly LIMITED_BATCH_SIZE: (modelName: string, maxBatchSize: number) => {
+        title: string;
+        message: string;
+        suggestions: string[];
+    };
+    readonly EXPERIMENTAL_FEATURES: (modelName: string, features: string[]) => {
+        title: string;
+        message: string;
+        suggestions: string[];
+    };
+    readonly PERFORMANCE_IMPACT: (modelName: string, reason: string) => {
+        title: string;
+        message: string;
+        suggestions: string[];
+    };
+};
+/**
+ * Format an error message for console output
+ */
+export declare function formatErrorMessage(error: ReturnType<typeof ERROR_MESSAGES[keyof typeof ERROR_MESSAGES]>): string;
+/**
+ * Format a warning message for console output
+ */
+export declare function formatWarningMessage(warning: ReturnType<typeof WARNING_MESSAGES[keyof typeof WARNING_MESSAGES]>): string;
+/**
+ * Create a comprehensive error message for model validation failure
+ */
+export declare function createValidationErrorMessage(modelName: string, errorType: 'not_found' | 'version_incompatible' | 'insufficient_memory' | 'platform_unsupported' | 'features_missing' | 'content_type_unsupported', context?: Record<string, any>): string;
+/**
+ * Create helpful suggestions based on model type and use case
+ */
+export declare function createModelSuggestions(modelType?: ModelType, contentTypes?: string[], memoryLimit?: number): string[];
+/**
+ * Get troubleshooting steps for common issues
+ */
+export declare function getTroubleshootingSteps(issue: string): string[];
+//# sourceMappingURL=validation-messages.d.ts.map

package/dist/esm/core/validation-messages.js

@@ -0,0 +1,334 @@
+/**
+ * CORE MODULE — Validation Messages and Error Descriptions
+ * Comprehensive error messages and user guidance for model validation
+ * Provides helpful, actionable error messages with troubleshooting steps
+ */
+import { ModelRegistry } from './model-registry.js';
+// =============================================================================
+// ERROR MESSAGE TEMPLATES
+// =============================================================================
+/**
+ * Error message templates for different validation scenarios
+ */
+export const ERROR_MESSAGES = {
+    MODEL_NOT_FOUND: (modelName, suggestions) => ({
+        title: `Model '${modelName}' not found`,
+        description: `The specified model is not supported by the Chameleon architecture.`,
+        details: [
+            `Model '${modelName}' is not in the supported models registry.`,
+            `This could be due to a typo in the model name or the model not being compatible with transformers.js.`
+        ],
+        suggestions: suggestions.length > 0 ? [
+            `Did you mean one of these models?`,
+            ...suggestions.map(s => `  • ${s}`)
+        ] : [
+            `Available models:`,
+            ...ModelRegistry.getSupportedModels().map(s => `  • ${s}`)
+        ],
+        actions: [
+            `Check the model name for typos`,
+            `Use 'ModelRegistry.getSupportedModels()' to see all available models`,
+            `Visit the documentation for the latest supported models list`
+        ]
+    }),
+    TRANSFORMERS_VERSION_INCOMPATIBLE: (modelName, required, current) => ({
+        title: `Transformers.js version incompatible`,
+        description: `Model '${modelName}' requires a newer version of transformers.js.`,
+        details: [
+            `Required version: ${required}`,
+            `Current version: ${current}`,
+            `The model uses features not available in the current transformers.js version.`
+        ],
+        suggestions: [
+            `Upgrade transformers.js to the latest version:`,
+            `  npm install @huggingface/transformers@latest`,
+            ``,
+            `Or install a specific compatible version:`,
+            `  npm install @huggingface/transformers@${required.replace(/[>=<~^]/g, '')}`
+        ],
+        actions: [
+            `Update your package.json dependencies`,
+            `Run npm install to update transformers.js`,
+            `Restart your application after updating`
+        ]
+    }),
+    INSUFFICIENT_MEMORY: (modelName, required, available) => ({
+        title: `Insufficient memory for model`,
+        description: `Model '${modelName}' requires more memory than available.`,
+        details: [
+            `Required memory: ${required}MB`,
+            `Available memory: ${available}MB`,
+            `Shortfall: ${required - available}MB`
+        ],
+        suggestions: [
+            `Consider using a smaller model variant:`,
+            ...ModelRegistry.getSupportedModels().filter(name => {
+                const info = ModelRegistry.getModelInfo(name);
+                return info &&
+                    info.requirements.minimumMemory &&
+                    info.requirements.minimumMemory <= available;
+            }).map(name => `  • ${name}`),
+            ``,
+            `Or increase available memory by:`,
+            `  • Closing other applications`,
+            `  • Increasing Node.js memory limit: --max-old-space-size=${required + 512}`,
+            `  • Using a machine with more RAM`
+        ],
+        actions: [
+            `Free up system memory`,
+            `Choose a more memory-efficient model`,
+            `Consider using model quantization if available`
+        ]
+    }),
+    PLATFORM_UNSUPPORTED: (modelName, currentPlatform, supportedPlatforms) => ({
+        title: `Platform not supported`,
+        description: `Model '${modelName}' is not supported on ${currentPlatform}.`,
+        details: [
+            `Current platform: ${currentPlatform}`,
+            `Supported platforms: ${supportedPlatforms.join(', ')}`,
+            `The model may use platform-specific features or optimizations.`
+        ],
+        suggestions: [
+            `Try running on a supported platform:`,
+            ...supportedPlatforms.map(platform => `  • ${platform}`),
+            ``,
+            `Or use a platform-agnostic model:`,
+            ...ModelRegistry.getSupportedModels().filter(name => {
+                const info = ModelRegistry.getModelInfo(name);
+                return info &&
+                    info.requirements.platformSupport &&
+                    info.requirements.platformSupport.includes(currentPlatform);
+            }).slice(0, 3).map(name => `  • ${name}`)
+        ],
+        actions: [
+            `Switch to a supported platform`,
+            `Use a different model that supports your platform`,
+            `Check if there are platform-specific installation instructions`
+        ]
+    }),
+    FEATURES_MISSING: (modelName, missingFeatures) => ({
+        title: `Required features not available`,
+        description: `Model '${modelName}' requires features not available in current transformers.js version.`,
+        details: [
+            `Missing features: ${missingFeatures.join(', ')}`,
+            `These features are required for the model to function properly.`
+        ],
+        suggestions: [
+            `Upgrade transformers.js to get missing features:`,
+            `  npm install @huggingface/transformers@latest`,
+            ``,
+            `Or use a model that doesn't require these features:`,
+            ...ModelRegistry.getSupportedModels().filter(name => {
+                const info = ModelRegistry.getModelInfo(name);
+                return info &&
+                    (!info.requirements.requiredFeatures ||
+                        info.requirements.requiredFeatures.every(f => !missingFeatures.includes(f)));
+            }).slice(0, 3).map(name => `  • ${name}`)
+        ],
+        actions: [
+            `Update transformers.js to the latest version`,
+            `Check the transformers.js changelog for feature availability`,
+            `Use an alternative model with fewer feature requirements`
+        ]
+    }),
+    CONTENT_TYPE_UNSUPPORTED: (contentType, modelName, supportedTypes) => ({
+        title: `Content type not supported`,
+        description: `Model '${modelName}' does not support '${contentType}' content.`,
+        details: [
+            `Requested content type: ${contentType}`,
+            `Supported content types: ${supportedTypes.join(', ')}`,
+            `The model was not trained to handle this type of content.`
+        ],
+        suggestions: [
+            `Use a model that supports '${contentType}' content:`,
+            ...ModelRegistry.getModelsByContentType(contentType).slice(0, 3).map(name => `  • ${name}`),
+            ``,
+            `Or convert your content to a supported type:`,
+            ...supportedTypes.map(type => `  • Convert to ${type}`)
+        ],
+        actions: [
+            `Choose a multimodal model for mixed content types`,
+            `Preprocess your content to match supported types`,
+            `Use separate models for different content types`
+        ]
+    })
+};
+// =============================================================================
+// WARNING MESSAGE TEMPLATES
+// =============================================================================
+/**
+ * Warning message templates for non-critical issues
+ */
+export const WARNING_MESSAGES = {
+    HIGH_MEMORY_USAGE: (modelName, memoryMB) => ({
+        title: `High memory usage`,
+        message: `Model '${modelName}' requires ${memoryMB}MB of memory, which may impact performance.`,
+        suggestions: [
+            `Monitor system memory usage during operation`,
+            `Consider using a smaller model variant if performance is affected`,
+            `Ensure sufficient swap space is available`
+        ]
+    }),
+    LIMITED_BATCH_SIZE: (modelName, maxBatchSize) => ({
+        title: `Limited batch processing`,
+        message: `Model '${modelName}' supports maximum batch size of ${maxBatchSize}.`,
+        suggestions: [
+            `Use smaller batch sizes for optimal performance`,
+            `Process large datasets in smaller chunks`,
+            `Consider parallel processing with multiple model instances`
+        ]
+    }),
+    EXPERIMENTAL_FEATURES: (modelName, features) => ({
+        title: `Experimental features in use`,
+        message: `Model '${modelName}' uses experimental features: ${features.join(', ')}.`,
+        suggestions: [
+            `Test thoroughly before using in production`,
+            `Monitor for unexpected behavior or errors`,
+            `Have fallback options ready`,
+            `Check for updates that may stabilize these features`
+        ]
+    }),
+    PERFORMANCE_IMPACT: (modelName, reason) => ({
+        title: `Potential performance impact`,
+        message: `Model '${modelName}' may have reduced performance: ${reason}.`,
+        suggestions: [
+            `Monitor processing times and resource usage`,
+            `Consider using GPU acceleration if available`,
+            `Optimize batch sizes for your use case`,
+            `Profile your application to identify bottlenecks`
+        ]
+    })
+};
+// =============================================================================
+// MESSAGE FORMATTING UTILITIES
+// =============================================================================
+/**
+ * Format an error message for console output
+ */
+export function formatErrorMessage(error) {
+    const lines = [];
+    lines.push(`❌ ${error.title}`);
+    lines.push('');
+    lines.push(error.description);
+    if (error.details.length > 0) {
+        lines.push('');
+        lines.push('Details:');
+        error.details.forEach(detail => lines.push(`  ${detail}`));
+    }
+    if (error.suggestions.length > 0) {
+        lines.push('');
+        lines.push('Suggestions:');
+        error.suggestions.forEach(suggestion => lines.push(`  ${suggestion}`));
+    }
+    if (error.actions.length > 0) {
+        lines.push('');
+        lines.push('Actions:');
+        error.actions.forEach((action, index) => lines.push(`  ${index + 1}. ${action}`));
+    }
+    return lines.join('\n');
+}
+/**
+ * Format a warning message for console output
+ */
+export function formatWarningMessage(warning) {
+    const lines = [];
+    lines.push(`⚠️  ${warning.title}`);
+    lines.push('');
+    lines.push(warning.message);
+    if (warning.suggestions.length > 0) {
+        lines.push('');
+        lines.push('Suggestions:');
+        warning.suggestions.forEach(suggestion => lines.push(`  • ${suggestion}`));
+    }
+    return lines.join('\n');
+}
+/**
+ * Create a comprehensive error message for model validation failure
+ */
+export function createValidationErrorMessage(modelName, errorType, context = {}) {
+    switch (errorType) {
+        case 'not_found':
+            return formatErrorMessage(ERROR_MESSAGES.MODEL_NOT_FOUND(modelName, context.suggestions || []));
+        case 'version_incompatible':
+            return formatErrorMessage(ERROR_MESSAGES.TRANSFORMERS_VERSION_INCOMPATIBLE(modelName, context.required || 'unknown', context.current || 'unknown'));
+        case 'insufficient_memory':
+            return formatErrorMessage(ERROR_MESSAGES.INSUFFICIENT_MEMORY(modelName, context.required || 0, context.available || 0));
+        case 'platform_unsupported':
+            return formatErrorMessage(ERROR_MESSAGES.PLATFORM_UNSUPPORTED(modelName, context.currentPlatform || 'unknown', context.supportedPlatforms || []));
+        case 'features_missing':
+            return formatErrorMessage(ERROR_MESSAGES.FEATURES_MISSING(modelName, context.missingFeatures || []));
+        case 'content_type_unsupported':
+            return formatErrorMessage(ERROR_MESSAGES.CONTENT_TYPE_UNSUPPORTED(context.contentType || 'unknown', modelName, context.supportedTypes || []));
+        default:
+            return `❌ Validation failed for model '${modelName}': ${errorType}`;
+    }
+}
+/**
+ * Create helpful suggestions based on model type and use case
+ */
+export function createModelSuggestions(modelType, contentTypes, memoryLimit) {
+    const suggestions = [];
+    if (modelType === 'sentence-transformer') {
+        suggestions.push('For text-only tasks, sentence-transformers are most efficient');
+        suggestions.push('all-MiniLM-L6-v2 offers the best balance of speed and accuracy');
+        suggestions.push('all-mpnet-base-v2 provides higher accuracy but uses more memory');
+    }
+    if (modelType === 'clip') {
+        suggestions.push('CLIP models support both text and image content');
+        suggestions.push('clip-vit-base-patch32 is recommended for most use cases');
+        suggestions.push('patch16 variants are more accurate but slower');
+    }
+    if (contentTypes?.includes('image')) {
+        suggestions.push('Multimodal content requires CLIP models');
+        suggestions.push('Ensure images are in supported formats (jpg, png, webp)');
+        suggestions.push('Consider image preprocessing for better results');
+    }
+    if (memoryLimit && memoryLimit < 1024) {
+        suggestions.push('Low memory environments should use smaller models');
+        suggestions.push('Consider model quantization to reduce memory usage');
+        suggestions.push('Process content in smaller batches');
+    }
+    return suggestions;
+}
+/**
+ * Get troubleshooting steps for common issues
+ */
+export function getTroubleshootingSteps(issue) {
+    const steps = {
+        'model_loading_failed': [
+            'Check internet connection for model download',
+            'Verify model name spelling and availability',
+            'Ensure sufficient disk space for model cache',
+            'Try clearing the model cache and re-downloading',
+            'Check transformers.js version compatibility'
+        ],
+        'out_of_memory': [
+            'Reduce batch size for processing',
+            'Use a smaller model variant',
+            'Increase Node.js memory limit with --max-old-space-size',
+            'Close other memory-intensive applications',
+            'Consider using model quantization'
+        ],
+        'slow_performance': [
+            'Use GPU acceleration if available',
+            'Optimize batch sizes for your hardware',
+            'Consider using a smaller, faster model',
+            'Profile your code to identify bottlenecks',
+            'Use appropriate hardware for your model size'
+        ],
+        'compatibility_issues': [
+            'Update transformers.js to the latest version',
+            'Check model requirements against your environment',
+            'Verify platform compatibility (Node.js vs browser)',
+            'Test with a known working model first',
+            'Check for conflicting dependencies'
+        ]
+    };
+    return steps[issue] || [
+        'Check the documentation for your specific issue',
+        'Search for similar issues in the project repository',
+        'Ensure all dependencies are up to date',
+        'Try with a minimal test case to isolate the problem'
+    ];
+}
+//# sourceMappingURL=validation-messages.js.map

package/dist/esm/core/vector-index-messages.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Message protocol for VectorIndex worker thread communication
+ */
+export interface VectorIndexRequest {
+    id: number;
+    type: 'init' | 'loadIndex' | 'saveIndex' | 'addVector' | 'addVectors' | 'search' | 'getCurrentCount' | 'resizeIndex' | 'reset' | 'cleanup' | 'indexExists' | 'setEf';
+    payload?: any;
+}
+export interface VectorIndexResponse {
+    id: number;
+    type: 'success' | 'error';
+    payload?: any;
+    error?: string;
+}
+export interface InitPayload {
+    dimensions: number;
+    maxElements: number;
+    M?: number;
+    efConstruction?: number;
+    seed?: number;
+    indexPath: string;
+}
+export interface LoadIndexPayload {
+    indexPath: string;
+}
+export interface AddVectorPayload {
+    id: number;
+    vector: ArrayBuffer;
+    dimensions: number;
+}
+export interface AddVectorsPayload {
+    vectors: Array<{
+        id: number;
+        vector: ArrayBuffer;
+        dimensions: number;
+    }>;
+}
+export interface SearchPayload {
+    queryVector: ArrayBuffer;
+    dimensions: number;
+    k: number;
+}
+export interface ResizeIndexPayload {
+    newMaxElements: number;
+}
+export interface SetEfPayload {
+    ef: number;
+}
+export interface IndexExistsPayload {
+    indexPath: string;
+}
+//# sourceMappingURL=vector-index-messages.d.ts.map

package/dist/esm/core/vector-index-messages.js

@@ -0,0 +1,5 @@
+/**
+ * Message protocol for VectorIndex worker thread communication
+ */
+export {};
+//# sourceMappingURL=vector-index-messages.js.map

package/dist/esm/core/vector-index-worker.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Worker thread for VectorIndex operations
+ * Isolates hnswlib-wasm WebAssembly memory to prevent accumulation
+ */
+export {};
+//# sourceMappingURL=vector-index-worker.d.ts.map