rag-lite-ts 1.0.2 → 2.0.1

package/dist/core/embedder-factory.d.ts

@@ -0,0 +1,176 @@
+/**
+ * CORE MODULE — Simple Embedder Creation Function
+ *
+ * Provides direct model instantiation with clear validation and error handling.
+ * No fallback mechanisms - models work reliably or fail clearly with actionable guidance.
+ *
+ * Supported Models:
+ * - Text Mode: sentence-transformers/all-MiniLM-L6-v2, Xenova/all-mpnet-base-v2
+ * - Multimodal Mode: Xenova/clip-vit-base-patch32, Xenova/clip-vit-base-patch16
+ *
+ * Mode Selection Guide:
+ * - Use text mode for text-only content (faster, optimized for text similarity)
+ * - Use multimodal mode for mixed text/image content (enables cross-modal search)
+ */
+import '../dom-polyfills.js';
+import type { UniversalEmbedder, ModelType, EmbedderCreationOptions } from './universal-embedder.js';
+/**
+ * Create a universal embedder for the specified model
+ *
+ * Simple function-based approach that validates model compatibility and creates
+ * the appropriate embedder. Models work reliably without fallback mechanisms -
+ * if there's an issue, you'll get clear error messages with actionable guidance.
+ *
+ * Mode Selection:
+ * - Text Mode: Use sentence-transformer models for text-only content
+ *   - Fast, optimized for text similarity
+ *   - Best for: document search, semantic similarity, text clustering
+ *
+ * - Multimodal Mode: Use CLIP models for mixed text/image content
+ *   - Unified embedding space for text and images
+ *   - Enables cross-modal search (text queries find images, image queries find text)
+ *   - Best for: image search, visual question answering, multimodal retrieval
+ *
+ * @param modelName - Name of the model to create
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a UniversalEmbedder instance
+ * @throws {Error} If model is not supported or validation fails
+ *
+ * @example
+ * ```typescript
+ * // Text mode - optimized for text-only content
+ * const textEmbedder = await createEmbedder('sentence-transformers/all-MiniLM-L6-v2');
+ * const textResult = await textEmbedder.embedText('machine learning');
+ *
+ * // Multimodal mode - enables cross-modal search
+ * const clipEmbedder = await createEmbedder('Xenova/clip-vit-base-patch32');
+ * const textResult = await clipEmbedder.embedText('red sports car');
+ * const imageResult = await clipEmbedder.embedImage('./car.jpg');
+ *
+ * // Create with custom options
+ * const embedder = await createEmbedder('sentence-transformers/all-MiniLM-L6-v2', {
+ *   maxBatchSize: 16,
+ *   cachePath: './models'
+ * });
+ * ```
+ */
+export declare function createEmbedder(modelName: string, options?: EmbedderCreationOptions): Promise<UniversalEmbedder>;
+/**
+ * Get supported models for a specific content type
+ * Convenience function for filtering models by capability
+ *
+ * @param contentType - Content type to filter by ('text', 'image', etc.)
+ * @returns Array of model names that support the content type
+ *
+ * @example
+ * ```typescript
+ * const textModels = getSupportedModelsForContentType('text');
+ * const imageModels = getSupportedModelsForContentType('image');
+ * ```
+ */
+export declare function getSupportedModelsForContentType(contentType: string): string[];
+/**
+ * Get recommended model for a specific use case
+ *
+ * Provides intelligent model selection based on content types and constraints.
+ * Returns models that work reliably for the specified requirements.
+ *
+ * Mode Selection Guide:
+ * - Text only (['text']): Returns sentence-transformer models
+ *   - Fast, optimized for text similarity
+ *   - Best for document search and text clustering
+ *
+ * - Text + Images (['text', 'image']): Returns CLIP models
+ *   - Unified embedding space for cross-modal search
+ *   - Text queries can find images, image queries can find text
+ *   - Best for visual search and multimodal retrieval
+ *
+ * @param contentTypes - Required content types
+ * @param constraints - Optional constraints (memory, performance, etc.)
+ * @param constraints.maxMemory - Maximum memory in MB
+ * @param constraints.preferPerformance - Prefer faster models
+ * @param constraints.preferAccuracy - Prefer more accurate models
+ * @returns Recommended model name or null if no suitable model found
+ *
+ * @example
+ * ```typescript
+ * // Get best text model (fast, optimized for text)
+ * const textModel = getRecommendedModel(['text']);
+ * // Returns: 'sentence-transformers/all-MiniLM-L6-v2'
+ *
+ * // Get best multimodal model (enables cross-modal search)
+ * const multimodalModel = getRecommendedModel(['text', 'image']);
+ * // Returns: 'Xenova/clip-vit-base-patch32'
+ *
+ * // Get performance-optimized model
+ * const fastModel = getRecommendedModel(['text'], { preferPerformance: true });
+ *
+ * // Get accuracy-optimized model
+ * const accurateModel = getRecommendedModel(['text'], { preferAccuracy: true });
+ * ```
+ */
+export declare function getRecommendedModel(contentTypes: string[], constraints?: {
+    maxMemory?: number;
+    preferPerformance?: boolean;
+    preferAccuracy?: boolean;
+}): string | null;
+/**
+ * Validate model compatibility before creation
+ * Useful for checking compatibility without creating the embedder
+ *
+ * @param modelName - Name of the model to validate
+ * @returns Promise resolving to validation result
+ *
+ * @example
+ * ```typescript
+ * const isValid = await validateModelCompatibility('sentence-transformers/all-MiniLM-L6-v2');
+ * if (isValid) {
+ *   const embedder = await createEmbedder('sentence-transformers/all-MiniLM-L6-v2');
+ * }
+ * ```
+ */
+export declare function validateModelCompatibility(modelName: string): Promise<boolean>;
+/**
+ * List all available models with their capabilities
+ * Useful for displaying model options to users
+ *
+ * @returns Array of model information objects
+ *
+ * @example
+ * ```typescript
+ * const models = listAvailableModels();
+ * models.forEach(model => {
+ *   console.log(`${model.name}: ${model.supportedContentTypes.join(', ')}`);
+ * });
+ * ```
+ */
+export declare function listAvailableModels(): Array<{
+    name: string;
+    type: ModelType;
+    dimensions: number;
+    supportedContentTypes: readonly string[];
+    memoryRequirement: number | undefined;
+}>;
+/**
+ * @deprecated Use createEmbedder() instead
+ * Legacy factory-style interface for backward compatibility
+ */
+export declare const UniversalEmbedderFactory: {
+    /**
+     * @deprecated Use createEmbedder() instead
+     */
+    readonly create: (modelName: string, options?: EmbedderCreationOptions) => Promise<UniversalEmbedder>;
+    /**
+     * @deprecated Use ModelRegistry.validateModel() instead
+     */
+    readonly validateModel: (modelName: string) => import("./universal-embedder.js").ModelValidationResult;
+    /**
+     * @deprecated Use ModelRegistry.getModelInfo() instead
+     */
+    readonly getModelInfo: (modelName: string) => import("./universal-embedder.js").ModelInfo | null;
+    /**
+     * @deprecated Use ModelRegistry.getSupportedModels() instead
+     */
+    readonly getSupportedModels: (modelType?: ModelType) => string[];
+};
+//# sourceMappingURL=embedder-factory.d.ts.map

package/dist/core/embedder-factory.js

@@ -0,0 +1,338 @@
+/**
+ * CORE MODULE — Simple Embedder Creation Function
+ *
+ * Provides direct model instantiation with clear validation and error handling.
+ * No fallback mechanisms - models work reliably or fail clearly with actionable guidance.
+ *
+ * Supported Models:
+ * - Text Mode: sentence-transformers/all-MiniLM-L6-v2, Xenova/all-mpnet-base-v2
+ * - Multimodal Mode: Xenova/clip-vit-base-patch32, Xenova/clip-vit-base-patch16
+ *
+ * Mode Selection Guide:
+ * - Use text mode for text-only content (faster, optimized for text similarity)
+ * - Use multimodal mode for mixed text/image content (enables cross-modal search)
+ */
+// Ensure DOM polyfills are set up before any transformers.js usage
+import '../dom-polyfills.js';
+import { ModelRegistry } from './model-registry.js';
+import { ModelValidator } from './model-validator.js';
+import { createModelValidationError } from './model-validator.js';
+import { createValidationErrorMessage } from './validation-messages.js';
+// =============================================================================
+// SIMPLE EMBEDDER CREATION FUNCTION
+// =============================================================================
+/**
+ * Create a universal embedder for the specified model
+ *
+ * Simple function-based approach that validates model compatibility and creates
+ * the appropriate embedder. Models work reliably without fallback mechanisms -
+ * if there's an issue, you'll get clear error messages with actionable guidance.
+ *
+ * Mode Selection:
+ * - Text Mode: Use sentence-transformer models for text-only content
+ *   - Fast, optimized for text similarity
+ *   - Best for: document search, semantic similarity, text clustering
+ *
+ * - Multimodal Mode: Use CLIP models for mixed text/image content
+ *   - Unified embedding space for text and images
+ *   - Enables cross-modal search (text queries find images, image queries find text)
+ *   - Best for: image search, visual question answering, multimodal retrieval
+ *
+ * @param modelName - Name of the model to create
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a UniversalEmbedder instance
+ * @throws {Error} If model is not supported or validation fails
+ *
+ * @example
+ * ```typescript
+ * // Text mode - optimized for text-only content
+ * const textEmbedder = await createEmbedder('sentence-transformers/all-MiniLM-L6-v2');
+ * const textResult = await textEmbedder.embedText('machine learning');
+ *
+ * // Multimodal mode - enables cross-modal search
+ * const clipEmbedder = await createEmbedder('Xenova/clip-vit-base-patch32');
+ * const textResult = await clipEmbedder.embedText('red sports car');
+ * const imageResult = await clipEmbedder.embedImage('./car.jpg');
+ *
+ * // Create with custom options
+ * const embedder = await createEmbedder('sentence-transformers/all-MiniLM-L6-v2', {
+ *   maxBatchSize: 16,
+ *   cachePath: './models'
+ * });
+ * ```
+ */
+export async function createEmbedder(modelName, options = {}) {
+    // Step 0: Ensure polyfills are set up before any transformers.js usage
+    if (typeof globalThis.self === 'undefined') {
+        globalThis.self = globalThis;
+    }
+    if (typeof global.self === 'undefined') {
+        global.self = global;
+    }
+    // Step 1: Initialize model validator if not already done
+    if (!ModelValidator.getTransformersVersion()) {
+        await ModelValidator.detectTransformersVersion();
+    }
+    // Step 1: Validate the model
+    const modelInfo = ModelRegistry.getModelInfo(modelName);
+    if (!modelInfo) {
+        const errorMessage = createValidationErrorMessage(modelName, 'not_found', {
+            suggestions: ModelRegistry.getSupportedModels()
+        });
+        console.error(errorMessage);
+        throw createModelValidationError(modelName, 'Model not found in supported models registry');
+    }
+    // Step 2: Perform detailed validation
+    try {
+        const detailedValidation = await ModelValidator.validateModelDetailed(modelName);
+        if (!detailedValidation.isValid) {
+            const firstError = detailedValidation.errors[0] || 'Validation failed';
+            const errorMessage = createValidationErrorMessage(modelName, 'version_incompatible', {
+                required: modelInfo.requirements.transformersJsVersion,
+                current: ModelValidator.getTransformersVersion() || 'unknown'
+            });
+            console.error(errorMessage);
+            throw createModelValidationError(modelName, firstError);
+        }
+        // Log warnings if any
+        if (detailedValidation.warnings.length > 0) {
+            console.warn(`⚠️  Warnings for model '${modelName}':`);
+            detailedValidation.warnings.forEach(warning => console.warn(`  • ${warning}`));
+        }
+        // Log suggestions if any
+        if (detailedValidation.suggestions.length > 0) {
+            console.info(`💡 Suggestions for model '${modelName}':`);
+            detailedValidation.suggestions.forEach(suggestion => console.info(`  • ${suggestion}`));
+        }
+    }
+    catch (error) {
+        // Re-throw validation errors
+        if (error instanceof Error && error.name === 'ModelValidationError') {
+            throw error;
+        }
+        // Handle unexpected validation errors
+        console.warn(`Warning: Could not perform detailed validation for '${modelName}': ${error}`);
+        console.info('Proceeding with basic validation only...');
+    }
+    // Step 3: Create the appropriate embedder based on model type
+    const modelType = modelInfo.type;
+    switch (modelType) {
+        case 'sentence-transformer':
+            return await createSentenceTransformerEmbedder(modelName, options);
+        case 'clip':
+            return await createCLIPEmbedder(modelName, options);
+        default:
+            const errorMessage = createValidationErrorMessage(modelName, 'not_found', {
+                suggestions: [`Unsupported model type: ${modelType}`]
+            });
+            console.error(errorMessage);
+            throw createModelValidationError(modelName, `Unsupported model type: ${modelType}. Supported types: sentence-transformer, clip`);
+    }
+}
+// =============================================================================
+// MODEL-SPECIFIC CREATION FUNCTIONS
+// =============================================================================
+/**
+ * Create a sentence transformer embedder using lazy loading
+ * @private
+ */
+async function createSentenceTransformerEmbedder(modelName, options) {
+    // Use lazy loading to avoid loading text dependencies unless needed
+    const { LazyEmbedderLoader } = await import('./lazy-dependency-loader.js');
+    return LazyEmbedderLoader.loadSentenceTransformerEmbedder(modelName, options);
+}
+/**
+ * Create a CLIP embedder using lazy loading
+ * @private
+ */
+async function createCLIPEmbedder(modelName, options) {
+    // Use lazy loading to avoid loading multimodal dependencies unless needed
+    const { LazyEmbedderLoader } = await import('./lazy-dependency-loader.js');
+    return LazyEmbedderLoader.loadCLIPEmbedder(modelName, options);
+}
+// =============================================================================
+// UTILITY FUNCTIONS
+// =============================================================================
+/**
+ * Get supported models for a specific content type
+ * Convenience function for filtering models by capability
+ *
+ * @param contentType - Content type to filter by ('text', 'image', etc.)
+ * @returns Array of model names that support the content type
+ *
+ * @example
+ * ```typescript
+ * const textModels = getSupportedModelsForContentType('text');
+ * const imageModels = getSupportedModelsForContentType('image');
+ * ```
+ */
+export function getSupportedModelsForContentType(contentType) {
+    return ModelRegistry.getModelsByContentType(contentType);
+}
+/**
+ * Get recommended model for a specific use case
+ *
+ * Provides intelligent model selection based on content types and constraints.
+ * Returns models that work reliably for the specified requirements.
+ *
+ * Mode Selection Guide:
+ * - Text only (['text']): Returns sentence-transformer models
+ *   - Fast, optimized for text similarity
+ *   - Best for document search and text clustering
+ *
+ * - Text + Images (['text', 'image']): Returns CLIP models
+ *   - Unified embedding space for cross-modal search
+ *   - Text queries can find images, image queries can find text
+ *   - Best for visual search and multimodal retrieval
+ *
+ * @param contentTypes - Required content types
+ * @param constraints - Optional constraints (memory, performance, etc.)
+ * @param constraints.maxMemory - Maximum memory in MB
+ * @param constraints.preferPerformance - Prefer faster models
+ * @param constraints.preferAccuracy - Prefer more accurate models
+ * @returns Recommended model name or null if no suitable model found
+ *
+ * @example
+ * ```typescript
+ * // Get best text model (fast, optimized for text)
+ * const textModel = getRecommendedModel(['text']);
+ * // Returns: 'sentence-transformers/all-MiniLM-L6-v2'
+ *
+ * // Get best multimodal model (enables cross-modal search)
+ * const multimodalModel = getRecommendedModel(['text', 'image']);
+ * // Returns: 'Xenova/clip-vit-base-patch32'
+ *
+ * // Get performance-optimized model
+ * const fastModel = getRecommendedModel(['text'], { preferPerformance: true });
+ *
+ * // Get accuracy-optimized model
+ * const accurateModel = getRecommendedModel(['text'], { preferAccuracy: true });
+ * ```
+ */
+export function getRecommendedModel(contentTypes, constraints = {}) {
+    const transformersVersion = ModelValidator.getTransformersVersion();
+    const compatibleModels = ModelValidator.getRecommendedModels(contentTypes, constraints.maxMemory, transformersVersion || undefined);
+    if (compatibleModels.length === 0) {
+        return null;
+    }
+    // Apply preference-based sorting
+    if (constraints.preferPerformance) {
+        // Prefer smaller, faster models
+        const performanceOrder = [
+            'sentence-transformers/all-MiniLM-L6-v2',
+            'Xenova/clip-vit-base-patch32',
+            'Xenova/all-mpnet-base-v2',
+            'Xenova/clip-vit-base-patch16'
+        ];
+        for (const preferred of performanceOrder) {
+            if (compatibleModels.includes(preferred)) {
+                return preferred;
+            }
+        }
+    }
+    if (constraints.preferAccuracy) {
+        // Prefer larger, more accurate models
+        const accuracyOrder = [
+            'Xenova/all-mpnet-base-v2',
+            'Xenova/clip-vit-base-patch16',
+            'sentence-transformers/all-MiniLM-L6-v2',
+            'Xenova/clip-vit-base-patch32'
+        ];
+        for (const preferred of accuracyOrder) {
+            if (compatibleModels.includes(preferred)) {
+                return preferred;
+            }
+        }
+    }
+    // Default: return first compatible model
+    return compatibleModels[0];
+}
+/**
+ * Validate model compatibility before creation
+ * Useful for checking compatibility without creating the embedder
+ *
+ * @param modelName - Name of the model to validate
+ * @returns Promise resolving to validation result
+ *
+ * @example
+ * ```typescript
+ * const isValid = await validateModelCompatibility('sentence-transformers/all-MiniLM-L6-v2');
+ * if (isValid) {
+ *   const embedder = await createEmbedder('sentence-transformers/all-MiniLM-L6-v2');
+ * }
+ * ```
+ */
+export async function validateModelCompatibility(modelName) {
+    try {
+        const validation = await ModelValidator.validateModelDetailed(modelName);
+        return validation.isValid;
+    }
+    catch (error) {
+        console.warn(`Validation failed for '${modelName}': ${error}`);
+        return false;
+    }
+}
+/**
+ * List all available models with their capabilities
+ * Useful for displaying model options to users
+ *
+ * @returns Array of model information objects
+ *
+ * @example
+ * ```typescript
+ * const models = listAvailableModels();
+ * models.forEach(model => {
+ *   console.log(`${model.name}: ${model.supportedContentTypes.join(', ')}`);
+ * });
+ * ```
+ */
+export function listAvailableModels() {
+    return ModelRegistry.getSupportedModels().map(modelName => {
+        const info = ModelRegistry.getModelInfo(modelName);
+        return {
+            name: info.name,
+            type: info.type,
+            dimensions: info.dimensions,
+            supportedContentTypes: info.supportedContentTypes,
+            memoryRequirement: info.requirements.minimumMemory
+        };
+    });
+}
+// =============================================================================
+// BACKWARD COMPATIBILITY
+// =============================================================================
+/**
+ * @deprecated Use createEmbedder() instead
+ * Legacy factory-style interface for backward compatibility
+ */
+export const UniversalEmbedderFactory = {
+    /**
+     * @deprecated Use createEmbedder() instead
+     */
+    async create(modelName, options) {
+        console.warn('UniversalEmbedderFactory.create() is deprecated. Use createEmbedder() instead.');
+        return createEmbedder(modelName, options);
+    },
+    /**
+     * @deprecated Use ModelRegistry.validateModel() instead
+     */
+    validateModel(modelName) {
+        console.warn('UniversalEmbedderFactory.validateModel() is deprecated. Use ModelRegistry.validateModel() instead.');
+        return ModelRegistry.validateModel(modelName);
+    },
+    /**
+     * @deprecated Use ModelRegistry.getModelInfo() instead
+     */
+    getModelInfo(modelName) {
+        console.warn('UniversalEmbedderFactory.getModelInfo() is deprecated. Use ModelRegistry.getModelInfo() instead.');
+        return ModelRegistry.getModelInfo(modelName);
+    },
+    /**
+     * @deprecated Use ModelRegistry.getSupportedModels() instead
+     */
+    getSupportedModels(modelType) {
+        console.warn('UniversalEmbedderFactory.getSupportedModels() is deprecated. Use ModelRegistry.getSupportedModels() instead.');
+        return ModelRegistry.getSupportedModels(modelType);
+    }
+};
+//# sourceMappingURL=embedder-factory.js.map

package/dist/core/index.d.ts

@@ -47,11 +47,13 @@ export { type ContentDocument, type ContentChunk, type Document, type Chunk, typ
 export { type EmbedFunction, type RerankFunction, type EmbeddingQueryInterface, type RerankingInterface, type SearchEngineConfig, type ContentTypeStrategy, type ModelAgnosticInterface, type ExtendedEmbeddingInterface, type ExtendedRerankingInterface, type SearchPipelineInterface, type SearchDependencyFactory, InterfaceValidator } from './interfaces.js';
 export * from './adapters.js';
 export * from './config.js';
-export { type DatabaseConnection, openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, getModelVersion, setModelVersion, getStoredModelInfo, setStoredModelInfo } from './db.js';
+export { type DatabaseConnection, type ContentMetadata, openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, getModelVersion, setModelVersion, getStoredModelInfo, setStoredModelInfo, insertContentMetadata, getContentMetadata, getContentMetadataByHash, getContentMetadataByStorageType, deleteContentMetadata, getStorageStats, updateStorageStats } from './db.js';
 export { type VectorIndexOptions, VectorIndex } from './vector-index.js';
 export { type ChunkConfig, type GenericDocument, type GenericChunk, type ChunkingStrategy, ChunkingStrategyRegistry, DEFAULT_CHUNK_CONFIG, chunkingRegistry, chunkGenericDocument, registerTextChunkingStrategy } from './chunker.js';
 export * from './search.js';
 export * from './ingestion.js';
 export * from './path-manager.js';
+export { ContentManager, type MemoryContentMetadata, type ContentIngestionResult, type ContentManagerConfig } from './content-manager.js';
+export { ContentResolver, type ContentRequest, type ContentResult } from './content-resolver.js';
 export * from './error-handler.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/core/index.js

@@ -50,7 +50,7 @@ export * from './adapters.js';
 // Core configuration management - model-agnostic settings
 export * from './config.js';
 // Database operations - supports different content types through metadata
-export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, getModelVersion, setModelVersion, getStoredModelInfo, setStoredModelInfo } from './db.js';
+export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, getModelVersion, setModelVersion, getStoredModelInfo, setStoredModelInfo, insertContentMetadata, getContentMetadata, getContentMetadataByHash, getContentMetadataByStorageType, deleteContentMetadata, getStorageStats, updateStorageStats } from './db.js';
 // Vector index operations - works with any embedding dimensions
 export { VectorIndex } from './vector-index.js';
 // Generic chunking interfaces and strategies - supports text, image metadata, etc.
@@ -61,6 +61,9 @@ export * from './search.js';
 export * from './ingestion.js';
 // Path management utilities - content-type agnostic
 export * from './path-manager.js';
+// Unified content system - handles both filesystem and memory content
+export { ContentManager } from './content-manager.js';
+export { ContentResolver } from './content-resolver.js';
 // Error handling framework - supports implementation-specific error contexts
 export * from './error-handler.js';
 //# sourceMappingURL=index.js.map