rag-lite-ts 2.1.1 → 2.2.0

package/dist/esm/multimodal/clip-embedder.d.ts

@@ -0,0 +1,327 @@
+/**
+ * MULTIMODAL IMPLEMENTATION — CLIP Embedder Implementation
+ *
+ * Implements UniversalEmbedder interface for CLIP models with full multimodal support.
+ * Provides reliable text and image embedding using CLIPTextModelWithProjection and
+ * CLIPVisionModelWithProjection for true cross-modal search capabilities.
+ *
+ * Features:
+ * - Text embedding using CLIP text encoder (512-dimensional vectors)
+ * - Image embedding using CLIP vision encoder (512-dimensional vectors)
+ * - Unified embedding space enabling cross-modal similarity search
+ * - Text queries can find semantically similar images
+ * - Image queries can find semantically similar text
+ * - Batch processing optimization for both text and images
+ *
+ * Supported Models:
+ * - Xenova/clip-vit-base-patch32 (recommended, faster)
+ * - Xenova/clip-vit-base-patch16 (higher accuracy, slower)
+ */
+import { BaseUniversalEmbedder, type EmbedderOptions } from '../core/abstract-embedder.js';
+import type { EmbeddingResult } from '../types.js';
+/**
+ * CLIP embedder implementation for multimodal content
+ *
+ * Provides reliable text and image embedding using separate CLIP model components:
+ * - CLIPTextModelWithProjection for text-only embedding (no pixel_values errors)
+ * - CLIPVisionModelWithProjection for image embedding
+ * - AutoTokenizer for proper text tokenization with CLIP's 77 token limit
+ *
+ * All embeddings are 512-dimensional vectors in a unified embedding space,
+ * enabling true cross-modal search where text queries can find images and
+ * image queries can find text based on semantic similarity.
+ *
+ * Example Usage:
+ * ```typescript
+ * const embedder = await createEmbedder('Xenova/clip-vit-base-patch32');
+ *
+ * // Embed text
+ * const textResult = await embedder.embedText('a red sports car');
+ *
+ * // Embed image
+ * const imageResult = await embedder.embedImage('./car.jpg');
+ *
+ * // Calculate cross-modal similarity
+ * const similarity = cosineSimilarity(textResult.vector, imageResult.vector);
+ * ```
+ */
+export declare class CLIPEmbedder extends BaseUniversalEmbedder {
+    private tokenizer;
+    private textModel;
+    private imageModel;
+    private resourceManager;
+    private embedderResourceId?;
+    private tokenizerResourceId?;
+    private textModelResourceId?;
+    private imageModelResourceId?;
+    constructor(modelName: string, options?: EmbedderOptions);
+    /**
+     * Load the CLIP model components
+     *
+     * Loads three separate components for reliable multimodal embedding:
+     * 1. AutoTokenizer - Handles text tokenization with CLIP's 77 token limit
+     * 2. CLIPTextModelWithProjection - Generates text embeddings without pixel_values errors
+     * 3. CLIPVisionModelWithProjection - Generates image embeddings
+     *
+     * All components are registered with the resource manager for proper cleanup.
+     * Models are cached locally after first download for faster subsequent loads.
+     *
+     * @throws {Error} If model loading fails or components are not available
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Clean up model resources with comprehensive disposal
+     *
+     * Properly disposes of all CLIP model components:
+     * - Tokenizer resources
+     * - Text model resources
+     * - Vision model resources
+     *
+     * Uses the resource manager for coordinated cleanup and forces garbage
+     * collection to free memory from CLIP models which can be memory intensive.
+     *
+     * This method is safe to call multiple times and will not throw errors
+     * during cleanup - errors are logged but don't prevent cleanup completion.
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Apply L2-normalization to an embedding vector
+     *
+     * L2-normalization ensures that all embeddings have unit length (magnitude = 1),
+     * which is essential for CLIP models as they were trained with normalized embeddings.
+     * This normalization makes cosine similarity calculations more reliable and ensures
+     * that vector magnitudes don't affect similarity scores.
+     *
+     * @param embedding - The embedding vector to normalize (modified in-place)
+     * @returns The normalized embedding vector (same reference as input)
+     * @private
+     */
+    private normalizeEmbedding;
+    /**
+     * Embed text using CLIP text encoder
+     *
+     * Uses CLIPTextModelWithProjection for reliable text-only embedding without
+     * pixel_values errors. Text is tokenized with CLIP's 77 token limit and
+     * automatically truncated if necessary.
+     *
+     * Returns a 512-dimensional L2-normalized embedding vector in the unified CLIP
+     * embedding space, which is directly comparable to image embeddings for cross-modal search.
+     *
+     * @param text - The text to embed (will be trimmed and validated)
+     * @returns EmbeddingResult with 512-dimensional normalized vector and metadata
+     * @throws {Error} If text is empty, model not loaded, or embedding fails
+     *
+     * @example
+     * ```typescript
+     * const result = await embedder.embedText('a red sports car');
+     * console.log(result.vector.length); // 512
+     * console.log(result.contentType);   // 'text'
+     * ```
+     */
+    embedText(text: string): Promise<EmbeddingResult>;
+    /**
+     * Embed image using CLIP vision encoder
+     *
+     * Uses CLIPVisionModelWithProjection to generate image embeddings in the same
+     * unified embedding space as text embeddings, enabling true cross-modal search.
+     *
+     * Supports both local file paths and URLs. Images are automatically preprocessed:
+     * - Resized to 224x224 pixels (CLIP's expected input size)
+     * - Converted to proper pixel_values format using AutoProcessor
+     * - Normalized for CLIP vision model
+     *
+     * Returns a 512-dimensional L2-normalized embedding vector directly comparable to text embeddings.
+     *
+     * @param imagePath - Local file path or URL to the image
+     * @returns EmbeddingResult with 512-dimensional normalized vector and metadata
+     * @throws {Error} If image not found, unsupported format, or embedding fails
+     *
+     * @example
+     * ```typescript
+     * // Local file
+     * const result = await embedder.embedImage('./car.jpg');
+     *
+     * // URL
+     * const result = await embedder.embedImage('https://example.com/car.jpg');
+     *
+     * console.log(result.vector.length); // 512
+     * console.log(result.contentType);   // 'image'
+     * ```
+     *
+     * Supported formats: PNG, JPEG, GIF, BMP, WebP
+     */
+    embedImage(imagePath: string): Promise<EmbeddingResult>;
+    /**
+     * Load and preprocess image for CLIP vision model
+     *
+     * Handles image loading from both local files and URLs with automatic format
+     * detection and preprocessing. Uses Sharp library when available for better
+     * Node.js support, falls back to RawImage for browser compatibility.
+     *
+     * Preprocessing steps:
+     * 1. Load image from path or URL
+     * 2. Resize to 224x224 pixels (CLIP's expected input size)
+     * 3. Convert to RGB format if needed
+     * 4. Return RawImage object for AutoProcessor
+     *
+     * @param imagePath - Local file path or URL to the image
+     * @returns RawImage object ready for AutoProcessor
+     * @throws {Error} If image loading or preprocessing fails
+     * @private
+     */
+    private loadAndPreprocessImage;
+    /**
+     * Optimized batch processing for CLIP models
+     *
+     * Processes mixed batches of text and image content efficiently using the
+     * BatchProcessingOptimizer for memory management and progress tracking.
+     *
+     * Features:
+     * - Automatic separation of text and image items
+     * - Memory-efficient processing for large batches
+     * - Progress reporting for batches > 20 items
+     * - Garbage collection between batches
+     * - Detailed statistics logging
+     *
+     * @param batch - Array of items with content, contentType, and optional metadata
+     * @returns Array of EmbeddingResult objects in the same order as input
+     * @throws {Error} If batch processing fails
+     * @protected
+     */
+    protected processBatch(batch: Array<{
+        content: string;
+        contentType: string;
+        metadata?: Record<string, any>;
+    }>): Promise<EmbeddingResult[]>;
+    /**
+     * Process batch of text items using CLIPTextModelWithProjection
+     *
+     * Efficiently processes multiple text items by tokenizing all texts first,
+     * then generating embeddings sequentially. This approach balances memory
+     * usage with processing speed.
+     *
+     * @param textItems - Array of text items to process
+     * @returns Array of EmbeddingResult objects
+     * @throws {Error} If batch processing fails or dimension mismatch occurs
+     * @private
+     */
+    private processBatchText;
+    /**
+     * Get comprehensive model information including CLIP-specific capabilities
+     *
+     * Extends base model info with CLIP-specific capabilities including multimodal
+     * support, zero-shot classification, and cross-modal retrieval features.
+     *
+     * @returns Object with model information and capabilities
+     */
+    getModelInfo(): {
+        capabilities: {
+            supportsMultimodal: boolean;
+            supportsZeroShotClassification: boolean;
+            supportsImageTextSimilarity: boolean;
+            supportsTextImageRetrieval: boolean;
+            recommendedUseCase: string;
+            imageEmbeddingStatus: string;
+            supportsText: boolean;
+            supportsImages: boolean;
+            supportsBatchProcessing: boolean;
+            supportsMetadata: boolean;
+            maxBatchSize?: number;
+            maxTextLength?: number;
+            supportedImageFormats?: readonly string[];
+            supportsCrossModalSearch?: boolean;
+            unifiedEmbeddingSpace?: boolean;
+            reliableImplementation?: boolean;
+        };
+        name: string;
+        type: import("../core/universal-embedder.js").ModelType;
+        dimensions: number;
+        version: string;
+        supportedContentTypes: readonly string[];
+        requirements: import("../types.js").ModelRequirements;
+    };
+    /**
+     * Check if the model is suitable for a specific task
+     *
+     * CLIP models excel at similarity, classification, retrieval, and multimodal
+     * tasks due to their unified embedding space and zero-shot capabilities.
+     *
+     * @param task - The task type to check
+     * @returns true if CLIP is suitable for the task, false otherwise
+     */
+    isSuitableForTask(task: 'similarity' | 'classification' | 'clustering' | 'retrieval' | 'multimodal'): boolean;
+    /**
+     * Get information about multimodal capabilities
+     *
+     * Returns detailed information about what content types are supported and
+     * what features are planned for future implementation.
+     *
+     * @returns Object describing multimodal support status
+     */
+    getMultimodalCapabilities(): {
+        textSupport: boolean;
+        imageSupport: boolean;
+        videoSupport: boolean;
+        audioSupport: boolean;
+        plannedFeatures: string[];
+    };
+    /**
+     * Get CLIP model variant information
+     *
+     * Extracts architecture details from the model name to provide variant-specific
+     * configuration parameters like patch size, image size, and text length limits.
+     *
+     * @returns Object with architecture details
+     */
+    getModelVariant(): {
+        architecture: string;
+        patchSize: number;
+        imageSize: number;
+        textMaxLength: number;
+    };
+    /**
+     * Check if text length is within CLIP's token limit
+     *
+     * Estimates token count based on character length (rough approximation of
+     * ~4 characters per token for English text). CLIP has a hard limit of 77 tokens.
+     *
+     * @param text - Text to validate
+     * @returns true if text is within token limit, false otherwise
+     */
+    isTextLengthValid(text: string): boolean;
+    /**
+     * Get performance characteristics for this CLIP variant
+     *
+     * Provides guidance on speed, accuracy, memory usage, and recommended batch
+     * sizes based on the CLIP model variant (patch32 vs patch16).
+     *
+     * @returns Object with performance characteristics
+     */
+    getPerformanceInfo(): {
+        speed: 'fast' | 'medium' | 'slow';
+        accuracy: 'good' | 'better' | 'best';
+        memoryUsage: 'low' | 'medium' | 'high';
+        recommendedBatchSize: number;
+    };
+    /**
+     * Check if all CLIP model components are loaded
+     *
+     * Verifies that tokenizer, text model, and vision model are all loaded and
+     * ready for use. All three components must be available for the embedder
+     * to be considered fully loaded.
+     *
+     * @returns true if all components are loaded, false otherwise
+     */
+    isLoaded(): boolean;
+    /**
+     * Validate that this is a supported CLIP model
+     *
+     * Checks the model name against the list of supported CLIP models. Currently
+     * supports Xenova/clip-vit-base-patch32 and Xenova/clip-vit-base-patch16.
+     *
+     * @throws {Error} If model is not in the supported list
+     * @private
+     */
+    private validateCLIPModel;
+}
+//# sourceMappingURL=clip-embedder.d.ts.map