rag-lite-ts 2.0.1 → 2.0.2

package/dist/core/batch-processing-optimizer.js

@@ -15,8 +15,8 @@ export const DEFAULT_BATCH_CONFIG = {
     textBatchSize: 16,
     imageBatchSize: 4, // Smaller for memory-intensive image processing
     maxConcurrentBatches: 2,
-    // Memory management (256MB threshold)
-    memoryThresholdMB: 256,
+    // Memory management (512MB threshold for multimodal processing)
+    memoryThresholdMB: 512,
     enableMemoryMonitoring: true,
     enableGarbageCollection: true,
     // Progress reporting every 5 batches
@@ -402,13 +402,8 @@ export class BatchProcessingOptimizer {
      */
     async preloadImageProcessingModels() {
         try {
-            if (!this.resourcePool.has('imageToText')) {
-                console.log('Preloading image-to-text processor...');
-                const processor = await LazyMultimodalLoader.loadImageToTextProcessor();
-                this.resourcePool.set('imageToText', processor);
-                // Register with resource manager
-                this.resourceManager.registerImageProcessor(processor, 'image-to-text');
-            }
+            // Note: Image-to-text processor is loaded on-demand by file-processor.ts
+            // to avoid conflicts with different pipeline configurations
             if (!this.resourcePool.has('metadataExtractor')) {
                 console.log('Preloading image metadata extractor...');
                 const extractor = await LazyMultimodalLoader.loadImageMetadataExtractor();
@@ -519,7 +514,7 @@ export function createImageBatchProcessor() {
     return new BatchProcessingOptimizer({
         imageBatchSize: 2, // Very small batches for memory efficiency
         textBatchSize: 8,
-        memoryThresholdMB: 128, // Lower threshold for images
+        memoryThresholdMB: 512, // Higher threshold for memory-intensive image processing
         enableMemoryMonitoring: true,
         enableGarbageCollection: true,
         enableParallelProcessing: false, // Sequential for better memory control
@@ -534,7 +529,7 @@ export function createTextBatchProcessor() {
         textBatchSize: 32, // Larger batches for text
         imageBatchSize: 4,
         enableParallelProcessing: true, // Parallel processing for text
-        memoryThresholdMB: 512, // Higher threshold for text
+        memoryThresholdMB: 256, // Lower threshold sufficient for text processing
         progressReportInterval: 10
     });
 }

package/dist/core/ingestion.js

@@ -290,7 +290,7 @@ export class IngestionPipeline {
                 chunkSize: config.chunk_size,
                 chunkOverlap: config.chunk_overlap
             };
-            const chunkingResult = await this.chunkDocumentsWithContentTypes(fileResult.documents, effectiveChunkConfig);
+            const chunkingResult = await this.chunkDocumentsWithContentTypes(fileResult.documents, effectiveChunkConfig, options.mode);
             if (chunkingResult.totalChunks === 0) {
                 console.log('No chunks created from documents');
                 return {
@@ -364,7 +364,7 @@ export class IngestionPipeline {
      * Chunk all documents and organize results with content-type awareness
      * Enhanced to handle different content types appropriately
      */
-    async chunkDocumentsWithContentTypes(documents, chunkConfig) {
+    async chunkDocumentsWithContentTypes(documents, chunkConfig, mode) {
         const documentChunks = [];
         const allChunks = [];
         let totalChunks = 0;
@@ -384,8 +384,18 @@ export class IngestionPipeline {
                             metadata: document.metadata
                         }];
                 }
+                else if (mode === 'multimodal') {
+                    // In multimodal mode, don't chunk text - CLIP handles truncation at 77 tokens
+                    // Chunking doesn't make sense because CLIP can't handle long text anyway
+                    chunks = [{
+                            text: document.content,
+                            chunkIndex: 0,
+                            contentType: 'text',
+                            metadata: document.metadata
+                        }];
+                }
                 else {
-                    // For text documents, use normal chunking
+                    // For text mode, use normal chunking
                     const textChunks = await chunkDocument(document, chunkConfig);
                     chunks = textChunks.map(chunk => ({
                         ...chunk,

package/dist/core/model-registry.js

@@ -69,7 +69,7 @@ export const SUPPORTED_MODELS = {
             supportsMetadata: true,
             supportsMultimodal: true, // True cross-modal search capabilities
             maxBatchSize: 8,
-            maxTextLength: 77, // CLIP's text sequence length limit
+            maxTextLength: 77, // CLIP's token limit (tokenizer handles truncation)
             supportedImageFormats: ['jpg', 'jpeg', 'png', 'webp', 'gif']
         },
         requirements: {
@@ -92,7 +92,7 @@ export const SUPPORTED_MODELS = {
             supportsMetadata: true,
             supportsMultimodal: true, // True cross-modal search capabilities
             maxBatchSize: 4,
-            maxTextLength: 77, // CLIP's text sequence length limit
+            maxTextLength: 77, // CLIP's token limit (tokenizer handles truncation)
             supportedImageFormats: ['jpg', 'jpeg', 'png', 'webp', 'gif']
         },
         requirements: {
@@ -194,9 +194,9 @@ export class ModelRegistry {
             suggestions.push('Use smaller batch sizes for optimal performance');
         }
         // Text length limitations
-        if (modelInfo.capabilities.maxTextLength && modelInfo.capabilities.maxTextLength < 512) {
+        if (modelInfo.capabilities.maxTextLength && modelInfo.capabilities.maxTextLength < 256) {
             warnings.push(`Model has limited text length: ${modelInfo.capabilities.maxTextLength} characters`);
-            suggestions.push('Consider chunking long texts before processing');
+            suggestions.push('Long texts will be truncated by the tokenizer');
         }
         // Image format support
         if (modelInfo.capabilities.supportsImages && modelInfo.capabilities.supportedImageFormats) {

package/dist/core/reranking-strategies.d.ts

@@ -97,20 +97,10 @@ export declare class TextDerivedRerankingStrategy implements RerankingStrategy {
     readonly supportedContentTypes: string[];
     isEnabled: boolean;
     private crossEncoderReranker;
-    private imageToTextModel;
-    private imageToTextModelName;
-    private initialized;
     constructor(imageToTextModelName?: string, crossEncoderModelName?: string);
-    /**
-     * Initialize the image-to-text model if not already done
-     */
-    private ensureInitialized;
-    /**
-     * Ensure DOM polyfills are set up for transformers.js
-     */
-    private ensurePolyfills;
     /**
      * Generate text description for an image
+     * Uses the shared image-to-text functionality from file-processor
      */
     private generateImageDescription;
     /**
@@ -128,11 +118,6 @@ export declare class TextDerivedRerankingStrategy implements RerankingStrategy {
         description: string;
         requiredModels: string[];
         configOptions: {
-            imageToTextModel: {
-                type: string;
-                description: string;
-                default: string;
-            };
             crossEncoderModel: {
                 type: string;
                 description: string;

package/dist/core/reranking-strategies.js

@@ -174,69 +174,22 @@ export class TextDerivedRerankingStrategy {
     supportedContentTypes = ['text', 'image'];
     isEnabled = true;
     crossEncoderReranker;
-    imageToTextModel = null;
-    imageToTextModelName = 'Xenova/vit-gpt2-image-captioning';
-    initialized = false;
     constructor(imageToTextModelName, crossEncoderModelName) {
-        if (imageToTextModelName) {
-            this.imageToTextModelName = imageToTextModelName;
-        }
+        // Note: imageToTextModelName parameter is kept for backward compatibility
+        // but is no longer used since we delegate to file-processor's implementation
         // Create the underlying cross-encoder strategy
         this.crossEncoderReranker = new CrossEncoderRerankingStrategy(crossEncoderModelName);
     }
-    /**
-     * Initialize the image-to-text model if not already done
-     */
-    async ensureInitialized() {
-        if (!this.initialized) {
-            try {
-                console.log(`Loading image-to-text model: ${this.imageToTextModelName}`);
-                // Set up polyfills for transformers.js
-                this.ensurePolyfills();
-                const { pipeline } = await import('@huggingface/transformers');
-                this.imageToTextModel = await pipeline('image-to-text', this.imageToTextModelName);
-                this.initialized = true;
-                console.log(`Image-to-text model loaded successfully: ${this.imageToTextModelName}`);
-            }
-            catch (error) {
-                console.warn(`Image-to-text model initialization failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
-                this.isEnabled = false;
-            }
-        }
-    }
-    /**
-     * Ensure DOM polyfills are set up for transformers.js
-     */
-    ensurePolyfills() {
-        if (typeof window === 'undefined' && typeof globalThis !== 'undefined') {
-            if (typeof globalThis.self === 'undefined') {
-                globalThis.self = globalThis;
-            }
-            if (typeof global.self === 'undefined') {
-                global.self = global;
-            }
-        }
-    }
     /**
      * Generate text description for an image
+     * Uses the shared image-to-text functionality from file-processor
      */
     async generateImageDescription(imagePath) {
-        await this.ensureInitialized();
-        if (!this.imageToTextModel) {
-            throw new Error('Image-to-text model not loaded');
-        }
         try {
-            const result = await this.imageToTextModel(imagePath);
-            // Handle different response formats from the pipeline
-            if (Array.isArray(result) && result.length > 0) {
-                return result[0].generated_text || result[0].text || String(result[0]);
-            }
-            else if (result && typeof result === 'object') {
-                return result.generated_text || result.text || String(result);
-            }
-            else {
-                return String(result);
-            }
+            // Use the file-processor's image description function which has proven to work reliably
+            const { generateImageDescriptionForFile } = await import('../file-processor.js');
+            const result = await generateImageDescriptionForFile(imagePath);
+            return result.description;
         }
         catch (error) {
             console.warn(`Failed to generate description for image ${imagePath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
@@ -249,22 +202,11 @@ export class TextDerivedRerankingStrategy {
      * Rerank search results using text-derived approach
      */
     rerank = async (query, results, contentType) => {
-        // If strategy is disabled, return results unchanged
-        if (!this.isEnabled) {
-            return results;
-        }
         // Validate content type
         if (contentType && !this.supportedContentTypes.includes(contentType)) {
             throw new Error(`Text-derived strategy does not support content type '${contentType}'. ` +
                 `Supported types: ${this.supportedContentTypes.join(', ')}`);
         }
-        // Ensure models are initialized
-        await this.ensureInitialized();
-        // If initialization failed, return results unchanged
-        if (!this.isEnabled) {
-            console.warn('Text-derived reranker not enabled, returning results unchanged');
-            return results;
-        }
         try {
             // Step 1: Convert images to text descriptions
             const processedResults = await Promise.all(results.map(async (result) => {
@@ -314,12 +256,8 @@ export class TextDerivedRerankingStrategy {
      * Configure the reranking strategy
      */
     configure(config) {
-        if (config.imageToTextModel && typeof config.imageToTextModel === 'string') {
-            this.imageToTextModelName = config.imageToTextModel;
-            // Reset initialization to use new model
-            this.initialized = false;
-            this.imageToTextModel = null;
-        }
+        // Note: imageToTextModel configuration is no longer used
+        // since we delegate to file-processor's implementation
         if (config.crossEncoderModel && typeof config.crossEncoderModel === 'string') {
             this.crossEncoderReranker.configure({ modelName: config.crossEncoderModel });
         }
@@ -334,15 +272,10 @@ export class TextDerivedRerankingStrategy {
         return {
             description: 'Text-derived reranking that converts images to text descriptions then applies cross-encoder reranking',
             requiredModels: [
-                'Xenova/vit-gpt2-image-captioning', // Image-to-text model
+                'Xenova/vit-gpt2-image-captioning', // Image-to-text model (via file-processor)
                 'Xenova/ms-marco-MiniLM-L-6-v2' // Cross-encoder model
             ],
             configOptions: {
-                imageToTextModel: {
-                    type: 'string',
-                    description: 'Image-to-text model name for generating descriptions',
-                    default: 'Xenova/vit-gpt2-image-captioning'
-                },
                 crossEncoderModel: {
                     type: 'string',
                     description: 'Cross-encoder model name for text reranking',
@@ -360,16 +293,15 @@ export class TextDerivedRerankingStrategy {
      * Check if the strategy is ready to use
      */
     async isReady() {
-        await this.ensureInitialized();
         const crossEncoderReady = await this.crossEncoderReranker.isReady();
-        return this.isEnabled && this.imageToTextModel !== null && crossEncoderReady;
+        return this.isEnabled && crossEncoderReady;
     }
     /**
      * Get the current model names being used
      */
     getModelNames() {
         return {
-            imageToText: this.imageToTextModelName,
+            imageToText: 'Xenova/vit-gpt2-image-captioning', // Fixed model via file-processor
             crossEncoder: this.crossEncoderReranker.getModelName()
         };
     }
@@ -377,8 +309,6 @@ export class TextDerivedRerankingStrategy {
      * Clean up resources
      */
     async cleanup() {
-        this.initialized = false;
-        this.imageToTextModel = null;
         await this.crossEncoderReranker.cleanup();
     }
 }

package/dist/dom-polyfills.js

@@ -30,11 +30,8 @@ if (typeof window === 'undefined') {
     if (typeof globalThis.navigator === 'undefined') {
         globalThis.navigator = dom.window.navigator;
     }
-    // Polyfill createImageBitmap if needed (for image processing)
-    if (typeof globalThis.createImageBitmap === 'undefined') {
-        globalThis.createImageBitmap = dom.window.createImageBitmap || (() => {
-            throw new Error('createImageBitmap not available in Node.js environment');
-        });
-    }
+    // Note: Do NOT polyfill createImageBitmap with a fake implementation
+    // RawImage.fromURL() will handle image loading correctly without it
+    // Setting a fake createImageBitmap that throws errors breaks image loading
 }
 //# sourceMappingURL=dom-polyfills.js.map

package/dist/file-processor.js

@@ -346,24 +346,35 @@ function extractTitle(content, filePath) {
  * Cache for image-to-text pipeline to avoid reloading
  */
 let imageToTextPipeline = null;
+let imageToTextPipelinePromise = null;
 /**
- * Initialize the image-to-text pipeline
+ * Initialize the image-to-text pipeline with proper async locking
  */
 async function initializeImageToTextPipeline(modelName = 'Xenova/vit-gpt2-image-captioning') {
+    // Return cached pipeline if available
     if (imageToTextPipeline) {
         return imageToTextPipeline;
     }
-    try {
-        const { pipeline } = await import('@huggingface/transformers');
-        console.log(`Loading image-to-text model: ${modelName}`);
-        imageToTextPipeline = await pipeline('image-to-text', modelName);
-        console.log(`Successfully loaded image-to-text model: ${modelName}`);
-        return imageToTextPipeline;
-    }
-    catch (error) {
-        console.error(`Failed to load image-to-text model ${modelName}:`, error);
-        throw new Error(`Failed to initialize image-to-text pipeline: ${error instanceof Error ? error.message : String(error)}`);
+    // If pipeline is currently loading, wait for it
+    if (imageToTextPipelinePromise) {
+        return imageToTextPipelinePromise;
     }
+    // Start loading pipeline
+    imageToTextPipelinePromise = (async () => {
+        try {
+            const { pipeline } = await import('@huggingface/transformers');
+            console.log(`Loading image-to-text model: ${modelName}`);
+            imageToTextPipeline = await pipeline('image-to-text', modelName);
+            console.log(`Successfully loaded image-to-text model: ${modelName}`);
+            return imageToTextPipeline;
+        }
+        catch (error) {
+            console.error(`Failed to load image-to-text model ${modelName}:`, error);
+            imageToTextPipelinePromise = null; // Reset on error so it can be retried
+            throw new Error(`Failed to initialize image-to-text pipeline: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    })();
+    return imageToTextPipelinePromise;
 }
 /**
  * Parse PNG image dimensions from file buffer
@@ -545,8 +556,11 @@ async function extractImageMetadata(imagePath) {
 async function generateImageDescription(imagePath, options = DEFAULT_IMAGE_TO_TEXT_OPTIONS) {
     try {
         const pipeline = await initializeImageToTextPipeline(options.model);
-        // Generate description
-        const result = await pipeline(imagePath, {
+        // Load image using RawImage.fromURL which works with local file paths
+        const { RawImage } = await import('@huggingface/transformers');
+        const image = await RawImage.fromURL(imagePath);
+        // Generate description with loaded image
+        const result = await pipeline(image, {
             max_length: options.maxLength || 50,
             num_beams: 4,
             early_stopping: true
@@ -597,93 +611,6 @@ async function generateImageDescriptionsBatch(imagePaths, options = DEFAULT_IMAG
     }
     return results;
 }
-/**
- * Generate text descriptions for multiple images using optimized batch processing
- * Uses BatchProcessingOptimizer for memory-efficient processing of large image collections
- */
-async function generateImageDescriptionsBatchOptimized(imagePaths, options = DEFAULT_IMAGE_TO_TEXT_OPTIONS) {
-    // For small batches, use the existing implementation
-    if (imagePaths.length <= 10) {
-        return generateImageDescriptionsBatch(imagePaths, options);
-    }
-    try {
-        // Import batch processing optimizer
-        const { createImageBatchProcessor } = await import('./core/batch-processing-optimizer.js');
-        const batchProcessor = createImageBatchProcessor();
-        // Convert image paths to batch items
-        const batchItems = imagePaths.map(path => ({
-            content: path,
-            contentType: 'image',
-            metadata: { originalPath: path }
-        }));
-        // Create image description function
-        const imageDescriptionFunction = async (item) => {
-            try {
-                const result = await generateImageDescription(item.content, options);
-                return {
-                    embedding_id: `img_desc_${Date.now()}_${Math.random()}`,
-                    vector: new Float32Array([0]), // Placeholder vector
-                    contentType: 'image',
-                    metadata: {
-                        path: item.content,
-                        description: result.description,
-                        confidence: result.confidence,
-                        model: result.model
-                    }
-                };
-            }
-            catch (error) {
-                throw new Error(`Failed to generate description for ${item.content}: ${error instanceof Error ? error.message : String(error)}`);
-            }
-        };
-        // Process with optimization and progress reporting
-        const batchResult = await batchProcessor.processBatch(batchItems, imageDescriptionFunction, (stats) => {
-            console.log(`Image description progress: ${stats.processedItems}/${stats.totalItems} (${Math.round((stats.processedItems / stats.totalItems) * 100)}%)`);
-            console.log(`  Memory usage: ${stats.memoryUsageMB}MB (peak: ${stats.peakMemoryUsageMB}MB)`);
-            if (stats.failedItems > 0) {
-                console.log(`  Failed items: ${stats.failedItems}`);
-            }
-        });
-        // Log final statistics
-        console.log(`✓ Image description generation complete:`);
-        console.log(`  Processed: ${batchResult.stats.processedItems}/${batchResult.stats.totalItems}`);
-        console.log(`  Failed: ${batchResult.stats.failedItems}`);
-        console.log(`  Processing time: ${Math.round(batchResult.stats.processingTimeMs / 1000)}s`);
-        console.log(`  Rate: ${Math.round(batchResult.stats.itemsPerSecond)} images/sec`);
-        console.log(`  Peak memory usage: ${batchResult.stats.peakMemoryUsageMB}MB`);
-        if (batchResult.stats.retryCount > 0) {
-            console.log(`  Retries: ${batchResult.stats.retryCount}`);
-        }
-        // Convert results back to expected format
-        const results = [];
-        // Add successful results
-        for (const result of batchResult.results) {
-            if (result.metadata?.description) {
-                results.push({
-                    path: result.metadata.path,
-                    result: {
-                        description: result.metadata.description,
-                        confidence: result.metadata.confidence,
-                        model: result.metadata.model
-                    }
-                });
-            }
-        }
-        // Add failed results
-        for (const error of batchResult.errors) {
-            results.push({
-                path: error.item.content,
-                error: error.error
-            });
-        }
-        return results;
-    }
-    catch (error) {
-        console.warn(`Optimized batch processing failed, falling back to standard batch processing: ${error instanceof Error ? error.message : String(error)}`);
-        // Fall back to existing implementation
-        return generateImageDescriptionsBatch(imagePaths, options);
-    }
-}
 /**
  * Process image file to extract text description and metadata
  */
@@ -834,8 +761,8 @@ export async function processFiles(filePaths, pathManager, imageToTextOptions) {
     if (imageFiles.length > 0) {
         console.log(`Processing ${imageFiles.length} image files with optimized batch processing`);
         try {
-            // Use optimized batch processing for image descriptions
-            const batchResults = await generateImageDescriptionsBatchOptimized(imageFiles, imageToTextOptions);
+            // Use batch processing for image descriptions
+            const batchResults = await generateImageDescriptionsBatch(imageFiles, imageToTextOptions);
             // Convert batch results to documents with metadata extraction
             for (const batchResult of batchResults) {
                 try {
@@ -961,6 +888,7 @@ export async function cleanupImageProcessingResources() {
                 await imageToTextPipeline.dispose();
             }
             imageToTextPipeline = null;
+            imageToTextPipelinePromise = null;
             console.log('Image-to-text pipeline cleaned up');
         }
         catch (error) {

package/dist/ingestion.js

@@ -64,7 +64,12 @@ export class IngestionPipeline {
         if (!this.corePipeline) {
             throw new Error('IngestionPipeline failed to initialize');
         }
-        return this.corePipeline.ingestFile(filePath, options);
+        // Merge mode from constructor options with runtime options
+        const mergedOptions = {
+            ...options,
+            mode: options?.mode || this.options.mode
+        };
+        return this.corePipeline.ingestFile(filePath, mergedOptions);
     }
     /**
      * Ingest all documents in a directory
@@ -74,7 +79,12 @@ export class IngestionPipeline {
         if (!this.corePipeline) {
             throw new Error('IngestionPipeline failed to initialize');
         }
-        return this.corePipeline.ingestDirectory(directoryPath, options);
+        // Merge mode from constructor options with runtime options
+        const mergedOptions = {
+            ...options,
+            mode: options?.mode || this.options.mode
+        };
+        return this.corePipeline.ingestDirectory(directoryPath, mergedOptions);
     }
     /**
      * Ingest content from memory buffer
@@ -95,7 +105,12 @@ export class IngestionPipeline {
         if (!this.corePipeline) {
             throw new Error('IngestionPipeline failed to initialize');
         }
-        return this.corePipeline.ingestFromMemory(content, metadata, options);
+        // Merge mode from constructor options with runtime options
+        const mergedOptions = {
+            ...options,
+            mode: options?.mode || this.options.mode
+        };
+        return this.corePipeline.ingestFromMemory(content, metadata, mergedOptions);
     }
     /**
      * Clean up resources

package/dist/multimodal/clip-embedder.js

@@ -339,15 +339,13 @@ export class CLIPEmbedder extends BaseUniversalEmbedder {
             throw new Error('CLIP text model or tokenizer not initialized');
         }
         try {
-            // Validate and truncate text if necessary (CLIP has a 77 token limit)
-            this.validateTextLength(text);
-            const finalProcessedText = this.truncateText(processedText);
             // Use the validated CLIPTextModelWithProjection approach (no pixel_values errors)
             // Tokenize text with CLIP's requirements
-            const tokens = await this.tokenizer(finalProcessedText, {
+            // The tokenizer handles truncation at 77 TOKENS (not characters)
+            const tokens = await this.tokenizer(processedText, {
                 padding: true,
                 truncation: true,
-                max_length: 77, // CLIP's text sequence length limit
+                max_length: 77, // CLIP's text sequence length limit (77 tokens)
                 return_tensors: 'pt'
             });
             // Log token information for debugging (only in development)
@@ -355,7 +353,7 @@ export class CLIPEmbedder extends BaseUniversalEmbedder {
                 const tokenIds = tokens.input_ids?.data || [];
                 const actualTokenCount = Array.from(tokenIds).filter((id) => id !== 0).length;
                 if (actualTokenCount >= 77) {
-                    console.warn(`Text truncated: "${finalProcessedText.substring(0, 50)}..." (${actualTokenCount}+ tokens -> 77 tokens)`);
+                    console.warn(`Text truncated by tokenizer: "${processedText.substring(0, 50)}..." (truncated to 77 tokens)`);
                 }
             }
             // Generate text embedding using CLIPTextModelWithProjection
@@ -389,15 +387,15 @@ export class CLIPEmbedder extends BaseUniversalEmbedder {
                 console.warn(`Warning: Embedding normalization may be imprecise (magnitude: ${magnitudeAfterNorm.toFixed(6)})`);
             }
             // Generate unique embedding ID
-            const embeddingId = this.generateEmbeddingId(finalProcessedText, 'text');
+            const embeddingId = this.generateEmbeddingId(processedText, 'text');
             return {
                 embedding_id: embeddingId,
                 vector: embedding,
                 contentType: 'text',
                 metadata: {
                     originalText: text,
-                    processedText: finalProcessedText,
-                    textLength: finalProcessedText.length,
+                    processedText: processedText,
+                    textLength: processedText.length,
                     embeddingMagnitudeBeforeNorm: magnitudeBeforeNorm,
                     embeddingMagnitudeAfterNorm: magnitudeAfterNorm,
                     normalized: true,
@@ -682,8 +680,9 @@ export class CLIPEmbedder extends BaseUniversalEmbedder {
                     const { createTextBatchProcessor } = await import('../core/batch-processing-optimizer.js');
                     const batchProcessor = createTextBatchProcessor();
                     // Convert to EmbeddingBatchItem format
+                    // Let tokenizer handle truncation at 77 tokens (not characters)
                     const batchItems = textItems.map(item => ({
-                        content: this.truncateText(item.content.trim()),
+                        content: item.content.trim(),
                         contentType: item.contentType,
                         metadata: item.metadata
                     }));
@@ -773,7 +772,8 @@ export class CLIPEmbedder extends BaseUniversalEmbedder {
      */
     async processBatchText(textItems) {
         // Prepare texts for batch processing
-        const texts = textItems.map(item => this.truncateText(item.content.trim()));
+        // Let tokenizer handle truncation at 77 tokens (not characters)
+        const texts = textItems.map(item => item.content.trim());
         // Tokenize all texts in batch
         const tokensBatch = await Promise.all(texts.map(text => this.tokenizer(text, {
             padding: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rag-lite-ts",
-  "version": "2.0.1",
+  "version": "2.0.2",
   "description": "Local-first TypeScript retrieval engine with Chameleon Multimodal Architecture for semantic search over text and image content",
   "type": "module",
   "main": "./dist/index.js",