rag-lite-ts 1.0.2 → 2.0.0

package/dist/multimodal/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Multimodal implementation layer exports
+ * Provides CLIP-based embedders for cross-modal search capabilities
+ */
+export { CLIPEmbedder } from './clip-embedder.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/multimodal/index.js

@@ -0,0 +1,6 @@
+/**
+ * Multimodal implementation layer exports
+ * Provides CLIP-based embedders for cross-modal search capabilities
+ */
+export { CLIPEmbedder } from './clip-embedder.js';
+//# sourceMappingURL=index.js.map

package/dist/run-error-recovery-tests.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Test Runner for Chameleon Error Recovery and Reliability Tests
+ * Runs the comprehensive error recovery test suite
+ */
+declare function runTests(): Promise<boolean>;
+export { runTests };
+//# sourceMappingURL=run-error-recovery-tests.d.ts.map

package/dist/run-error-recovery-tests.js

@@ -0,0 +1,101 @@
+/**
+ * Test Runner for Chameleon Error Recovery and Reliability Tests
+ * Runs the comprehensive error recovery test suite
+ */
+import { spawn } from 'child_process';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+async function runTests() {
+    console.log('🧪 Running Chameleon Error Recovery and Reliability Tests...\n');
+    const testFiles = [
+        'chameleon-error-recovery.test.ts',
+        'chameleon-reliability-integration.test.ts',
+        'chameleon-stress-testing.test.ts',
+        'chameleon-error-simulation.test.ts'
+    ];
+    let totalTests = 0;
+    let passedTests = 0;
+    let failedTests = 0;
+    for (const testFile of testFiles) {
+        console.log(`\n📋 Running ${testFile}...`);
+        try {
+            // Build the test file first
+            const buildProcess = spawn('npx', ['tsc', '--project', 'tsconfig.test.json'], {
+                stdio: 'pipe',
+                shell: true
+            });
+            await new Promise((resolve, reject) => {
+                buildProcess.on('close', (code) => {
+                    if (code === 0) {
+                        resolve(code);
+                    }
+                    else {
+                        reject(new Error(`Build failed with code ${code}`));
+                    }
+                });
+            });
+            // Run the compiled test
+            const testProcess = spawn('node', ['--test', `dist/${testFile.replace('.ts', '.js')}`], {
+                stdio: 'pipe',
+                shell: true
+            });
+            let output = '';
+            let errorOutput = '';
+            testProcess.stdout?.on('data', (data) => {
+                output += data.toString();
+            });
+            testProcess.stderr?.on('data', (data) => {
+                errorOutput += data.toString();
+            });
+            await new Promise((resolve) => {
+                testProcess.on('close', (code) => {
+                    console.log(`Exit code: ${code}`);
+                    if (output) {
+                        console.log('Output:', output);
+                    }
+                    if (errorOutput) {
+                        console.log('Errors:', errorOutput);
+                    }
+                    // Count tests (this is a simple approximation)
+                    const testMatches = output.match(/✓|×/g);
+                    const currentTests = testMatches ? testMatches.length : 0;
+                    totalTests += currentTests;
+                    if (code === 0) {
+                        passedTests += currentTests;
+                        console.log(`✅ ${testFile} completed successfully`);
+                    }
+                    else {
+                        failedTests += currentTests;
+                        console.log(`❌ ${testFile} failed`);
+                    }
+                    resolve(code);
+                });
+            });
+        }
+        catch (error) {
+            console.error(`❌ Failed to run ${testFile}:`, error instanceof Error ? error.message : String(error));
+            failedTests++;
+        }
+    }
+    console.log('\n📊 Test Summary:');
+    console.log(`Total Tests: ${totalTests}`);
+    console.log(`Passed: ${passedTests}`);
+    console.log(`Failed: ${failedTests}`);
+    if (failedTests === 0) {
+        console.log('\n🎉 All error recovery tests completed!');
+        console.log('✅ System demonstrates robust error handling and recovery mechanisms');
+    }
+    else {
+        console.log('\n⚠️  Some tests failed - this may be expected in test environments');
+        console.log('🔍 Review the output above for specific failure details');
+    }
+    return failedTests === 0;
+}
+// Run tests if this file is executed directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+    runTests().catch(console.error);
+}
+export { runTests };
+//# sourceMappingURL=run-error-recovery-tests.js.map

package/dist/search.d.ts

@@ -40,6 +40,32 @@ export declare class SearchEngine {
      * Perform semantic search
      */
     search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Retrieve content by ID in the specified format
+     * @param contentId - Content ID to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to content in requested format
+     */
+    getContent(contentId: string, format?: 'file' | 'base64'): Promise<string>;
+    /**
+     * Retrieve multiple content items efficiently in batch
+     * @param contentIds - Array of content IDs to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to array of content in requested format
+     */
+    getContentBatch(contentIds: string[], format?: 'file' | 'base64'): Promise<string[]>;
+    /**
+     * Retrieve content metadata for result enhancement
+     * @param contentId - Content ID to get metadata for
+     * @returns Promise that resolves to content metadata
+     */
+    getContentMetadata(contentId: string): Promise<import('./core/content-resolver.js').ContentMetadata>;
+    /**
+     * Verify that content exists and is accessible
+     * @param contentId - Content ID to verify
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    verifyContentExists(contentId: string): Promise<boolean>;
     /**
      * Clean up resources
      */

package/dist/search.js

@@ -74,8 +74,11 @@ export class SearchEngine {
                 const db = await openDatabase(this.dbPath);
                 const indexManager = new IndexManager(this.indexPath, this.dbPath, modelDefaults.dimensions, this.options.embeddingModel);
                 await indexManager.initialize();
+                // Create ContentResolver for unified content system
+                const { ContentResolver } = await import('./core/content-resolver.js');
+                const contentResolver = new ContentResolver(db);
                 // Create core engine with dependency injection
-                this.coreEngine = new CoreSearchEngine(embedFn, indexManager, db, this.options.rerankFn);
+                this.coreEngine = new CoreSearchEngine(embedFn, indexManager, db, this.options.rerankFn, contentResolver);
             }
             else {
                 // Use factory for standard initialization
@@ -94,6 +97,56 @@ export class SearchEngine {
         }
         return this.coreEngine.search(query, options);
     }
+    /**
+     * Retrieve content by ID in the specified format
+     * @param contentId - Content ID to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to content in requested format
+     */
+    async getContent(contentId, format = 'file') {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.getContent(contentId, format);
+    }
+    /**
+     * Retrieve multiple content items efficiently in batch
+     * @param contentIds - Array of content IDs to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to array of content in requested format
+     */
+    async getContentBatch(contentIds, format = 'file') {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.getContentBatch(contentIds, format);
+    }
+    /**
+     * Retrieve content metadata for result enhancement
+     * @param contentId - Content ID to get metadata for
+     * @returns Promise that resolves to content metadata
+     */
+    async getContentMetadata(contentId) {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.getContentMetadata(contentId);
+    }
+    /**
+     * Verify that content exists and is accessible
+     * @param contentId - Content ID to verify
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    async verifyContentExists(contentId) {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.verifyContentExists(contentId);
+    }
     /**
      * Clean up resources
      */

package/dist/test-utils.d.ts

@@ -2,35 +2,17 @@
  * Test utilities for multi-model support
  * Provides common configurations and helpers for testing with different embedding models
  */
-export declare const TEST_MODELS: readonly [{
-    readonly name: "sentence-transformers/all-MiniLM-L6-v2";
-    readonly dimensions: 384;
-    readonly chunkSize: 250;
-    readonly batchSize: 16;
-}, {
-    readonly name: "Xenova/all-mpnet-base-v2";
-    readonly dimensions: 768;
-    readonly chunkSize: 400;
-    readonly batchSize: 8;
-}];
+export interface TestModel {
+    name: string;
+    dimensions: number;
+    chunkSize: number;
+    batchSize: number;
+}
+export declare const TEST_MODELS: TestModel[];
 /**
  * Retrieve model configuration by name
  * @param modelName - The name of the model to retrieve
  * @returns Model configuration object or undefined if not found
  */
-export declare function getTestModel(modelName: string): {
-    readonly name: "sentence-transformers/all-MiniLM-L6-v2";
-    readonly dimensions: 384;
-    readonly chunkSize: 250;
-    readonly batchSize: 16;
-} | {
-    readonly name: "Xenova/all-mpnet-base-v2";
-    readonly dimensions: 768;
-    readonly chunkSize: 400;
-    readonly batchSize: 8;
-} | undefined;
-/**
- * Type for test model configuration
- */
-export type TestModel = typeof TEST_MODELS[number];
+export declare function getTestModel(modelName: string): TestModel | undefined;
 //# sourceMappingURL=test-utils.d.ts.map

package/dist/text/chunker.d.ts

@@ -11,6 +11,7 @@ export interface Document {
     source: string;
     title: string;
     content: string;
+    metadata?: Record<string, any>;
 }
 export interface Chunk {
     text: string;

package/dist/text/embedder.js

@@ -2,6 +2,7 @@ import '../dom-polyfills.js';
 import { createHash } from 'crypto';
 import { config } from '../core/config.js';
 import { handleError, ErrorCategory, ErrorSeverity, safeExecute } from '../core/error-handler.js';
+import { createModelLoadingError, createInvalidContentError, createMissingDependencyError } from '../core/actionable-error-messages.js';
 /**
  * List of supported embedding models
  */
@@ -22,8 +23,7 @@ export class EmbeddingEngine {
         this.batchSize = batchSize || config.batch_size;
         // Validate that the model is supported
         if (!SUPPORTED_MODELS.includes(this.modelName)) {
-            throw new Error(`Unsupported model: ${this.modelName}\n` +
-                `Supported models: ${SUPPORTED_MODELS.join(', ')}`);
+            throw createModelLoadingError(this.modelName, `Model not in supported list. Supported models: ${SUPPORTED_MODELS.join(', ')}`, { operationContext: 'EmbeddingEngine constructor' });
         }
         console.log(`🤖 EmbeddingEngine initialized with model: ${this.modelName}, batchSize: ${this.batchSize}`);
     }
@@ -88,7 +88,10 @@ export class EmbeddingEngine {
      */
     async embedBatch(texts) {
         if (!this.model) {
-            throw new Error('Model not loaded. Call loadModel() first.');
+            throw createMissingDependencyError('model', 'object', {
+                operationContext: 'embedBatch',
+                includeTroubleshooting: true
+            });
         }
         if (texts.length === 0) {
             return [];
@@ -123,7 +126,8 @@ export class EmbeddingEngine {
                 const vector = new Float32Array(embeddingData[i]);
                 results.push({
                     embedding_id,
-                    vector
+                    vector,
+                    contentType: 'text'
                 });
             }
             return results;
@@ -173,7 +177,8 @@ export class EmbeddingEngine {
             const vector = new Float32Array(embeddingData[0]);
             return {
                 embedding_id,
-                vector
+                vector,
+                contentType: 'text'
             };
         }
         catch (error) {
@@ -189,7 +194,9 @@ export class EmbeddingEngine {
     async embedSingle(text) {
         const results = await this.embedBatch([text]);
         if (results.length === 0) {
-            throw new Error('Failed to generate embedding for single text');
+            throw createInvalidContentError('text', 'empty', {
+                operationContext: 'embedText'
+            });
         }
         return results[0];
     }
@@ -357,10 +364,10 @@ export function createTextEmbedFunction(modelName, batchSize) {
         }
         // Use the existing embedSingle method
         const result = await engine.embedSingle(query);
-        // Add contentType to the result
+        // Ensure contentType is present (should already be included from embedSingle)
         return {
             ...result,
-            contentType: 'text'
+            contentType: result.contentType || 'text'
         };
     };
     return embedFunction;

package/dist/text/index.d.ts

@@ -3,5 +3,6 @@ export { CrossEncoderReranker, createTextRerankFunction, createTextReranker } fr
 export { countTokens, getTokenizer, resetTokenizer } from './tokenizer.js';
 export { chunkDocument, type Chunk, type Document } from '../core/chunker.js';
 export { type ChunkConfig } from '../core/chunker.js';
+export { SentenceTransformerEmbedder } from './sentence-transformer-embedder.js';
 export * from './preprocessors/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/text/index.js

@@ -3,6 +3,7 @@ export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createT
 export { CrossEncoderReranker, createTextRerankFunction, createTextReranker } from './reranker.js';
 export { countTokens, getTokenizer, resetTokenizer } from './tokenizer.js';
 export { chunkDocument } from '../core/chunker.js';
+export { SentenceTransformerEmbedder } from './sentence-transformer-embedder.js';
 // Re-export preprocessors
 export * from './preprocessors/index.js';
 //# sourceMappingURL=index.js.map

package/dist/text/reranker.d.ts

@@ -8,13 +8,12 @@ export declare class CrossEncoderReranker {
     private model;
     private tokenizer;
     private modelName;
-    private static readonly FALLBACK_MODELS;
     /**
      * Ensure DOM polyfills are set up for transformers.js
      */
     private ensurePolyfills;
     /**
-     * Load the embedding model with graceful fallback
+     * Load the embedding model
      */
     loadModel(): Promise<void>;
     /**

package/dist/text/reranker.js

@@ -18,12 +18,6 @@ export class CrossEncoderReranker {
     model = null; // Use any to avoid complex transformers.js typing issues
     tokenizer = null;
     modelName = 'Xenova/ms-marco-MiniLM-L-6-v2'; // Use working cross-encoder model
-    // Alternative models in case the primary fails
-    static FALLBACK_MODELS = [
-        'Xenova/ms-marco-MiniLM-L-6-v2', // Primary - proven to work in standalone test
-        'cross-encoder/ms-marco-MiniLM-L-6-v2', // Original (may have issues)
-        'cross-encoder/ms-marco-MiniLM-L-2-v2', // Smaller original (may have issues)
-    ];
     /**
      * Ensure DOM polyfills are set up for transformers.js
      */
@@ -40,54 +34,30 @@ export class CrossEncoderReranker {
         }
     }
     /**
-     * Load the embedding model with graceful fallback
+     * Load the embedding model
      */
     async loadModel() {
-        // Try primary model first (should work since it's Xenova)
-        if (await this.tryLoadModel(this.modelName)) {
-            return;
-        }
-        // Try fallback models if primary fails
-        console.warn(`Primary model ${this.modelName} failed, trying fallbacks...`);
-        for (const fallbackModel of CrossEncoderReranker.FALLBACK_MODELS) {
-            if (fallbackModel === this.modelName)
-                continue; // Skip already tried model
-            console.warn(`Trying fallback model: ${fallbackModel}`);
-            if (await this.tryLoadModel(fallbackModel)) {
-                this.modelName = fallbackModel;
-                return;
-            }
-        }
-        console.warn('All embedding models failed to load. Reranking will be disabled.');
-        this.model = null;
-        this.tokenizer = null;
+        await this.tryLoadModel(this.modelName);
     }
     /**
      * Try to load a specific model
      */
     async tryLoadModel(modelName) {
-        try {
-            console.log(`Loading cross-encoder model: ${modelName}`);
-            // Ensure polyfills are set up exactly like the working standalone version
-            this.ensurePolyfills();
-            // Use the exact same approach as the working standalone test
-            const { AutoTokenizer, AutoModelForSequenceClassification } = await import('@huggingface/transformers');
-            console.log('Loading model...');
-            this.model = await AutoModelForSequenceClassification.from_pretrained(modelName, {
-                cache_dir: config.model_cache_path,
-                dtype: 'fp32'
-            });
-            console.log('Loading tokenizer...');
-            this.tokenizer = await AutoTokenizer.from_pretrained(modelName, {
-                cache_dir: config.model_cache_path
-            });
-            console.log(`Cross-encoder model loaded successfully: ${modelName}`);
-            return true;
-        }
-        catch (error) {
-            console.warn(`Failed to load model ${modelName}: ${error instanceof Error ? error.message : 'Unknown error'}`);
-            return false;
-        }
+        console.log(`Loading cross-encoder model: ${modelName}`);
+        // Ensure polyfills are set up exactly like the working standalone version
+        this.ensurePolyfills();
+        // Use the exact same approach as the working standalone test
+        const { AutoTokenizer, AutoModelForSequenceClassification } = await import('@huggingface/transformers');
+        console.log('Loading model...');
+        this.model = await AutoModelForSequenceClassification.from_pretrained(modelName, {
+            cache_dir: config.model_cache_path,
+            dtype: 'fp32'
+        });
+        console.log('Loading tokenizer...');
+        this.tokenizer = await AutoTokenizer.from_pretrained(modelName, {
+            cache_dir: config.model_cache_path
+        });
+        console.log(`Cross-encoder model loaded successfully: ${modelName}`);
     }
     /**
      * Rerank search results using embedding similarity scoring

package/dist/text/sentence-transformer-embedder.d.ts

@@ -0,0 +1,96 @@
+/**
+ * TEXT IMPLEMENTATION — Sentence Transformer Embedder Implementation
+ * Implements UniversalEmbedder interface for sentence-transformer models
+ * Adapts existing text embedding logic to the universal interface
+ */
+import '../dom-polyfills.js';
+import { BaseUniversalEmbedder, type EmbedderOptions } from '../core/abstract-embedder.js';
+import type { EmbeddingResult } from '../types.js';
+/**
+ * Sentence transformer embedder implementation
+ * Supports sentence-transformers/all-MiniLM-L6-v2 and Xenova/all-mpnet-base-v2
+ * Ensures consistent EmbeddingResult format with contentType='text'
+ * Adapts existing EmbeddingEngine to UniversalEmbedder interface
+ */
+export declare class SentenceTransformerEmbedder extends BaseUniversalEmbedder {
+    private embeddingEngine;
+    private resourceManager;
+    private embedderResourceId?;
+    private engineResourceId?;
+    constructor(modelName: string, options?: EmbedderOptions);
+    /**
+     * Load the sentence transformer model using existing EmbeddingEngine
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Clean up model resources with comprehensive disposal
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Embed text using the existing EmbeddingEngine
+     */
+    embedText(text: string): Promise<EmbeddingResult>;
+    /**
+     * Optimized batch processing using existing EmbeddingEngine and BatchProcessingOptimizer
+     * Overrides the base implementation for better performance with progress reporting
+     */
+    protected processBatch(batch: Array<{
+        content: string;
+        contentType: string;
+        metadata?: Record<string, any>;
+    }>): Promise<EmbeddingResult[]>;
+    /**
+     * Get model-specific information
+     */
+    getModelInfo(): {
+        capabilities: {
+            supportsSemanticSimilarity: boolean;
+            supportsTextClassification: boolean;
+            supportsTextClustering: boolean;
+            recommendedUseCase: string;
+            supportsText: boolean;
+            supportsImages: boolean;
+            supportsBatchProcessing: boolean;
+            supportsMetadata: boolean;
+            maxBatchSize?: number;
+            maxTextLength?: number;
+            supportedImageFormats?: readonly string[];
+            supportsMultimodal?: boolean;
+            supportsCrossModalSearch?: boolean;
+            unifiedEmbeddingSpace?: boolean;
+            reliableImplementation?: boolean;
+        };
+        name: string;
+        type: import("../core/universal-embedder.js").ModelType;
+        dimensions: number;
+        version: string;
+        supportedContentTypes: readonly string[];
+        requirements: import("../types.js").ModelRequirements;
+    };
+    /**
+     * Check if the model is suitable for a specific task
+     */
+    isSuitableForTask(task: 'similarity' | 'classification' | 'clustering' | 'retrieval'): boolean;
+    /**
+     * Embed document batch using existing EmbeddingEngine's optimized method
+     * This method provides compatibility with the existing document ingestion pipeline
+     */
+    embedDocumentBatch(chunks: string[]): Promise<EmbeddingResult[]>;
+    /**
+     * Get the model version from the underlying EmbeddingEngine
+     */
+    getModelVersion(): string;
+    /**
+     * Get the batch size from the underlying EmbeddingEngine
+     */
+    getBatchSize(): number;
+    /**
+     * Check if the underlying EmbeddingEngine is loaded
+     */
+    isEngineLoaded(): boolean;
+    /**
+     * Override isLoaded to check both internal state and engine state
+     */
+    isLoaded(): boolean;
+}
+//# sourceMappingURL=sentence-transformer-embedder.d.ts.map