rag-lite-ts 1.0.2 → 2.0.1

package/dist/multimodal/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Multimodal implementation layer exports
+ * Provides CLIP-based embedders for cross-modal search capabilities
+ */
+export { CLIPEmbedder } from './clip-embedder.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/multimodal/index.js

@@ -0,0 +1,6 @@
+/**
+ * Multimodal implementation layer exports
+ * Provides CLIP-based embedders for cross-modal search capabilities
+ */
+export { CLIPEmbedder } from './clip-embedder.js';
+//# sourceMappingURL=index.js.map

package/dist/run-error-recovery-tests.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Test Runner for Chameleon Error Recovery and Reliability Tests
+ * Runs the comprehensive error recovery test suite
+ */
+declare function runTests(): Promise<boolean>;
+export { runTests };
+//# sourceMappingURL=run-error-recovery-tests.d.ts.map

package/dist/run-error-recovery-tests.js

@@ -0,0 +1,101 @@
+/**
+ * Test Runner for Chameleon Error Recovery and Reliability Tests
+ * Runs the comprehensive error recovery test suite
+ */
+import { spawn } from 'child_process';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+async function runTests() {
+    console.log('🧪 Running Chameleon Error Recovery and Reliability Tests...\n');
+    const testFiles = [
+        'chameleon-error-recovery.test.ts',
+        'chameleon-reliability-integration.test.ts',
+        'chameleon-stress-testing.test.ts',
+        'chameleon-error-simulation.test.ts'
+    ];
+    let totalTests = 0;
+    let passedTests = 0;
+    let failedTests = 0;
+    for (const testFile of testFiles) {
+        console.log(`\n📋 Running ${testFile}...`);
+        try {
+            // Build the test file first
+            const buildProcess = spawn('npx', ['tsc', '--project', 'tsconfig.test.json'], {
+                stdio: 'pipe',
+                shell: true
+            });
+            await new Promise((resolve, reject) => {
+                buildProcess.on('close', (code) => {
+                    if (code === 0) {
+                        resolve(code);
+                    }
+                    else {
+                        reject(new Error(`Build failed with code ${code}`));
+                    }
+                });
+            });
+            // Run the compiled test
+            const testProcess = spawn('node', ['--test', `dist/${testFile.replace('.ts', '.js')}`], {
+                stdio: 'pipe',
+                shell: true
+            });
+            let output = '';
+            let errorOutput = '';
+            testProcess.stdout?.on('data', (data) => {
+                output += data.toString();
+            });
+            testProcess.stderr?.on('data', (data) => {
+                errorOutput += data.toString();
+            });
+            await new Promise((resolve) => {
+                testProcess.on('close', (code) => {
+                    console.log(`Exit code: ${code}`);
+                    if (output) {
+                        console.log('Output:', output);
+                    }
+                    if (errorOutput) {
+                        console.log('Errors:', errorOutput);
+                    }
+                    // Count tests (this is a simple approximation)
+                    const testMatches = output.match(/✓|×/g);
+                    const currentTests = testMatches ? testMatches.length : 0;
+                    totalTests += currentTests;
+                    if (code === 0) {
+                        passedTests += currentTests;
+                        console.log(`✅ ${testFile} completed successfully`);
+                    }
+                    else {
+                        failedTests += currentTests;
+                        console.log(`❌ ${testFile} failed`);
+                    }
+                    resolve(code);
+                });
+            });
+        }
+        catch (error) {
+            console.error(`❌ Failed to run ${testFile}:`, error instanceof Error ? error.message : String(error));
+            failedTests++;
+        }
+    }
+    console.log('\n📊 Test Summary:');
+    console.log(`Total Tests: ${totalTests}`);
+    console.log(`Passed: ${passedTests}`);
+    console.log(`Failed: ${failedTests}`);
+    if (failedTests === 0) {
+        console.log('\n🎉 All error recovery tests completed!');
+        console.log('✅ System demonstrates robust error handling and recovery mechanisms');
+    }
+    else {
+        console.log('\n⚠️  Some tests failed - this may be expected in test environments');
+        console.log('🔍 Review the output above for specific failure details');
+    }
+    return failedTests === 0;
+}
+// Run tests if this file is executed directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+    runTests().catch(console.error);
+}
+export { runTests };
+//# sourceMappingURL=run-error-recovery-tests.js.map

package/dist/search.d.ts

@@ -1,25 +1,44 @@
 /**
- * Public API SearchEngine - Simple constructor interface with internal factory usage
+ * Public API SearchEngine - Simple constructor with Chameleon Architecture
  *
- * This class provides a clean, simple API while using the new core architecture
- * internally. It handles dependency injection automatically.
+ * This class provides a clean, simple API that automatically adapts to the mode
+ * (text or multimodal) stored in the database during ingestion. The system detects
+ * the mode and creates the appropriate embedder and reranker without user intervention.
+ *
+ * Chameleon Architecture Features:
+ * - Automatic mode detection from database configuration
+ * - Seamless switching between text and multimodal modes
+ * - Appropriate embedder selection (sentence-transformer or CLIP)
+ * - Mode-specific reranking strategies
  *
  * @example
  * ```typescript
- * // Simple usage
+ * // Simple usage - mode automatically detected from database
  * const search = new SearchEngine('./index.bin', './db.sqlite');
  * const results = await search.search('query');
  *
- * // With options
+ * // Works for both text and multimodal databases
+ * // Text mode: uses sentence-transformer embeddings
+ * // Multimodal mode: uses CLIP embeddings for cross-modal search
+ *
+ * // With options (advanced)
  * const search = new SearchEngine('./index.bin', './db.sqlite', {
- *   embeddingModel: 'all-MiniLM-L6-v2',
  *   enableReranking: true
  * });
  * ```
  */
-import { type TextSearchOptions } from './factories/index.js';
 import type { SearchResult, SearchOptions, EmbedFunction, RerankFunction } from './core/types.js';
-export interface SearchEngineOptions extends TextSearchOptions {
+export interface SearchEngineOptions {
+    /** Embedding model name override */
+    embeddingModel?: string;
+    /** Embedding batch size override */
+    batchSize?: number;
+    /** Reranking model name override */
+    rerankingModel?: string;
+    /** Whether to enable reranking (default: true) */
+    enableReranking?: boolean;
+    /** Top-k results to return (default: from config) */
+    topK?: number;
     /** Custom embedding function (advanced usage) */
     embedFn?: EmbedFunction;
     /** Custom reranking function (advanced usage) */
@@ -33,13 +52,45 @@ export declare class SearchEngine {
     private initPromise;
     constructor(indexPath: string, dbPath: string, options?: SearchEngineOptions);
     /**
-     * Initialize the search engine using the factory or direct injection
+     * Initialize the search engine using polymorphic factory or direct injection
+     *
+     * Chameleon Architecture Implementation:
+     * - Automatically detects mode from database (text or multimodal)
+     * - Creates appropriate embedder based on detected mode
+     * - Applies mode-specific reranking strategies
+     * - Provides seamless polymorphic behavior
      */
     private initialize;
     /**
      * Perform semantic search
      */
     search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Retrieve content by ID in the specified format
+     * @param contentId - Content ID to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to content in requested format
+     */
+    getContent(contentId: string, format?: 'file' | 'base64'): Promise<string>;
+    /**
+     * Retrieve multiple content items efficiently in batch
+     * @param contentIds - Array of content IDs to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to array of content in requested format
+     */
+    getContentBatch(contentIds: string[], format?: 'file' | 'base64'): Promise<string[]>;
+    /**
+     * Retrieve content metadata for result enhancement
+     * @param contentId - Content ID to get metadata for
+     * @returns Promise that resolves to content metadata
+     */
+    getContentMetadata(contentId: string): Promise<import('./core/content-resolver.js').ContentMetadata>;
+    /**
+     * Verify that content exists and is accessible
+     * @param contentId - Content ID to verify
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    verifyContentExists(contentId: string): Promise<boolean>;
     /**
      * Clean up resources
      */

package/dist/search.js

@@ -1,24 +1,33 @@
 /**
- * Public API SearchEngine - Simple constructor interface with internal factory usage
+ * Public API SearchEngine - Simple constructor with Chameleon Architecture
  *
- * This class provides a clean, simple API while using the new core architecture
- * internally. It handles dependency injection automatically.
+ * This class provides a clean, simple API that automatically adapts to the mode
+ * (text or multimodal) stored in the database during ingestion. The system detects
+ * the mode and creates the appropriate embedder and reranker without user intervention.
+ *
+ * Chameleon Architecture Features:
+ * - Automatic mode detection from database configuration
+ * - Seamless switching between text and multimodal modes
+ * - Appropriate embedder selection (sentence-transformer or CLIP)
+ * - Mode-specific reranking strategies
  *
  * @example
  * ```typescript
- * // Simple usage
+ * // Simple usage - mode automatically detected from database
  * const search = new SearchEngine('./index.bin', './db.sqlite');
  * const results = await search.search('query');
  *
- * // With options
+ * // Works for both text and multimodal databases
+ * // Text mode: uses sentence-transformer embeddings
+ * // Multimodal mode: uses CLIP embeddings for cross-modal search
+ *
+ * // With options (advanced)
  * const search = new SearchEngine('./index.bin', './db.sqlite', {
- *   embeddingModel: 'all-MiniLM-L6-v2',
  *   enableReranking: true
  * });
  * ```
  */
 import { SearchEngine as CoreSearchEngine } from './core/search.js';
-import { TextSearchFactory } from './factories/index.js';
 export class SearchEngine {
     indexPath;
     dbPath;
@@ -42,7 +51,13 @@ export class SearchEngine {
         }
     }
     /**
-     * Initialize the search engine using the factory or direct injection
+     * Initialize the search engine using polymorphic factory or direct injection
+     *
+     * Chameleon Architecture Implementation:
+     * - Automatically detects mode from database (text or multimodal)
+     * - Creates appropriate embedder based on detected mode
+     * - Applies mode-specific reranking strategies
+     * - Provides seamless polymorphic behavior
      */
     async initialize() {
         if (this.coreEngine) {
@@ -74,12 +89,18 @@ export class SearchEngine {
                 const db = await openDatabase(this.dbPath);
                 const indexManager = new IndexManager(this.indexPath, this.dbPath, modelDefaults.dimensions, this.options.embeddingModel);
                 await indexManager.initialize();
+                // Create ContentResolver for unified content system
+                const { ContentResolver } = await import('./core/content-resolver.js');
+                const contentResolver = new ContentResolver(db);
                 // Create core engine with dependency injection
-                this.coreEngine = new CoreSearchEngine(embedFn, indexManager, db, this.options.rerankFn);
+                this.coreEngine = new CoreSearchEngine(embedFn, indexManager, db, this.options.rerankFn, contentResolver);
             }
             else {
-                // Use factory for standard initialization
-                this.coreEngine = await TextSearchFactory.create(this.indexPath, this.dbPath, this.options);
+                // Use core polymorphic factory for automatic mode detection (Chameleon Architecture)
+                // This enables SearchEngine to automatically adapt to text or multimodal mode
+                // based on the configuration stored in the database during ingestion
+                const { PolymorphicSearchFactory } = await import('./core/polymorphic-search-factory.js');
+                this.coreEngine = await PolymorphicSearchFactory.create(this.indexPath, this.dbPath);
             }
         })();
         return this.initPromise;
@@ -94,6 +115,56 @@ export class SearchEngine {
         }
         return this.coreEngine.search(query, options);
     }
+    /**
+     * Retrieve content by ID in the specified format
+     * @param contentId - Content ID to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to content in requested format
+     */
+    async getContent(contentId, format = 'file') {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.getContent(contentId, format);
+    }
+    /**
+     * Retrieve multiple content items efficiently in batch
+     * @param contentIds - Array of content IDs to retrieve
+     * @param format - Format to return ('file' for CLI clients, 'base64' for MCP clients)
+     * @returns Promise that resolves to array of content in requested format
+     */
+    async getContentBatch(contentIds, format = 'file') {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.getContentBatch(contentIds, format);
+    }
+    /**
+     * Retrieve content metadata for result enhancement
+     * @param contentId - Content ID to get metadata for
+     * @returns Promise that resolves to content metadata
+     */
+    async getContentMetadata(contentId) {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.getContentMetadata(contentId);
+    }
+    /**
+     * Verify that content exists and is accessible
+     * @param contentId - Content ID to verify
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    async verifyContentExists(contentId) {
+        await this.initialize();
+        if (!this.coreEngine) {
+            throw new Error('SearchEngine failed to initialize');
+        }
+        return this.coreEngine.verifyContentExists(contentId);
+    }
     /**
      * Clean up resources
      */

package/dist/test-utils.d.ts

@@ -2,35 +2,17 @@
  * Test utilities for multi-model support
  * Provides common configurations and helpers for testing with different embedding models
  */
-export declare const TEST_MODELS: readonly [{
-    readonly name: "sentence-transformers/all-MiniLM-L6-v2";
-    readonly dimensions: 384;
-    readonly chunkSize: 250;
-    readonly batchSize: 16;
-}, {
-    readonly name: "Xenova/all-mpnet-base-v2";
-    readonly dimensions: 768;
-    readonly chunkSize: 400;
-    readonly batchSize: 8;
-}];
+export interface TestModel {
+    name: string;
+    dimensions: number;
+    chunkSize: number;
+    batchSize: number;
+}
+export declare const TEST_MODELS: TestModel[];
 /**
  * Retrieve model configuration by name
  * @param modelName - The name of the model to retrieve
  * @returns Model configuration object or undefined if not found
  */
-export declare function getTestModel(modelName: string): {
-    readonly name: "sentence-transformers/all-MiniLM-L6-v2";
-    readonly dimensions: 384;
-    readonly chunkSize: 250;
-    readonly batchSize: 16;
-} | {
-    readonly name: "Xenova/all-mpnet-base-v2";
-    readonly dimensions: 768;
-    readonly chunkSize: 400;
-    readonly batchSize: 8;
-} | undefined;
-/**
- * Type for test model configuration
- */
-export type TestModel = typeof TEST_MODELS[number];
+export declare function getTestModel(modelName: string): TestModel | undefined;
 //# sourceMappingURL=test-utils.d.ts.map

package/dist/text/chunker.d.ts

@@ -11,6 +11,7 @@ export interface Document {
     source: string;
     title: string;
     content: string;
+    metadata?: Record<string, any>;
 }
 export interface Chunk {
     text: string;

package/dist/text/embedder.js

@@ -2,6 +2,7 @@ import '../dom-polyfills.js';
 import { createHash } from 'crypto';
 import { config } from '../core/config.js';
 import { handleError, ErrorCategory, ErrorSeverity, safeExecute } from '../core/error-handler.js';
+import { createModelLoadingError, createInvalidContentError, createMissingDependencyError } from '../core/actionable-error-messages.js';
 /**
  * List of supported embedding models
  */
@@ -22,8 +23,7 @@ export class EmbeddingEngine {
         this.batchSize = batchSize || config.batch_size;
         // Validate that the model is supported
         if (!SUPPORTED_MODELS.includes(this.modelName)) {
-            throw new Error(`Unsupported model: ${this.modelName}\n` +
-                `Supported models: ${SUPPORTED_MODELS.join(', ')}`);
+            throw createModelLoadingError(this.modelName, `Model not in supported list. Supported models: ${SUPPORTED_MODELS.join(', ')}`, { operationContext: 'EmbeddingEngine constructor' });
         }
         console.log(`🤖 EmbeddingEngine initialized with model: ${this.modelName}, batchSize: ${this.batchSize}`);
     }
@@ -88,7 +88,10 @@ export class EmbeddingEngine {
      */
     async embedBatch(texts) {
         if (!this.model) {
-            throw new Error('Model not loaded. Call loadModel() first.');
+            throw createMissingDependencyError('model', 'object', {
+                operationContext: 'embedBatch',
+                includeTroubleshooting: true
+            });
         }
         if (texts.length === 0) {
             return [];
@@ -123,7 +126,8 @@ export class EmbeddingEngine {
                 const vector = new Float32Array(embeddingData[i]);
                 results.push({
                     embedding_id,
-                    vector
+                    vector,
+                    contentType: 'text'
                 });
             }
             return results;
@@ -173,7 +177,8 @@ export class EmbeddingEngine {
             const vector = new Float32Array(embeddingData[0]);
             return {
                 embedding_id,
-                vector
+                vector,
+                contentType: 'text'
             };
         }
         catch (error) {
@@ -189,7 +194,9 @@ export class EmbeddingEngine {
     async embedSingle(text) {
         const results = await this.embedBatch([text]);
         if (results.length === 0) {
-            throw new Error('Failed to generate embedding for single text');
+            throw createInvalidContentError('text', 'empty', {
+                operationContext: 'embedText'
+            });
         }
         return results[0];
     }
@@ -357,10 +364,10 @@ export function createTextEmbedFunction(modelName, batchSize) {
         }
         // Use the existing embedSingle method
         const result = await engine.embedSingle(query);
-        // Add contentType to the result
+        // Ensure contentType is present (should already be included from embedSingle)
         return {
             ...result,
-            contentType: 'text'
+            contentType: result.contentType || 'text'
         };
     };
     return embedFunction;

package/dist/text/index.d.ts

@@ -3,5 +3,6 @@ export { CrossEncoderReranker, createTextRerankFunction, createTextReranker } fr
 export { countTokens, getTokenizer, resetTokenizer } from './tokenizer.js';
 export { chunkDocument, type Chunk, type Document } from '../core/chunker.js';
 export { type ChunkConfig } from '../core/chunker.js';
+export { SentenceTransformerEmbedder } from './sentence-transformer-embedder.js';
 export * from './preprocessors/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/text/index.js

@@ -3,6 +3,7 @@ export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createT
 export { CrossEncoderReranker, createTextRerankFunction, createTextReranker } from './reranker.js';
 export { countTokens, getTokenizer, resetTokenizer } from './tokenizer.js';
 export { chunkDocument } from '../core/chunker.js';
+export { SentenceTransformerEmbedder } from './sentence-transformer-embedder.js';
 // Re-export preprocessors
 export * from './preprocessors/index.js';
 //# sourceMappingURL=index.js.map

package/dist/text/reranker.d.ts

@@ -8,13 +8,12 @@ export declare class CrossEncoderReranker {
     private model;
     private tokenizer;
     private modelName;
-    private static readonly FALLBACK_MODELS;
     /**
      * Ensure DOM polyfills are set up for transformers.js
      */
     private ensurePolyfills;
     /**
-     * Load the embedding model with graceful fallback
+     * Load the embedding model
      */
     loadModel(): Promise<void>;
     /**

package/dist/text/reranker.js

@@ -18,12 +18,6 @@ export class CrossEncoderReranker {
     model = null; // Use any to avoid complex transformers.js typing issues
     tokenizer = null;
     modelName = 'Xenova/ms-marco-MiniLM-L-6-v2'; // Use working cross-encoder model
-    // Alternative models in case the primary fails
-    static FALLBACK_MODELS = [
-        'Xenova/ms-marco-MiniLM-L-6-v2', // Primary - proven to work in standalone test
-        'cross-encoder/ms-marco-MiniLM-L-6-v2', // Original (may have issues)
-        'cross-encoder/ms-marco-MiniLM-L-2-v2', // Smaller original (may have issues)
-    ];
     /**
      * Ensure DOM polyfills are set up for transformers.js
      */
@@ -40,54 +34,30 @@ export class CrossEncoderReranker {
         }
     }
     /**
-     * Load the embedding model with graceful fallback
+     * Load the embedding model
      */
     async loadModel() {
-        // Try primary model first (should work since it's Xenova)
-        if (await this.tryLoadModel(this.modelName)) {
-            return;
-        }
-        // Try fallback models if primary fails
-        console.warn(`Primary model ${this.modelName} failed, trying fallbacks...`);
-        for (const fallbackModel of CrossEncoderReranker.FALLBACK_MODELS) {
-            if (fallbackModel === this.modelName)
-                continue; // Skip already tried model
-            console.warn(`Trying fallback model: ${fallbackModel}`);
-            if (await this.tryLoadModel(fallbackModel)) {
-                this.modelName = fallbackModel;
-                return;
-            }
-        }
-        console.warn('All embedding models failed to load. Reranking will be disabled.');
-        this.model = null;
-        this.tokenizer = null;
+        await this.tryLoadModel(this.modelName);
     }
     /**
      * Try to load a specific model
      */
     async tryLoadModel(modelName) {
-        try {
-            console.log(`Loading cross-encoder model: ${modelName}`);
-            // Ensure polyfills are set up exactly like the working standalone version
-            this.ensurePolyfills();
-            // Use the exact same approach as the working standalone test
-            const { AutoTokenizer, AutoModelForSequenceClassification } = await import('@huggingface/transformers');
-            console.log('Loading model...');
-            this.model = await AutoModelForSequenceClassification.from_pretrained(modelName, {
-                cache_dir: config.model_cache_path,
-                dtype: 'fp32'
-            });
-            console.log('Loading tokenizer...');
-            this.tokenizer = await AutoTokenizer.from_pretrained(modelName, {
-                cache_dir: config.model_cache_path
-            });
-            console.log(`Cross-encoder model loaded successfully: ${modelName}`);
-            return true;
-        }
-        catch (error) {
-            console.warn(`Failed to load model ${modelName}: ${error instanceof Error ? error.message : 'Unknown error'}`);
-            return false;
-        }
+        console.log(`Loading cross-encoder model: ${modelName}`);
+        // Ensure polyfills are set up exactly like the working standalone version
+        this.ensurePolyfills();
+        // Use the exact same approach as the working standalone test
+        const { AutoTokenizer, AutoModelForSequenceClassification } = await import('@huggingface/transformers');
+        console.log('Loading model...');
+        this.model = await AutoModelForSequenceClassification.from_pretrained(modelName, {
+            cache_dir: config.model_cache_path,
+            dtype: 'fp32'
+        });
+        console.log('Loading tokenizer...');
+        this.tokenizer = await AutoTokenizer.from_pretrained(modelName, {
+            cache_dir: config.model_cache_path
+        });
+        console.log(`Cross-encoder model loaded successfully: ${modelName}`);
     }
     /**
      * Rerank search results using embedding similarity scoring