rag-lite-ts 2.2.0 → 2.3.1

package/dist/esm/factories/generator-factory.d.ts

@@ -0,0 +1,88 @@
+/**
+ * FACTORY MODULE — Generator Factory for RAG Response Generation
+ *
+ * Factory functions for creating response generator instances.
+ * Handles model validation, lazy loading, and proper initialization.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, DEFAULT, uses top 3 chunks)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, uses top 5 chunks)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import type { ResponseGenerator, GeneratorCreationOptions, GenerateFunction } from '../core/response-generator.js';
+/**
+ * Create a response generator for the specified model
+ *
+ * Uses lazy loading to defer model initialization until first use.
+ * Validates model compatibility before creation.
+ *
+ * @param modelName - Name of the generator model (default: SmolLM2-135M-Instruct)
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a ResponseGenerator instance
+ * @throws {GeneratorValidationError} If model is not supported
+ *
+ * @example
+ * ```typescript
+ * // Create default generator (recommended)
+ * const generator = await createResponseGenerator();
+ *
+ * // Create higher quality generator
+ * const generator = await createResponseGenerator('HuggingFaceTB/SmolLM2-360M-Instruct');
+ *
+ * // Create with options
+ * const generator = await createResponseGenerator('HuggingFaceTB/SmolLM2-360M-Instruct', {
+ *   cachePath: './models'
+ * });
+ * ```
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export declare function createResponseGenerator(modelName?: string, options?: GeneratorCreationOptions): Promise<ResponseGenerator>;
+/**
+ * Create a GenerateFunction from a model name
+ *
+ * This is a convenience function that creates a generator and wraps it
+ * in a function suitable for dependency injection into SearchEngine.
+ *
+ * @param modelName - Name of the generator model
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a GenerateFunction
+ *
+ * @example
+ * ```typescript
+ * const generateFn = await createGenerateFunctionFromModel();
+ * const result = await generateFn(query, chunks);
+ * ```
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export declare function createGenerateFunctionFromModel(modelName?: string, options?: GeneratorCreationOptions): Promise<GenerateFunction>;
+/**
+ * Get the default generator model name
+ */
+export declare function getDefaultGeneratorModel(): string;
+/**
+ * List available generator models
+ */
+export declare function listGeneratorModels(): string[];
+/**
+ * Check if a model name is a valid generator
+ */
+export declare function isValidGeneratorModel(modelName: string): boolean;
+/**
+ * Get recommended generator for specific use case
+ *
+ * @param preferSpeed - Prefer faster generation over quality
+ * @param preferQuality - Prefer higher quality over speed
+ * @returns Recommended model name
+ */
+export declare function getRecommendedGenerator(options?: {
+    preferSpeed?: boolean;
+    preferQuality?: boolean;
+}): string;
+export { GeneratorRegistry, DEFAULT_GENERATOR_MODEL } from '../core/generator-registry.js';
+//# sourceMappingURL=generator-factory.d.ts.map

package/dist/esm/factories/generator-factory.js

@@ -0,0 +1,151 @@
+/**
+ * FACTORY MODULE — Generator Factory for RAG Response Generation
+ *
+ * Factory functions for creating response generator instances.
+ * Handles model validation, lazy loading, and proper initialization.
+ *
+ * SUPPORTED MODELS:
+ * - HuggingFaceTB/SmolLM2-135M-Instruct (instruct, balanced, DEFAULT, uses top 3 chunks)
+ * - HuggingFaceTB/SmolLM2-360M-Instruct (instruct, higher quality, uses top 5 chunks)
+ *
+ * PREREQUISITES:
+ * - Reranking must be enabled for response generation
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import { createGenerateFunction, GeneratorValidationError } from '../core/response-generator.js';
+import { GeneratorRegistry, DEFAULT_GENERATOR_MODEL, getGeneratorType } from '../core/generator-registry.js';
+import { LazyGeneratorLoader } from '../core/lazy-dependency-loader.js';
+// =============================================================================
+// GENERATOR FACTORY
+// =============================================================================
+/**
+ * Create a response generator for the specified model
+ *
+ * Uses lazy loading to defer model initialization until first use.
+ * Validates model compatibility before creation.
+ *
+ * @param modelName - Name of the generator model (default: SmolLM2-135M-Instruct)
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a ResponseGenerator instance
+ * @throws {GeneratorValidationError} If model is not supported
+ *
+ * @example
+ * ```typescript
+ * // Create default generator (recommended)
+ * const generator = await createResponseGenerator();
+ *
+ * // Create higher quality generator
+ * const generator = await createResponseGenerator('HuggingFaceTB/SmolLM2-360M-Instruct');
+ *
+ * // Create with options
+ * const generator = await createResponseGenerator('HuggingFaceTB/SmolLM2-360M-Instruct', {
+ *   cachePath: './models'
+ * });
+ * ```
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export async function createResponseGenerator(modelName = DEFAULT_GENERATOR_MODEL, options = {}) {
+    console.log(`🏭 [EXPERIMENTAL] Creating response generator: ${modelName}`);
+    // Step 1: Validate model
+    const validation = GeneratorRegistry.validateGenerator(modelName);
+    if (!validation.isValid) {
+        throw new GeneratorValidationError(modelName, GeneratorRegistry.getSupportedGenerators(), validation.errors.join('; '));
+    }
+    // Log warnings
+    if (validation.warnings.length > 0) {
+        console.warn(`⚠️  Warnings for generator '${modelName}':`);
+        validation.warnings.forEach(w => console.warn(`  • ${w}`));
+    }
+    // Log suggestions
+    if (validation.suggestions.length > 0) {
+        console.info(`💡 Suggestions for generator '${modelName}':`);
+        validation.suggestions.forEach(s => console.info(`  • ${s}`));
+    }
+    // Step 2: Get model type and create appropriate generator
+    const modelType = getGeneratorType(modelName);
+    if (!modelType) {
+        throw new GeneratorValidationError(modelName, GeneratorRegistry.getSupportedGenerators(), `Could not determine model type for '${modelName}'`);
+    }
+    // Step 3: Use lazy loading to create the generator
+    let generator;
+    switch (modelType) {
+        case 'instruct':
+            generator = await LazyGeneratorLoader.loadInstructGenerator(modelName, options);
+            break;
+        case 'causal-lm':
+            generator = await LazyGeneratorLoader.loadCausalLMGenerator(modelName, options);
+            break;
+        default:
+            throw new GeneratorValidationError(modelName, GeneratorRegistry.getSupportedGenerators(), `Unsupported generator type: ${modelType}`);
+    }
+    console.log(`✅ [EXPERIMENTAL] Response generator created: ${modelName}`);
+    return generator;
+}
+/**
+ * Create a GenerateFunction from a model name
+ *
+ * This is a convenience function that creates a generator and wraps it
+ * in a function suitable for dependency injection into SearchEngine.
+ *
+ * @param modelName - Name of the generator model
+ * @param options - Optional configuration options
+ * @returns Promise resolving to a GenerateFunction
+ *
+ * @example
+ * ```typescript
+ * const generateFn = await createGenerateFunctionFromModel();
+ * const result = await generateFn(query, chunks);
+ * ```
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export async function createGenerateFunctionFromModel(modelName = DEFAULT_GENERATOR_MODEL, options = {}) {
+    const generator = await createResponseGenerator(modelName, options);
+    return createGenerateFunction(generator);
+}
+// =============================================================================
+// UTILITY FUNCTIONS
+// =============================================================================
+/**
+ * Get the default generator model name
+ */
+export function getDefaultGeneratorModel() {
+    return DEFAULT_GENERATOR_MODEL;
+}
+/**
+ * List available generator models
+ */
+export function listGeneratorModels() {
+    return GeneratorRegistry.getSupportedGenerators();
+}
+/**
+ * Check if a model name is a valid generator
+ */
+export function isValidGeneratorModel(modelName) {
+    return GeneratorRegistry.getGeneratorInfo(modelName) !== null;
+}
+/**
+ * Get recommended generator for specific use case
+ *
+ * @param preferSpeed - Prefer faster generation over quality
+ * @param preferQuality - Prefer higher quality over speed
+ * @returns Recommended model name
+ */
+export function getRecommendedGenerator(options = {}) {
+    const { preferSpeed, preferQuality } = options;
+    if (preferSpeed) {
+        return 'Xenova/distilgpt2';
+    }
+    if (preferQuality) {
+        return 'HuggingFaceTB/SmolLM2-360M-Instruct';
+    }
+    // Default: balanced option
+    return DEFAULT_GENERATOR_MODEL;
+}
+// =============================================================================
+// EXPORTS
+// =============================================================================
+export { GeneratorRegistry, DEFAULT_GENERATOR_MODEL } from '../core/generator-registry.js';
+//# sourceMappingURL=generator-factory.js.map

package/dist/esm/factories/index.d.ts

@@ -24,4 +24,5 @@
 export { IngestionFactory } from './ingestion-factory.js';
 export { SearchFactory } from './search-factory.js';
 export type { IngestionFactoryOptions, ContentSystemConfig } from './ingestion-factory.js';
+export { createResponseGenerator, createGenerateFunctionFromModel, getDefaultGeneratorModel, listGeneratorModels, isValidGeneratorModel, getRecommendedGenerator } from './generator-factory.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/factories/index.js

@@ -26,4 +26,9 @@ export { IngestionFactory } from './ingestion-factory.js';
 // Polymorphic search factory (recommended for automatic mode detection)
 // Re-exported from core for convenience
 export { SearchFactory } from './search-factory.js';
+// =============================================================================
+// EXPERIMENTAL: Generator Factory
+// =============================================================================
+// Generator factory for creating response generators (experimental)
+export { createResponseGenerator, createGenerateFunctionFromModel, getDefaultGeneratorModel, listGeneratorModels, isValidGeneratorModel, getRecommendedGenerator } from './generator-factory.js';
 //# sourceMappingURL=index.js.map

package/dist/esm/factories/ingestion-factory.js

@@ -296,8 +296,7 @@ export class IngestionFactory {
             // Preserve custom error messages for model mismatch and mode mismatch
             if (error instanceof Error && (error.message.includes('Model mismatch') ||
                 error.message.includes('Mode mismatch') ||
-                error.message.includes('--force-rebuild') ||
-                error.message.includes('--rebuild-if-needed'))) {
+                error.message.includes('--force-rebuild'))) {
                 throw error; // Re-throw custom validation errors as-is
             }
             throw createFactoryCreationError('IngestionFactory', error instanceof Error ? error.message : 'Unknown error', { operationContext: 'ingestion pipeline creation' });
@@ -366,13 +365,10 @@ export class IngestionFactory {
                             `❌ Model mismatch: Database is configured for '${existingSystemInfo.modelName}', but '${effectiveModel}' was requested.`,
                             '',
                             '🛠️  How to fix this:',
-                            '   1. Use --force-rebuild to change models:',
+                            '   1. Use --force-rebuild to rebuild from scratch:',
                             '      raglite ingest <path> --model ' + effectiveModel + ' --force-rebuild',
                             '',
-                            '   2. Or use --rebuild-if-needed for automatic handling:',
-                            '      raglite ingest <path> --model ' + effectiveModel + ' --rebuild-if-needed',
-                            '',
-                            '   3. Or continue using the existing model:',
+                            '   2. Or continue using the existing model:',
                             '      raglite ingest <path>  # Uses ' + existingSystemInfo.modelName,
                             '',
                             '🔍 Model switching requires rebuilding the vector index because different models',

package/dist/esm/factories/search-factory.js

@@ -281,6 +281,17 @@ export class SearchFactory {
                 enhancedMessage += '\n   The index was created with a different model. Rebuild the index:';
                 enhancedMessage += '\n   raglite ingest <directory> --force-rebuild';
             }
+            else if (error.message.includes('Cannot enlarge memory') ||
+                error.message.includes('WebAssembly memory limit') ||
+                error.message.includes('memory limit exceeded')) {
+                enhancedMessage += '\n\n💡 WebAssembly memory limit exceeded.';
+                enhancedMessage += '\n   Your vector index is too large for the 2GB WebAssembly memory limit.';
+                enhancedMessage += '\n   Solutions:';
+                enhancedMessage += '\n   1. Increase Node.js memory: node --max-old-space-size=4096 ...';
+                enhancedMessage += '\n   2. Split your data into smaller indexes';
+                enhancedMessage += '\n   3. Use a smaller embedding model (fewer dimensions)';
+                enhancedMessage += '\n   4. Rebuild the index with fewer vectors';
+            }
             return new Error(enhancedMessage);
         }
         return new Error(`SearchFactory.create failed: Unknown error`);

package/dist/esm/index-manager.d.ts

@@ -86,11 +86,12 @@ export declare class IndexManager {
     saveGroupedIndex(textEmbeddings: EmbeddingResult[], imageEmbeddings: EmbeddingResult[]): Promise<void>;
     /**
      * Search for similar vectors
+     * Now async due to worker-based VectorIndex implementation
      */
-    search(queryVector: Float32Array, k?: number, contentType?: 'text' | 'image' | 'combined'): {
+    search(queryVector: Float32Array, k?: number, contentType?: 'text' | 'image' | 'combined'): Promise<{
         embeddingIds: string[];
         distances: number[];
-    };
+    }>;
     /**
      * Get index statistics
      */
@@ -109,8 +110,27 @@ export declare class IndexManager {
      */
     private unhashEmbeddingId;
     /**
-     * Close database connection
+     * Close database connection and cleanup vector index worker
      */
     close(): Promise<void>;
+    /**
+     * Reset the vector index by clearing all vectors while keeping the index structure.
+     * This is a safer alternative to file deletion that avoids file locking issues on Windows.
+     *
+     * The reset operation:
+     * 1. Clears in-memory HNSW index
+     * 2. Clears in-memory vector storage and ID mappings
+     * 3. Reinitializes an empty index with the same parameters
+     * 4. Saves the empty index to disk (overwrites existing file)
+     *
+     * @returns Promise that resolves when reset is complete
+     */
+    reset(): Promise<void>;
+    /**
+     * Check if the index has any vectors
+     * @returns true if the index contains vectors, false if empty
+     * Now async due to worker-based VectorIndex implementation
+     */
+    hasVectors(): Promise<boolean>;
 }
 //# sourceMappingURL=index-manager.d.ts.map

package/dist/esm/index-manager.js

@@ -64,7 +64,7 @@ export class IndexManager {
                 this.hashEmbeddingId(chunk.embedding_id); // This will populate the mapping
             }
             this.isInitialized = true;
-            const vectorCount = this.vectorIndex.getCurrentCount();
+            const vectorCount = await this.vectorIndex.getCurrentCount();
             console.log(`Index manager initialized with ${vectorCount} vectors${this.textIndex && this.imageIndex ? ' (multi-graph mode)' : ''}`);
         }
         catch (error) {
@@ -145,17 +145,18 @@ export class IndexManager {
                 vector: embedding.vector
             }));
             // Check if we need to resize the index before adding
-            const currentCount = this.vectorIndex.getCurrentCount();
-            const newCount = currentCount + vectors.length;
+            const initialCount = await this.vectorIndex.getCurrentCount();
+            const newCount = initialCount + vectors.length;
             const currentCapacity = 100000; // This should match the initial capacity
             if (newCount > currentCapacity * 0.9) {
                 const newCapacity = Math.ceil(newCount * 1.5);
                 console.log(`Resizing index from ${currentCapacity} to ${newCapacity} to accommodate new vectors`);
-                this.vectorIndex.resizeIndex(newCapacity);
+                await this.vectorIndex.resizeIndex(newCapacity);
             }
             // Add vectors incrementally (this is the key requirement - no rebuild needed)
-            this.vectorIndex.addVectors(vectors);
-            console.log(`Incrementally added ${embeddings.length} vectors to index (total: ${this.vectorIndex.getCurrentCount()})`);
+            await this.vectorIndex.addVectors(vectors);
+            const finalCount = await this.vectorIndex.getCurrentCount();
+            console.log(`Incrementally added ${embeddings.length} vectors to index (total: ${finalCount})`);
             // Save the updated index
             await this.saveIndex();
         }
@@ -223,7 +224,7 @@ export class IndexManager {
             const currentCapacity = 100000; // Default capacity
             if (chunkData.length > currentCapacity * 0.8) {
                 const newCapacity = Math.ceil(chunkData.length * 1.5);
-                this.vectorIndex.resizeIndex(newCapacity);
+                await this.vectorIndex.resizeIndex(newCapacity);
                 console.log(`Resized index capacity to ${newCapacity} for ${chunkData.length} chunks`);
             }
             // Update model version if provided
@@ -279,7 +280,7 @@ export class IndexManager {
             const currentCapacity = 100000;
             if (chunkData.length > currentCapacity * 0.8) {
                 const newCapacity = Math.ceil(chunkData.length * 1.5);
-                this.vectorIndex.resizeIndex(newCapacity);
+                await this.vectorIndex.resizeIndex(newCapacity);
                 console.log(`Resized index capacity to ${newCapacity}`);
             }
             // Re-generate embeddings for all chunks
@@ -294,7 +295,7 @@ export class IndexManager {
                 id: this.hashEmbeddingId(embedding.embedding_id),
                 vector: embedding.vector
             }));
-            this.vectorIndex.addVectors(vectors);
+            await this.vectorIndex.addVectors(vectors);
             console.log(`Added ${vectors.length} vectors to rebuilt index`);
             // Update model version
             await this.updateModelVersion(embeddingEngine.getModelVersion());
@@ -414,12 +415,12 @@ export class IndexManager {
                     // Create text-only index
                     this.textIndex = new VectorIndex(`${this.indexPath}.text`, this.vectorIndexOptions);
                     await this.textIndex.initialize();
-                    this.textIndex.addVectors(indexData.textVectors);
+                    await this.textIndex.addVectors(indexData.textVectors);
                     console.log(`✓ Text index created with ${indexData.textVectors.length} vectors`);
                     // Create image-only index
                     this.imageIndex = new VectorIndex(`${this.indexPath}.image`, this.vectorIndexOptions);
                     await this.imageIndex.initialize();
-                    this.imageIndex.addVectors(indexData.imageVectors);
+                    await this.imageIndex.addVectors(indexData.imageVectors);
                     console.log(`✓ Image index created with ${indexData.imageVectors.length} vectors`);
                     console.log('✓ Specialized indexes ready for content type filtering');
                 }
@@ -475,8 +476,9 @@ export class IndexManager {
     }
     /**
      * Search for similar vectors
+     * Now async due to worker-based VectorIndex implementation
      */
-    search(queryVector, k = 5, contentType) {
+    async search(queryVector, k = 5, contentType) {
         if (!this.isInitialized) {
             throw new Error('Index manager not initialized');
         }
@@ -499,7 +501,7 @@ export class IndexManager {
             // No specialized indexes (text-only mode) - ignore contentType and use combined index
             targetIndex = this.vectorIndex;
         }
-        const results = targetIndex.search(queryVector, k);
+        const results = await targetIndex.search(queryVector, k);
         // Convert numeric IDs back to embedding IDs
         const embeddingIds = results.neighbors.map(id => this.unhashEmbeddingId(id));
         return {
@@ -514,7 +516,7 @@ export class IndexManager {
         if (!this.db) {
             throw new Error('Database not initialized');
         }
-        const totalVectors = this.vectorIndex.getCurrentCount();
+        const totalVectors = await this.vectorIndex.getCurrentCount();
         try {
             const systemInfo = await getSystemInfo(this.db);
             const modelVersion = systemInfo?.modelVersion || null;
@@ -586,13 +588,80 @@ export class IndexManager {
         return embeddingId;
     }
     /**
-     * Close database connection
+     * Close database connection and cleanup vector index worker
      */
     async close() {
         if (this.db) {
             await this.db.close();
             this.db = null;
         }
+        // Clean up vector index worker to free WebAssembly memory
+        if (this.vectorIndex && typeof this.vectorIndex.cleanup === 'function') {
+            await this.vectorIndex.cleanup();
+        }
+        // Also clean up specialized indexes
+        if (this.textIndex && typeof this.textIndex.cleanup === 'function') {
+            await this.textIndex.cleanup();
+        }
+        if (this.imageIndex && typeof this.imageIndex.cleanup === 'function') {
+            await this.imageIndex.cleanup();
+        }
+    }
+    /**
+     * Reset the vector index by clearing all vectors while keeping the index structure.
+     * This is a safer alternative to file deletion that avoids file locking issues on Windows.
+     *
+     * The reset operation:
+     * 1. Clears in-memory HNSW index
+     * 2. Clears in-memory vector storage and ID mappings
+     * 3. Reinitializes an empty index with the same parameters
+     * 4. Saves the empty index to disk (overwrites existing file)
+     *
+     * @returns Promise that resolves when reset is complete
+     */
+    async reset() {
+        console.log('🔄 Starting index reset...');
+        const startTime = Date.now();
+        try {
+            // Clear in-memory mappings
+            const previousVectorCount = await this.vectorIndex.getCurrentCount();
+            this.hashToEmbeddingId.clear();
+            this.embeddingIdToHash.clear();
+            // Clear grouped embeddings if any
+            this.groupedEmbeddings = undefined;
+            // Clear specialized indexes if they exist
+            if (this.textIndex) {
+                this.textIndex = undefined;
+            }
+            if (this.imageIndex) {
+                this.imageIndex = undefined;
+            }
+            // Reset the vector index (clears all vectors and reinitializes empty HNSW graph)
+            console.log('  Resetting HNSW index...');
+            await this.vectorIndex.reset();
+            // Save the empty index to disk (this overwrites the existing file)
+            console.log('  Saving empty index to disk...');
+            await this.vectorIndex.saveIndex();
+            const resetTimeMs = Date.now() - startTime;
+            const currentCount = await this.vectorIndex.getCurrentCount();
+            console.log(`✓ Index reset complete in ${resetTimeMs}ms`);
+            console.log(`  Vectors cleared: ${previousVectorCount}`);
+            console.log(`  Current vector count: ${currentCount}`);
+        }
+        catch (error) {
+            const resetTimeMs = Date.now() - startTime;
+            console.error(`❌ Index reset failed after ${resetTimeMs}ms:`, error);
+            throw new Error(`Failed to reset index: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Check if the index has any vectors
+     * @returns true if the index contains vectors, false if empty
+     * Now async due to worker-based VectorIndex implementation
+     */
+    async hasVectors() {
+        const count = await this.vectorIndex.getCurrentCount();
+        return count > 0;
     }
 }
 //# sourceMappingURL=index-manager.js.map

package/dist/esm/index.d.ts

@@ -59,7 +59,8 @@ export { CrossEncoderReranker, createTextRerankFunction } from './text/reranker.
 export { countTokens } from './text/tokenizer.js';
 export type { RerankingStrategyType, RerankingConfig } from './core/reranking-config.js';
 export { validateRerankingStrategy, validateRerankingConfig, getDefaultRerankingConfig, isStrategySupported, getSupportedStrategies, RerankingConfigBuilder, DEFAULT_TEXT_RERANKING_CONFIG, DEFAULT_MULTIMODAL_RERANKING_CONFIG } from './core/reranking-config.js';
-export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, type DatabaseConnection } from './core/db.js';
+export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, resetDatabase, hasDatabaseData, type DatabaseConnection, type DatabaseResetOptions, type DatabaseResetResult } from './core/db.js';
+export { KnowledgeBaseManager, type KnowledgeBaseResetOptions, type KnowledgeBaseResetResult } from './core/knowledge-base-manager.js';
 export { IndexManager } from './index-manager.js';
 export { VectorIndex } from './core/vector-index.js';
 export { config, getModelDefaults, type CoreConfig, type ExtensibleConfig, type ModelDefaults, EXIT_CODES, ConfigurationError, getDefaultModelCachePath, handleUnrecoverableError, logError } from './core/config.js';
@@ -72,4 +73,13 @@ export type { Chunk, Preprocessor, PreprocessorOptions, PreprocessingConfig } fr
 export type { IngestionOptions, IngestionResult } from './core/ingestion.js';
 export { handleError, safeExecute, ErrorCategory, ErrorSeverity, createError, type ErrorContext } from './core/error-handler.js';
 export { APIError, IngestionError, SearchError, ResourceError, ModelCompatibilityError, ErrorFactory, CommonErrors, handleAPIError } from './api-errors.js';
+export { createResponseGenerator, createGenerateFunctionFromModel, getDefaultGeneratorModel, listGeneratorModels, isValidGeneratorModel, getRecommendedGenerator } from './factories/generator-factory.js';
+export type { ResponseGenerator, GeneratorModelType, GenerationRequest, GenerationResult, GeneratorCapabilities, GeneratorRequirements, GeneratorModelInfo, GeneratorValidationResult, GeneratorCreationOptions, GenerateFunction, CreateGeneratorFunction } from './core/response-generator.js';
+export { GeneratorValidationError, GenerationError, ContextWindowError } from './core/response-generator.js';
+export { supportsStreaming, isInstructModel, createGenerateFunction } from './core/response-generator.js';
+export { SUPPORTED_GENERATORS, DEFAULT_GENERATOR_MODEL, GeneratorRegistry, getGeneratorType, isInstructionTunedModel, getMaxContextLength, getRecommendedSettings, getDefaultMaxChunksForContext } from './core/generator-registry.js';
+export { DEFAULT_SYSTEM_PROMPT, formatContextChunks, buildPrompt, estimateTokenCount, type ContextFormattingOptions, type FormattedContext, type PromptBuildOptions, type BuiltPrompt } from './core/prompt-templates.js';
+export { InstructGenerator } from './text/generators/instruct-generator.js';
+export { CausalLMGenerator } from './text/generators/causal-lm-generator.js';
+export type { ExtendedSearchOptions, SearchResultWithGeneration } from './core/search.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/index.js

@@ -83,7 +83,9 @@ export { validateRerankingStrategy, validateRerankingConfig, getDefaultReranking
 // CORE INFRASTRUCTURE (FOR ADVANCED USERS)
 // =============================================================================
 // Database operations
-export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds } from './core/db.js';
+export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, resetDatabase, hasDatabaseData } from './core/db.js';
+// Knowledge Base Manager (for reset operations)
+export { KnowledgeBaseManager } from './core/knowledge-base-manager.js';
 // Vector index management
 export { IndexManager } from './index-manager.js';
 export { VectorIndex } from './core/vector-index.js';
@@ -107,4 +109,20 @@ export { resolveRagLitePaths, ensureRagLiteStructure, migrateToRagLiteStructure,
 export { handleError, safeExecute, ErrorCategory, ErrorSeverity, createError } from './core/error-handler.js';
 // API-specific errors
 export { APIError, IngestionError, SearchError, ResourceError, ModelCompatibilityError, ErrorFactory, CommonErrors, handleAPIError } from './api-errors.js';
+// =============================================================================
+// EXPERIMENTAL: RESPONSE GENERATION
+// =============================================================================
+// Generator factory functions
+export { createResponseGenerator, createGenerateFunctionFromModel, getDefaultGeneratorModel, listGeneratorModels, isValidGeneratorModel, getRecommendedGenerator } from './factories/generator-factory.js';
+// Generator errors
+export { GeneratorValidationError, GenerationError, ContextWindowError } from './core/response-generator.js';
+// Generator utilities
+export { supportsStreaming, isInstructModel, createGenerateFunction } from './core/response-generator.js';
+// Generator registry
+export { SUPPORTED_GENERATORS, DEFAULT_GENERATOR_MODEL, GeneratorRegistry, getGeneratorType, isInstructionTunedModel, getMaxContextLength, getRecommendedSettings, getDefaultMaxChunksForContext } from './core/generator-registry.js';
+// Prompt templates
+export { DEFAULT_SYSTEM_PROMPT, formatContextChunks, buildPrompt, estimateTokenCount } from './core/prompt-templates.js';
+// Generator implementations
+export { InstructGenerator } from './text/generators/instruct-generator.js';
+export { CausalLMGenerator } from './text/generators/causal-lm-generator.js';
 //# sourceMappingURL=index.js.map

package/dist/esm/text/generators/causal-lm-generator.d.ts

@@ -0,0 +1,65 @@
+/**
+ * TEXT IMPLEMENTATION — Causal LM Generator for DistilGPT2
+ *
+ * Implements ResponseGenerator interface for causal language models.
+ * Supports Xenova/distilgpt2 for fast, basic text generation.
+ *
+ * Features:
+ * - Simple prompt formatting (no chat template)
+ * - Fast generation with smaller model
+ * - Streaming generation support
+ * - Resource management via ResourceManager
+ *
+ * Note: Causal LM models don't support system prompts, so responses
+ * may be less focused than instruction-tuned models.
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import '../../dom-polyfills.js';
+import { BaseResponseGenerator, type GeneratorOptions } from '../../core/abstract-generator.js';
+/**
+ * Causal LM generator implementation for DistilGPT2
+ *
+ * Uses causal language models that generate text based on simple prompts.
+ * Faster but may produce less focused responses than instruct models.
+ */
+export declare class CausalLMGenerator extends BaseResponseGenerator {
+    private pipeline;
+    private tokenizer;
+    private resourceManager;
+    private resourceId?;
+    constructor(modelName: string, options?: GeneratorOptions);
+    /**
+     * Load the causal LM model using transformers.js
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Clean up model resources
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Generate text using the causal LM model
+     */
+    protected generateText(prompt: string, options: {
+        maxTokens: number;
+        temperature: number;
+        topP: number;
+        topK: number;
+        repetitionPenalty: number;
+        stopSequences: string[];
+    }): Promise<{
+        text: string;
+        promptTokens: number;
+        completionTokens: number;
+        finishReason: 'complete' | 'length' | 'stop_sequence' | 'error';
+    }>;
+    /**
+     * Generate text with streaming output
+     */
+    generateStream(request: import('../../core/response-generator.js').GenerationRequest): AsyncIterable<string>;
+    /**
+     * Count tokens in a text string
+     */
+    private countTokens;
+}
+//# sourceMappingURL=causal-lm-generator.d.ts.map