rag-lite-ts 2.2.0 → 2.3.1

package/dist/cjs/core/knowledge-base-manager.js

@@ -0,0 +1,256 @@
+/**
+ * Knowledge Base Manager
+ *
+ * Provides a unified API for managing the knowledge base (database + vector index).
+ * This module is designed to solve file locking issues on Windows by using
+ * in-place reset operations instead of file deletion.
+ *
+ * Key Features:
+ * - Reset database and index without file deletion (avoids EBUSY/EACCES errors)
+ * - Coordinated reset of both database and index in a single operation
+ * - Connection management to prevent orphaned handles
+ * - Cross-platform compatibility (especially Windows)
+ *
+ * @module knowledge-base-manager
+ */
+import { openDatabase, resetDatabase, hasDatabaseData } from './db.js';
+import { IndexManager } from '../index-manager.js';
+import { DatabaseConnectionManager } from './database-connection-manager.js';
+import { getModelDefaults, config } from './config.js';
+import { existsSync } from 'fs';
+/**
+ * Knowledge Base Manager
+ *
+ * Manages the complete knowledge base lifecycle including database and vector index.
+ * Provides safe reset operations that avoid file locking issues on Windows.
+ *
+ * @example
+ * ```typescript
+ * // Reset knowledge base for force rebuild
+ * const result = await KnowledgeBaseManager.reset('./db.sqlite', './index.bin');
+ * console.log(`Reset ${result.database.documentsDeleted} documents and ${result.index.vectorsCleared} vectors`);
+ *
+ * // Reset with options
+ * const result = await KnowledgeBaseManager.reset('./db.sqlite', './index.bin', {
+ *   preserveSystemInfo: true,  // Keep mode/model configuration
+ *   modelName: 'all-MiniLM-L6-v2'  // Specify model for index
+ * });
+ * ```
+ */
+export class KnowledgeBaseManager {
+    /**
+     * Reset the knowledge base by clearing all data while keeping files intact.
+     * This is a safer alternative to file deletion that avoids file locking issues on Windows.
+     *
+     * The reset operation:
+     * 1. Closes any existing connections via DatabaseConnectionManager
+     * 2. Opens a fresh connection to the database
+     * 3. Deletes all rows from documents, chunks, content_metadata tables
+     * 4. Optionally runs VACUUM to reclaim disk space
+     * 5. Reinitializes the vector index (clears all vectors)
+     * 6. Saves the empty index to disk (overwrites existing file content)
+     *
+     * This approach works because:
+     * - We don't delete files, so no EBUSY/EACCES errors
+     * - The same file handles can be reused or replaced safely
+     * - SQLite transactions ensure data integrity
+     * - Index overwrite uses standard file write operations
+     *
+     * @param dbPath - Path to the SQLite database file
+     * @param indexPath - Path to the vector index file
+     * @param options - Reset options
+     * @returns Promise resolving to reset result statistics
+     *
+     * @throws Error if database or index reset fails
+     */
+    static async reset(dbPath, indexPath, options = {}) {
+        const startTime = Date.now();
+        const warnings = [];
+        console.log('🔄 Starting knowledge base reset...');
+        console.log(`  Database: ${dbPath}`);
+        console.log(`  Index: ${indexPath}`);
+        // Step 1: Close any existing managed connections to prevent conflicts
+        console.log('\n📡 Step 1: Closing existing connections...');
+        try {
+            if (DatabaseConnectionManager.hasConnection(dbPath)) {
+                await DatabaseConnectionManager.forceCloseConnection(dbPath);
+                console.log('  ✓ Closed existing database connection');
+            }
+            else {
+                console.log('  ✓ No existing connection to close');
+            }
+        }
+        catch (error) {
+            const warning = `Warning: Error closing existing connection: ${error instanceof Error ? error.message : 'Unknown error'}`;
+            warnings.push(warning);
+            console.warn(`  ⚠️ ${warning}`);
+        }
+        // Small delay to ensure handles are fully released
+        await new Promise(resolve => setTimeout(resolve, 50));
+        // Step 2: Reset the database
+        console.log('\n💾 Step 2: Resetting database...');
+        let db = null;
+        let dbResetResult;
+        try {
+            // Open a fresh connection
+            db = await openDatabase(dbPath);
+            // Perform the reset
+            dbResetResult = await resetDatabase(db, {
+                preserveSystemInfo: options.preserveSystemInfo,
+                runVacuum: options.runVacuum
+            });
+            console.log('  ✓ Database reset complete');
+        }
+        catch (error) {
+            console.error('  ❌ Database reset failed:', error);
+            throw new Error(`Failed to reset database: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+        finally {
+            // Close the database connection
+            if (db) {
+                try {
+                    await db.close();
+                }
+                catch (closeError) {
+                    warnings.push(`Warning: Error closing database after reset: ${closeError}`);
+                }
+            }
+        }
+        // Step 3: Reset the vector index
+        console.log('\n📇 Step 3: Resetting vector index...');
+        let indexResetResult;
+        const indexStartTime = Date.now();
+        try {
+            // Determine model and dimensions
+            const modelName = options.modelName || config.embedding_model;
+            const modelDefaults = getModelDefaults(modelName);
+            // Check if index file exists
+            if (!existsSync(indexPath)) {
+                console.log('  Index file does not exist, will be created during ingestion');
+                indexResetResult = {
+                    vectorsCleared: 0,
+                    resetTimeMs: Date.now() - indexStartTime
+                };
+            }
+            else {
+                // Create IndexManager and reset
+                // We need to handle dimension mismatch gracefully since the user might be
+                // switching models (e.g., from MPNet 768D to MiniLM 384D)
+                const indexManager = new IndexManager(indexPath, dbPath, modelDefaults.dimensions, modelName);
+                let previousVectorCount = 0;
+                try {
+                    // Try to initialize with forceRecreate=false first to get the vector count
+                    // skipModelCheck=true since we're resetting anyway
+                    await indexManager.initialize(true, false);
+                    // Get current vector count before reset
+                    previousVectorCount = (await indexManager.hasVectors()) ?
+                        (await indexManager.getStats()).totalVectors : 0;
+                    // Perform the reset
+                    await indexManager.reset();
+                }
+                catch (initError) {
+                    // If initialization failed (e.g., dimension mismatch), force recreate the index
+                    // This handles the case where user is switching models
+                    const errorMessage = initError?.message || String(initError);
+                    if (errorMessage.includes('dimension mismatch') || errorMessage.includes('Vector dimension')) {
+                        console.log('  ⚠️ Dimension mismatch detected - forcing index recreation');
+                        console.log('  (This is expected when switching embedding models)');
+                        // Create a fresh IndexManager and force recreate
+                        const freshIndexManager = new IndexManager(indexPath, dbPath, modelDefaults.dimensions, modelName);
+                        await freshIndexManager.initialize(true, true); // skipModelCheck=true, forceRecreate=true
+                        await freshIndexManager.saveIndex();
+                        await freshIndexManager.close();
+                        // We don't know the previous count since we couldn't load the old index
+                        // But we can estimate it was non-zero since the file existed
+                        previousVectorCount = -1; // Indicate unknown
+                    }
+                    else {
+                        // Re-throw other errors
+                        throw initError;
+                    }
+                }
+                // Close the index manager
+                await indexManager.close();
+                indexResetResult = {
+                    vectorsCleared: previousVectorCount,
+                    resetTimeMs: Date.now() - indexStartTime
+                };
+                console.log('  ✓ Index reset complete');
+            }
+        }
+        catch (error) {
+            console.error('  ❌ Index reset failed:', error);
+            throw new Error(`Failed to reset index: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+        const totalTimeMs = Date.now() - startTime;
+        // Summary
+        console.log('\n✅ Knowledge base reset complete!');
+        console.log(`  Total time: ${totalTimeMs}ms`);
+        console.log(`  Documents deleted: ${dbResetResult.documentsDeleted}`);
+        console.log(`  Chunks deleted: ${dbResetResult.chunksDeleted}`);
+        console.log(`  Vectors cleared: ${indexResetResult.vectorsCleared === -1 ? '(unknown - index recreated due to model change)' : indexResetResult.vectorsCleared}`);
+        if (warnings.length > 0) {
+            console.log(`  Warnings: ${warnings.length}`);
+        }
+        return {
+            success: true,
+            database: dbResetResult,
+            index: indexResetResult,
+            totalTimeMs,
+            warnings
+        };
+    }
+    /**
+     * Check if the knowledge base has any data
+     *
+     * @param dbPath - Path to the SQLite database file
+     * @returns Promise resolving to true if database has data, false if empty
+     */
+    static async hasData(dbPath) {
+        let db = null;
+        try {
+            db = await openDatabase(dbPath);
+            return await hasDatabaseData(db);
+        }
+        catch (error) {
+            // If we can't open the database, assume no data
+            return false;
+        }
+        finally {
+            if (db) {
+                try {
+                    await db.close();
+                }
+                catch {
+                    // Ignore close errors
+                }
+            }
+        }
+    }
+    /**
+     * Close all connections to the knowledge base
+     * Useful before operations that might conflict with open handles
+     *
+     * @param dbPath - Path to the SQLite database file
+     */
+    static async closeAllConnections(dbPath) {
+        console.log('🔒 Closing all knowledge base connections...');
+        try {
+            if (DatabaseConnectionManager.hasConnection(dbPath)) {
+                await DatabaseConnectionManager.forceCloseConnection(dbPath);
+            }
+            // Also close WAL/SHM connections if they exist
+            const sidecars = [`${dbPath}-wal`, `${dbPath}-shm`];
+            for (const sidecar of sidecars) {
+                if (DatabaseConnectionManager.hasConnection(sidecar)) {
+                    await DatabaseConnectionManager.forceCloseConnection(sidecar);
+                }
+            }
+            console.log('✓ All connections closed');
+        }
+        catch (error) {
+            console.warn('⚠️ Error closing connections:', error);
+        }
+    }
+}
+//# sourceMappingURL=knowledge-base-manager.js.map

package/dist/cjs/core/lazy-dependency-loader.d.ts

@@ -8,6 +8,7 @@
 import '../dom-polyfills.js';
 import type { UniversalEmbedder } from './universal-embedder.js';
 import type { RerankFunction } from './interfaces.js';
+import type { ResponseGenerator } from './response-generator.js';
 /**
  * Lazy loader for embedder implementations
  * Only loads the specific embedder type when needed
@@ -42,6 +43,42 @@ export declare class LazyEmbedderLoader {
         multimodalEmbedders: number;
     };
 }
+/**
+ * Lazy loader for response generator implementations
+ * Only loads the specific generator type when needed
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export declare class LazyGeneratorLoader {
+    private static cache;
+    /**
+     * Lazily load and create an instruct generator (SmolLM2-Instruct)
+     * Only imports the module when generation is actually requested
+     */
+    static loadInstructGenerator(modelName: string, options?: any): Promise<ResponseGenerator>;
+    /**
+     * Lazily load and create a causal LM generator (DistilGPT2)
+     * Only imports the module when generation is actually requested
+     */
+    static loadCausalLMGenerator(modelName: string, options?: any): Promise<ResponseGenerator>;
+    /**
+     * Check if a generator is already loaded in cache
+     */
+    static isGeneratorLoaded(modelName: string, modelType: 'instruct' | 'causal-lm'): boolean;
+    /**
+     * Remove a generator from the cache (called when generator is cleaned up)
+     */
+    static removeGeneratorFromCache(modelName: string, modelType: 'instruct' | 'causal-lm'): void;
+    /**
+     * Get statistics about loaded generators
+     */
+    static getLoadingStats(): {
+        loadedGenerators: string[];
+        totalLoaded: number;
+        instructGenerators: number;
+        causalLMGenerators: number;
+    };
+}
 /**
  * Lazy loader for reranking implementations
  * Only loads the specific reranker type when needed
@@ -107,6 +144,11 @@ export declare class LazyMultimodalLoader {
  * Provides a single entry point for dependency management
  */
 export declare class LazyDependencyManager {
+    /**
+     * Load response generator based on model type with lazy loading
+     * @experimental This feature is experimental and may change in future versions.
+     */
+    static loadGenerator(modelName: string, modelType: 'instruct' | 'causal-lm', options?: any): Promise<ResponseGenerator>;
     /**
      * Load embedder based on model type with lazy loading
      */
@@ -121,6 +163,7 @@ export declare class LazyDependencyManager {
     static getLoadingStatistics(): {
         embedders: ReturnType<typeof LazyEmbedderLoader.getLoadingStats>;
         rerankers: ReturnType<typeof LazyRerankerLoader.getLoadingStats>;
+        generators: ReturnType<typeof LazyGeneratorLoader.getLoadingStats>;
         multimodal: ReturnType<typeof LazyMultimodalLoader.getMultimodalLoadingStatus>;
         totalModulesLoaded: number;
         memoryImpact: 'low' | 'medium' | 'high';

package/dist/cjs/core/lazy-dependency-loader.js

@@ -149,6 +149,99 @@ export class LazyEmbedderLoader {
     }
 }
 // =============================================================================
+// LAZY GENERATOR LOADING
+// =============================================================================
+/**
+ * Lazy loader for response generator implementations
+ * Only loads the specific generator type when needed
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+export class LazyGeneratorLoader {
+    static cache = LazyLoadingCache.getInstance();
+    /**
+     * Lazily load and create an instruct generator (SmolLM2-Instruct)
+     * Only imports the module when generation is actually requested
+     */
+    static async loadInstructGenerator(modelName, options = {}) {
+        const cacheKey = `generator:instruct:${modelName}`;
+        return this.cache.getOrLoad(cacheKey, async () => {
+            try {
+                console.log(`🔄 [EXPERIMENTAL] Lazy loading instruct generator: ${modelName}`);
+                // Dynamic import - only loaded when generation is requested
+                const { InstructGenerator } = await import('../text/generators/instruct-generator.js');
+                const generator = new InstructGenerator(modelName, options);
+                await generator.loadModel();
+                console.log(`✅ Instruct generator loaded: ${modelName}`);
+                return generator;
+            }
+            catch (error) {
+                const enhancedError = createError.model(`Failed to lazy load instruct generator '${modelName}': ${error instanceof Error ? error.message : 'Unknown error'}`);
+                handleError(enhancedError, 'LazyGeneratorLoader', {
+                    severity: ErrorSeverity.ERROR,
+                    category: ErrorCategory.MODEL
+                });
+                throw enhancedError;
+            }
+        });
+    }
+    /**
+     * Lazily load and create a causal LM generator (DistilGPT2)
+     * Only imports the module when generation is actually requested
+     */
+    static async loadCausalLMGenerator(modelName, options = {}) {
+        const cacheKey = `generator:causal-lm:${modelName}`;
+        return this.cache.getOrLoad(cacheKey, async () => {
+            try {
+                console.log(`🔄 [EXPERIMENTAL] Lazy loading causal LM generator: ${modelName}`);
+                // Dynamic import - only loaded when generation is requested
+                const { CausalLMGenerator } = await import('../text/generators/causal-lm-generator.js');
+                const generator = new CausalLMGenerator(modelName, options);
+                await generator.loadModel();
+                console.log(`✅ Causal LM generator loaded: ${modelName}`);
+                return generator;
+            }
+            catch (error) {
+                const enhancedError = createError.model(`Failed to lazy load causal LM generator '${modelName}': ${error instanceof Error ? error.message : 'Unknown error'}`);
+                handleError(enhancedError, 'LazyGeneratorLoader', {
+                    severity: ErrorSeverity.ERROR,
+                    category: ErrorCategory.MODEL
+                });
+                throw enhancedError;
+            }
+        });
+    }
+    /**
+     * Check if a generator is already loaded in cache
+     */
+    static isGeneratorLoaded(modelName, modelType) {
+        const cacheKey = `generator:${modelType}:${modelName}`;
+        return this.cache.getLoadedModules().includes(cacheKey);
+    }
+    /**
+     * Remove a generator from the cache (called when generator is cleaned up)
+     */
+    static removeGeneratorFromCache(modelName, modelType) {
+        const cacheKey = `generator:${modelType}:${modelName}`;
+        this.cache.remove(cacheKey);
+        console.log(`🧹 Removed generator from cache: ${cacheKey}`);
+    }
+    /**
+     * Get statistics about loaded generators
+     */
+    static getLoadingStats() {
+        const loadedModules = this.cache.getLoadedModules().filter(key => key.startsWith('generator:'));
+        const instructGenerators = loadedModules.filter(key => key.includes(':instruct:')).length;
+        const causalLMGenerators = loadedModules.filter(key => key.includes(':causal-lm:')).length;
+        return {
+            loadedGenerators: loadedModules,
+            totalLoaded: loadedModules.length,
+            instructGenerators,
+            causalLMGenerators
+        };
+    }
+}
+// =============================================================================
 // LAZY RERANKER LOADING
 // =============================================================================
 /**
@@ -332,6 +425,20 @@ export class LazyMultimodalLoader {
  * Provides a single entry point for dependency management
  */
 export class LazyDependencyManager {
+    /**
+     * Load response generator based on model type with lazy loading
+     * @experimental This feature is experimental and may change in future versions.
+     */
+    static async loadGenerator(modelName, modelType, options = {}) {
+        switch (modelType) {
+            case 'instruct':
+                return LazyGeneratorLoader.loadInstructGenerator(modelName, options);
+            case 'causal-lm':
+                return LazyGeneratorLoader.loadCausalLMGenerator(modelName, options);
+            default:
+                throw createError.validation(`Unsupported generator model type for lazy loading: ${modelType}`);
+        }
+    }
     /**
      * Load embedder based on model type with lazy loading
      */
@@ -367,19 +474,21 @@ export class LazyDependencyManager {
     static getLoadingStatistics() {
         const embedderStats = LazyEmbedderLoader.getLoadingStats();
         const rerankerStats = LazyRerankerLoader.getLoadingStats();
+        const generatorStats = LazyGeneratorLoader.getLoadingStats();
         const multimodalStats = LazyMultimodalLoader.getMultimodalLoadingStatus();
-        const totalModules = embedderStats.totalLoaded + rerankerStats.totalLoaded + multimodalStats.loadedProcessors.length;
+        const totalModules = embedderStats.totalLoaded + rerankerStats.totalLoaded + generatorStats.totalLoaded + multimodalStats.loadedProcessors.length;
         // Estimate memory impact based on loaded modules
         let memoryImpact = 'low';
         if (embedderStats.multimodalEmbedders > 0 || multimodalStats.imageToTextLoaded) {
             memoryImpact = 'high';
         }
-        else if (totalModules > 2) {
+        else if (totalModules > 2 || generatorStats.totalLoaded > 0) {
             memoryImpact = 'medium';
         }
         return {
             embedders: embedderStats,
             rerankers: rerankerStats,
+            generators: generatorStats,
             multimodal: multimodalStats,
             totalModulesLoaded: totalModules,
             memoryImpact

package/dist/cjs/core/prompt-templates.d.ts

@@ -0,0 +1,138 @@
+/**
+ * CORE MODULE — Prompt Templates for RAG Response Generation
+ *
+ * Provides prompt engineering utilities for different generator model types.
+ * Handles context formatting, token budget management, and system prompts.
+ *
+ * PROMPT STRATEGIES:
+ * - Instruct models: Use chat template with system/user/assistant roles
+ * - Causal LM models: Use simple document + question format
+ *
+ * @experimental This feature is experimental and may change in future versions.
+ */
+import type { SearchResult } from './types.js';
+import type { GeneratorModelType } from './response-generator.js';
+/**
+ * Default system prompt for instruct models
+ * Emphasizes grounded responses using only provided context
+ */
+export declare const DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant that answers questions based ONLY on the provided context documents. Follow these rules strictly:\n\n1. Answer ONLY using information found in the context documents\n2. If the answer cannot be found in the context, say \"I cannot find this information in the provided documents\"\n3. Do not make up information or use external knowledge\n4. Be concise and direct in your response\n5. If the context is incomplete or unclear, acknowledge this limitation";
+/**
+ * Default system prompt for RAG with source attribution
+ */
+export declare const DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION = "You are a helpful assistant that answers questions based ONLY on the provided context documents. Follow these rules strictly:\n\n1. Answer ONLY using information found in the context documents\n2. When possible, mention which document the information comes from\n3. If the answer cannot be found in the context, say \"I cannot find this information in the provided documents\"\n4. Do not make up information or use external knowledge\n5. Be concise and direct in your response";
+/**
+ * SmolLM2 chat template format
+ * Uses <|im_start|> and <|im_end|> tokens
+ */
+export declare const SMOLLM2_CHAT_TEMPLATE: {
+    systemStart: string;
+    systemEnd: string;
+    userStart: string;
+    userEnd: string;
+    assistantStart: string;
+    assistantEnd: string;
+    endOfText: string;
+};
+/**
+ * Options for formatting context chunks
+ */
+export interface ContextFormattingOptions {
+    /** Maximum tokens available for context */
+    maxContextTokens: number;
+    /** Include document titles/sources */
+    includeDocumentInfo?: boolean;
+    /** Include relevance scores */
+    includeScores?: boolean;
+    /** Separator between chunks */
+    chunkSeparator?: string;
+    /** Token estimation function (chars to tokens ratio) */
+    tokenEstimationRatio?: number;
+}
+/**
+ * Result of context formatting
+ */
+export interface FormattedContext {
+    /** Formatted context string */
+    text: string;
+    /** Estimated token count */
+    estimatedTokens: number;
+    /** Number of chunks included */
+    chunksIncluded: number;
+    /** Total chunks available */
+    totalChunks: number;
+    /** Whether context was truncated */
+    truncated: boolean;
+}
+/**
+ * Format search result chunks into context string for the prompt
+ *
+ * @param chunks - Search result chunks to format
+ * @param options - Formatting options
+ * @returns Formatted context with metadata
+ */
+export declare function formatContextChunks(chunks: SearchResult[], options: ContextFormattingOptions): FormattedContext;
+/**
+ * Options for building the complete prompt
+ */
+export interface PromptBuildOptions {
+    /** User's query */
+    query: string;
+    /** Search result chunks */
+    chunks: SearchResult[];
+    /** Generator model type */
+    modelType: GeneratorModelType;
+    /** Custom system prompt (optional) */
+    systemPrompt?: string;
+    /** Maximum context window tokens */
+    maxContextLength: number;
+    /** Tokens reserved for output */
+    reservedOutputTokens: number;
+    /** Include source attribution hint */
+    includeSourceAttribution?: boolean;
+}
+/**
+ * Result of prompt building
+ */
+export interface BuiltPrompt {
+    /** Complete prompt string */
+    prompt: string;
+    /** Estimated total tokens */
+    estimatedTokens: number;
+    /** Context metadata */
+    contextInfo: FormattedContext;
+    /** System prompt used (if any) */
+    systemPromptUsed?: string;
+}
+/**
+ * Build a complete prompt for the generator model
+ *
+ * @param options - Prompt building options
+ * @returns Built prompt with metadata
+ */
+export declare function buildPrompt(options: PromptBuildOptions): BuiltPrompt;
+/**
+ * Estimate token count for a string
+ * Uses a simple character-based heuristic (~4 chars per token for English)
+ *
+ * @param text - Text to estimate tokens for
+ * @returns Estimated token count
+ */
+export declare function estimateTokenCount(text: string): number;
+/**
+ * Calculate available context budget
+ *
+ * @param maxContextLength - Maximum context window size
+ * @param reservedOutputTokens - Tokens reserved for generation
+ * @param promptOverhead - Tokens used by prompt formatting
+ * @returns Available tokens for context chunks
+ */
+export declare function calculateContextBudget(maxContextLength: number, reservedOutputTokens: number, promptOverhead?: number): number;
+/**
+ * Get default stop sequences for a model type
+ *
+ * @param modelType - Generator model type
+ * @returns Array of stop sequences
+ */
+export declare function getDefaultStopSequences(modelType: GeneratorModelType): string[];
+//# sourceMappingURL=prompt-templates.d.ts.map