rag-lite-ts 2.1.1 → 2.3.0

package/dist/{core → cjs/core}/db.js

@@ -892,4 +892,109 @@ export async function updateStorageStats(connection, stats) {
         throw new Error(`Failed to update storage stats: ${error instanceof Error ? error.message : 'Unknown error'}`);
     }
 }
+/**
+ * Reset the database by deleting all data while keeping the schema intact.
+ * This is a safer alternative to file deletion that avoids file locking issues on Windows.
+ *
+ * This function:
+ * 1. Deletes all rows from chunks, documents, content_metadata tables
+ * 2. Optionally clears system_info (mode/model configuration)
+ * 3. Resets storage_stats counters
+ * 4. Optionally runs VACUUM to reclaim disk space
+ *
+ * @param connection - Database connection object
+ * @param options - Reset options
+ * @returns Promise resolving to reset result statistics
+ *
+ * @example
+ * ```typescript
+ * const db = await openDatabase('./db.sqlite');
+ * const result = await resetDatabase(db, { preserveSystemInfo: false });
+ * console.log(`Deleted ${result.documentsDeleted} documents and ${result.chunksDeleted} chunks`);
+ * ```
+ */
+export async function resetDatabase(connection, options = {}) {
+    const startTime = Date.now();
+    const { preserveSystemInfo = false, runVacuum = true } = options;
+    try {
+        console.log('🔄 Starting database reset...');
+        // Get counts before deletion for reporting
+        const docCountResult = await connection.get('SELECT COUNT(*) as count FROM documents');
+        const chunkCountResult = await connection.get('SELECT COUNT(*) as count FROM chunks');
+        const contentMetadataCountResult = await connection.get('SELECT COUNT(*) as count FROM content_metadata');
+        const documentsDeleted = docCountResult?.count || 0;
+        const chunksDeleted = chunkCountResult?.count || 0;
+        const contentMetadataDeleted = contentMetadataCountResult?.count || 0;
+        // Delete in order respecting foreign key constraints
+        // chunks → documents → content_metadata (chunks reference documents, documents reference content_metadata)
+        console.log('  Deleting chunks...');
+        await connection.run('DELETE FROM chunks');
+        console.log('  Deleting documents...');
+        await connection.run('DELETE FROM documents');
+        console.log('  Deleting content_metadata...');
+        await connection.run('DELETE FROM content_metadata');
+        // Reset storage_stats counters
+        console.log('  Resetting storage_stats...');
+        await connection.run(`
+      UPDATE storage_stats SET 
+        content_dir_files = 0,
+        content_dir_size = 0,
+        filesystem_refs = 0,
+        updated_at = CURRENT_TIMESTAMP
+      WHERE id = 1
+    `);
+        // Optionally clear system_info
+        let systemInfoCleared = false;
+        if (!preserveSystemInfo) {
+            console.log('  Clearing system_info...');
+            await connection.run('DELETE FROM system_info WHERE id = 1');
+            systemInfoCleared = true;
+        }
+        else {
+            console.log('  Preserving system_info (mode/model configuration)');
+        }
+        // Run VACUUM to reclaim disk space
+        if (runVacuum) {
+            console.log('  Running VACUUM to reclaim disk space...');
+            await connection.run('VACUUM');
+        }
+        const resetTimeMs = Date.now() - startTime;
+        console.log(`✓ Database reset complete in ${resetTimeMs}ms`);
+        console.log(`  Documents deleted: ${documentsDeleted}`);
+        console.log(`  Chunks deleted: ${chunksDeleted}`);
+        console.log(`  Content metadata deleted: ${contentMetadataDeleted}`);
+        console.log(`  System info cleared: ${systemInfoCleared}`);
+        return {
+            success: true,
+            documentsDeleted,
+            chunksDeleted,
+            contentMetadataDeleted,
+            systemInfoCleared,
+            resetTimeMs
+        };
+    }
+    catch (error) {
+        const resetTimeMs = Date.now() - startTime;
+        console.error(`❌ Database reset failed after ${resetTimeMs}ms:`, error);
+        throw new Error(`Failed to reset database: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Check if the database has any data (documents, chunks, or content)
+ * Useful for determining if a reset is needed
+ *
+ * @param connection - Database connection object
+ * @returns Promise resolving to true if database has data, false if empty
+ */
+export async function hasDatabaseData(connection) {
+    try {
+        const docCount = await connection.get('SELECT COUNT(*) as count FROM documents');
+        const chunkCount = await connection.get('SELECT COUNT(*) as count FROM chunks');
+        return (docCount?.count || 0) > 0 || (chunkCount?.count || 0) > 0;
+    }
+    catch (error) {
+        // If tables don't exist, consider it empty
+        return false;
+    }
+}
 //# sourceMappingURL=db.js.map

package/dist/{core → cjs/core}/ingestion.js

@@ -340,6 +340,9 @@ export class IngestionPipeline {
             // Phase 5: Vector Index Updates
             console.log('\n--- Phase 5: Vector Index Updates ---');
             await this.updateVectorIndex(embeddingResult.embeddings);
+            // Final save to ensure all vectors are persisted
+            console.log('Performing final index save...');
+            await this.indexManager.saveIndex();
             const endTime = Date.now();
             const processingTimeMs = endTime - startTime;
             const result = {

package/dist/cjs/core/knowledge-base-manager.d.ts

@@ -0,0 +1,109 @@
+/**
+ * Knowledge Base Manager
+ *
+ * Provides a unified API for managing the knowledge base (database + vector index).
+ * This module is designed to solve file locking issues on Windows by using
+ * in-place reset operations instead of file deletion.
+ *
+ * Key Features:
+ * - Reset database and index without file deletion (avoids EBUSY/EACCES errors)
+ * - Coordinated reset of both database and index in a single operation
+ * - Connection management to prevent orphaned handles
+ * - Cross-platform compatibility (especially Windows)
+ *
+ * @module knowledge-base-manager
+ */
+import { type DatabaseResetResult } from './db.js';
+/**
+ * Result of a knowledge base reset operation
+ */
+export interface KnowledgeBaseResetResult {
+    /** Whether the overall reset was successful */
+    success: boolean;
+    /** Database reset result */
+    database: DatabaseResetResult;
+    /** Index reset statistics */
+    index: {
+        /** Number of vectors cleared */
+        vectorsCleared: number;
+        /** Time taken for index reset in milliseconds */
+        resetTimeMs: number;
+    };
+    /** Total time for the complete reset operation */
+    totalTimeMs: number;
+    /** Any warnings that occurred during reset */
+    warnings: string[];
+}
+/**
+ * Options for knowledge base reset operation
+ */
+export interface KnowledgeBaseResetOptions {
+    /** Whether to preserve system_info (mode, model configuration) - default: false */
+    preserveSystemInfo?: boolean;
+    /** Whether to run VACUUM after database reset - default: true */
+    runVacuum?: boolean;
+    /** Model name to use for index recreation - default: from config */
+    modelName?: string;
+}
+/**
+ * Knowledge Base Manager
+ *
+ * Manages the complete knowledge base lifecycle including database and vector index.
+ * Provides safe reset operations that avoid file locking issues on Windows.
+ *
+ * @example
+ * ```typescript
+ * // Reset knowledge base for force rebuild
+ * const result = await KnowledgeBaseManager.reset('./db.sqlite', './index.bin');
+ * console.log(`Reset ${result.database.documentsDeleted} documents and ${result.index.vectorsCleared} vectors`);
+ *
+ * // Reset with options
+ * const result = await KnowledgeBaseManager.reset('./db.sqlite', './index.bin', {
+ *   preserveSystemInfo: true,  // Keep mode/model configuration
+ *   modelName: 'all-MiniLM-L6-v2'  // Specify model for index
+ * });
+ * ```
+ */
+export declare class KnowledgeBaseManager {
+    /**
+     * Reset the knowledge base by clearing all data while keeping files intact.
+     * This is a safer alternative to file deletion that avoids file locking issues on Windows.
+     *
+     * The reset operation:
+     * 1. Closes any existing connections via DatabaseConnectionManager
+     * 2. Opens a fresh connection to the database
+     * 3. Deletes all rows from documents, chunks, content_metadata tables
+     * 4. Optionally runs VACUUM to reclaim disk space
+     * 5. Reinitializes the vector index (clears all vectors)
+     * 6. Saves the empty index to disk (overwrites existing file content)
+     *
+     * This approach works because:
+     * - We don't delete files, so no EBUSY/EACCES errors
+     * - The same file handles can be reused or replaced safely
+     * - SQLite transactions ensure data integrity
+     * - Index overwrite uses standard file write operations
+     *
+     * @param dbPath - Path to the SQLite database file
+     * @param indexPath - Path to the vector index file
+     * @param options - Reset options
+     * @returns Promise resolving to reset result statistics
+     *
+     * @throws Error if database or index reset fails
+     */
+    static reset(dbPath: string, indexPath: string, options?: KnowledgeBaseResetOptions): Promise<KnowledgeBaseResetResult>;
+    /**
+     * Check if the knowledge base has any data
+     *
+     * @param dbPath - Path to the SQLite database file
+     * @returns Promise resolving to true if database has data, false if empty
+     */
+    static hasData(dbPath: string): Promise<boolean>;
+    /**
+     * Close all connections to the knowledge base
+     * Useful before operations that might conflict with open handles
+     *
+     * @param dbPath - Path to the SQLite database file
+     */
+    static closeAllConnections(dbPath: string): Promise<void>;
+}
+//# sourceMappingURL=knowledge-base-manager.d.ts.map

package/dist/cjs/core/knowledge-base-manager.js

@@ -0,0 +1,256 @@
+/**
+ * Knowledge Base Manager
+ *
+ * Provides a unified API for managing the knowledge base (database + vector index).
+ * This module is designed to solve file locking issues on Windows by using
+ * in-place reset operations instead of file deletion.
+ *
+ * Key Features:
+ * - Reset database and index without file deletion (avoids EBUSY/EACCES errors)
+ * - Coordinated reset of both database and index in a single operation
+ * - Connection management to prevent orphaned handles
+ * - Cross-platform compatibility (especially Windows)
+ *
+ * @module knowledge-base-manager
+ */
+import { openDatabase, resetDatabase, hasDatabaseData } from './db.js';
+import { IndexManager } from '../index-manager.js';
+import { DatabaseConnectionManager } from './database-connection-manager.js';
+import { getModelDefaults, config } from './config.js';
+import { existsSync } from 'fs';
+/**
+ * Knowledge Base Manager
+ *
+ * Manages the complete knowledge base lifecycle including database and vector index.
+ * Provides safe reset operations that avoid file locking issues on Windows.
+ *
+ * @example
+ * ```typescript
+ * // Reset knowledge base for force rebuild
+ * const result = await KnowledgeBaseManager.reset('./db.sqlite', './index.bin');
+ * console.log(`Reset ${result.database.documentsDeleted} documents and ${result.index.vectorsCleared} vectors`);
+ *
+ * // Reset with options
+ * const result = await KnowledgeBaseManager.reset('./db.sqlite', './index.bin', {
+ *   preserveSystemInfo: true,  // Keep mode/model configuration
+ *   modelName: 'all-MiniLM-L6-v2'  // Specify model for index
+ * });
+ * ```
+ */
+export class KnowledgeBaseManager {
+    /**
+     * Reset the knowledge base by clearing all data while keeping files intact.
+     * This is a safer alternative to file deletion that avoids file locking issues on Windows.
+     *
+     * The reset operation:
+     * 1. Closes any existing connections via DatabaseConnectionManager
+     * 2. Opens a fresh connection to the database
+     * 3. Deletes all rows from documents, chunks, content_metadata tables
+     * 4. Optionally runs VACUUM to reclaim disk space
+     * 5. Reinitializes the vector index (clears all vectors)
+     * 6. Saves the empty index to disk (overwrites existing file content)
+     *
+     * This approach works because:
+     * - We don't delete files, so no EBUSY/EACCES errors
+     * - The same file handles can be reused or replaced safely
+     * - SQLite transactions ensure data integrity
+     * - Index overwrite uses standard file write operations
+     *
+     * @param dbPath - Path to the SQLite database file
+     * @param indexPath - Path to the vector index file
+     * @param options - Reset options
+     * @returns Promise resolving to reset result statistics
+     *
+     * @throws Error if database or index reset fails
+     */
+    static async reset(dbPath, indexPath, options = {}) {
+        const startTime = Date.now();
+        const warnings = [];
+        console.log('🔄 Starting knowledge base reset...');
+        console.log(`  Database: ${dbPath}`);
+        console.log(`  Index: ${indexPath}`);
+        // Step 1: Close any existing managed connections to prevent conflicts
+        console.log('\n📡 Step 1: Closing existing connections...');
+        try {
+            if (DatabaseConnectionManager.hasConnection(dbPath)) {
+                await DatabaseConnectionManager.forceCloseConnection(dbPath);
+                console.log('  ✓ Closed existing database connection');
+            }
+            else {
+                console.log('  ✓ No existing connection to close');
+            }
+        }
+        catch (error) {
+            const warning = `Warning: Error closing existing connection: ${error instanceof Error ? error.message : 'Unknown error'}`;
+            warnings.push(warning);
+            console.warn(`  ⚠️ ${warning}`);
+        }
+        // Small delay to ensure handles are fully released
+        await new Promise(resolve => setTimeout(resolve, 50));
+        // Step 2: Reset the database
+        console.log('\n💾 Step 2: Resetting database...');
+        let db = null;
+        let dbResetResult;
+        try {
+            // Open a fresh connection
+            db = await openDatabase(dbPath);
+            // Perform the reset
+            dbResetResult = await resetDatabase(db, {
+                preserveSystemInfo: options.preserveSystemInfo,
+                runVacuum: options.runVacuum
+            });
+            console.log('  ✓ Database reset complete');
+        }
+        catch (error) {
+            console.error('  ❌ Database reset failed:', error);
+            throw new Error(`Failed to reset database: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+        finally {
+            // Close the database connection
+            if (db) {
+                try {
+                    await db.close();
+                }
+                catch (closeError) {
+                    warnings.push(`Warning: Error closing database after reset: ${closeError}`);
+                }
+            }
+        }
+        // Step 3: Reset the vector index
+        console.log('\n📇 Step 3: Resetting vector index...');
+        let indexResetResult;
+        const indexStartTime = Date.now();
+        try {
+            // Determine model and dimensions
+            const modelName = options.modelName || config.embedding_model;
+            const modelDefaults = getModelDefaults(modelName);
+            // Check if index file exists
+            if (!existsSync(indexPath)) {
+                console.log('  Index file does not exist, will be created during ingestion');
+                indexResetResult = {
+                    vectorsCleared: 0,
+                    resetTimeMs: Date.now() - indexStartTime
+                };
+            }
+            else {
+                // Create IndexManager and reset
+                // We need to handle dimension mismatch gracefully since the user might be
+                // switching models (e.g., from MPNet 768D to MiniLM 384D)
+                const indexManager = new IndexManager(indexPath, dbPath, modelDefaults.dimensions, modelName);
+                let previousVectorCount = 0;
+                try {
+                    // Try to initialize with forceRecreate=false first to get the vector count
+                    // skipModelCheck=true since we're resetting anyway
+                    await indexManager.initialize(true, false);
+                    // Get current vector count before reset
+                    previousVectorCount = (await indexManager.hasVectors()) ?
+                        (await indexManager.getStats()).totalVectors : 0;
+                    // Perform the reset
+                    await indexManager.reset();
+                }
+                catch (initError) {
+                    // If initialization failed (e.g., dimension mismatch), force recreate the index
+                    // This handles the case where user is switching models
+                    const errorMessage = initError?.message || String(initError);
+                    if (errorMessage.includes('dimension mismatch') || errorMessage.includes('Vector dimension')) {
+                        console.log('  ⚠️ Dimension mismatch detected - forcing index recreation');
+                        console.log('  (This is expected when switching embedding models)');
+                        // Create a fresh IndexManager and force recreate
+                        const freshIndexManager = new IndexManager(indexPath, dbPath, modelDefaults.dimensions, modelName);
+                        await freshIndexManager.initialize(true, true); // skipModelCheck=true, forceRecreate=true
+                        await freshIndexManager.saveIndex();
+                        await freshIndexManager.close();
+                        // We don't know the previous count since we couldn't load the old index
+                        // But we can estimate it was non-zero since the file existed
+                        previousVectorCount = -1; // Indicate unknown
+                    }
+                    else {
+                        // Re-throw other errors
+                        throw initError;
+                    }
+                }
+                // Close the index manager
+                await indexManager.close();
+                indexResetResult = {
+                    vectorsCleared: previousVectorCount,
+                    resetTimeMs: Date.now() - indexStartTime
+                };
+                console.log('  ✓ Index reset complete');
+            }
+        }
+        catch (error) {
+            console.error('  ❌ Index reset failed:', error);
+            throw new Error(`Failed to reset index: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+        const totalTimeMs = Date.now() - startTime;
+        // Summary
+        console.log('\n✅ Knowledge base reset complete!');
+        console.log(`  Total time: ${totalTimeMs}ms`);
+        console.log(`  Documents deleted: ${dbResetResult.documentsDeleted}`);
+        console.log(`  Chunks deleted: ${dbResetResult.chunksDeleted}`);
+        console.log(`  Vectors cleared: ${indexResetResult.vectorsCleared === -1 ? '(unknown - index recreated due to model change)' : indexResetResult.vectorsCleared}`);
+        if (warnings.length > 0) {
+            console.log(`  Warnings: ${warnings.length}`);
+        }
+        return {
+            success: true,
+            database: dbResetResult,
+            index: indexResetResult,
+            totalTimeMs,
+            warnings
+        };
+    }
+    /**
+     * Check if the knowledge base has any data
+     *
+     * @param dbPath - Path to the SQLite database file
+     * @returns Promise resolving to true if database has data, false if empty
+     */
+    static async hasData(dbPath) {
+        let db = null;
+        try {
+            db = await openDatabase(dbPath);
+            return await hasDatabaseData(db);
+        }
+        catch (error) {
+            // If we can't open the database, assume no data
+            return false;
+        }
+        finally {
+            if (db) {
+                try {
+                    await db.close();
+                }
+                catch {
+                    // Ignore close errors
+                }
+            }
+        }
+    }
+    /**
+     * Close all connections to the knowledge base
+     * Useful before operations that might conflict with open handles
+     *
+     * @param dbPath - Path to the SQLite database file
+     */
+    static async closeAllConnections(dbPath) {
+        console.log('🔒 Closing all knowledge base connections...');
+        try {
+            if (DatabaseConnectionManager.hasConnection(dbPath)) {
+                await DatabaseConnectionManager.forceCloseConnection(dbPath);
+            }
+            // Also close WAL/SHM connections if they exist
+            const sidecars = [`${dbPath}-wal`, `${dbPath}-shm`];
+            for (const sidecar of sidecars) {
+                if (DatabaseConnectionManager.hasConnection(sidecar)) {
+                    await DatabaseConnectionManager.forceCloseConnection(sidecar);
+                }
+            }
+            console.log('✓ All connections closed');
+        }
+        catch (error) {
+            console.warn('⚠️ Error closing connections:', error);
+        }
+    }
+}
+//# sourceMappingURL=knowledge-base-manager.js.map

package/dist/{core → cjs/core}/model-validator.js

@@ -105,7 +105,7 @@ export class ModelValidator {
             }
             // Fallback: try to detect from package.json import
             try {
-                const packageInfo = await import('@huggingface/transformers/package.json');
+                const packageInfo = await import('@huggingface/transformers/package.json' + '');
                 if (packageInfo.version) {
                     this.currentTransformersVersion = packageInfo.version;
                     return packageInfo.version;

package/dist/{core → cjs/core}/search-pipeline.js

@@ -123,7 +123,7 @@ export class SearchPipelineCoordinator {
             throw new Error('Index manager not set. Set index manager before executing pipeline.');
         }
         try {
-            return this.indexManager.search(queryVector, topK);
+            return await this.indexManager.search(queryVector, topK);
         }
         catch (error) {
             if (error instanceof Error && error.message.includes('No embedding ID found for hash')) {

package/dist/{core → cjs/core}/search.js

@@ -140,7 +140,7 @@ export class SearchEngine {
             let searchResult;
             try {
                 const contentType = options.contentType;
-                searchResult = this.indexManager.search(queryVector, topK, contentType);
+                searchResult = await this.indexManager.search(queryVector, topK, contentType);
             }
             catch (error) {
                 if (error instanceof Error && error.message.includes('No embedding ID found for hash')) {

package/dist/cjs/core/vector-index-messages.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Message protocol for VectorIndex worker thread communication
+ */
+export interface VectorIndexRequest {
+    id: number;
+    type: 'init' | 'loadIndex' | 'saveIndex' | 'addVector' | 'addVectors' | 'search' | 'getCurrentCount' | 'resizeIndex' | 'reset' | 'cleanup' | 'indexExists' | 'setEf';
+    payload?: any;
+}
+export interface VectorIndexResponse {
+    id: number;
+    type: 'success' | 'error';
+    payload?: any;
+    error?: string;
+}
+export interface InitPayload {
+    dimensions: number;
+    maxElements: number;
+    M?: number;
+    efConstruction?: number;
+    seed?: number;
+    indexPath: string;
+}
+export interface LoadIndexPayload {
+    indexPath: string;
+}
+export interface AddVectorPayload {
+    id: number;
+    vector: ArrayBuffer;
+    dimensions: number;
+}
+export interface AddVectorsPayload {
+    vectors: Array<{
+        id: number;
+        vector: ArrayBuffer;
+        dimensions: number;
+    }>;
+}
+export interface SearchPayload {
+    queryVector: ArrayBuffer;
+    dimensions: number;
+    k: number;
+}
+export interface ResizeIndexPayload {
+    newMaxElements: number;
+}
+export interface SetEfPayload {
+    ef: number;
+}
+export interface IndexExistsPayload {
+    indexPath: string;
+}
+//# sourceMappingURL=vector-index-messages.d.ts.map

package/dist/cjs/core/vector-index-messages.js

@@ -0,0 +1,5 @@
+/**
+ * Message protocol for VectorIndex worker thread communication
+ */
+export {};
+//# sourceMappingURL=vector-index-messages.js.map

package/dist/cjs/core/vector-index-worker.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Worker thread for VectorIndex operations
+ * Isolates hnswlib-wasm WebAssembly memory to prevent accumulation
+ */
+export {};
+//# sourceMappingURL=vector-index-worker.d.ts.map