rag-lite-ts 2.2.0 → 2.3.0

package/dist/esm/index-manager.js

@@ -64,7 +64,7 @@ export class IndexManager {
                 this.hashEmbeddingId(chunk.embedding_id); // This will populate the mapping
             }
             this.isInitialized = true;
-            const vectorCount = this.vectorIndex.getCurrentCount();
+            const vectorCount = await this.vectorIndex.getCurrentCount();
             console.log(`Index manager initialized with ${vectorCount} vectors${this.textIndex && this.imageIndex ? ' (multi-graph mode)' : ''}`);
         }
         catch (error) {
@@ -145,17 +145,18 @@ export class IndexManager {
                 vector: embedding.vector
             }));
             // Check if we need to resize the index before adding
-            const currentCount = this.vectorIndex.getCurrentCount();
-            const newCount = currentCount + vectors.length;
+            const initialCount = await this.vectorIndex.getCurrentCount();
+            const newCount = initialCount + vectors.length;
             const currentCapacity = 100000; // This should match the initial capacity
             if (newCount > currentCapacity * 0.9) {
                 const newCapacity = Math.ceil(newCount * 1.5);
                 console.log(`Resizing index from ${currentCapacity} to ${newCapacity} to accommodate new vectors`);
-                this.vectorIndex.resizeIndex(newCapacity);
+                await this.vectorIndex.resizeIndex(newCapacity);
             }
             // Add vectors incrementally (this is the key requirement - no rebuild needed)
-            this.vectorIndex.addVectors(vectors);
-            console.log(`Incrementally added ${embeddings.length} vectors to index (total: ${this.vectorIndex.getCurrentCount()})`);
+            await this.vectorIndex.addVectors(vectors);
+            const finalCount = await this.vectorIndex.getCurrentCount();
+            console.log(`Incrementally added ${embeddings.length} vectors to index (total: ${finalCount})`);
             // Save the updated index
             await this.saveIndex();
         }
@@ -223,7 +224,7 @@ export class IndexManager {
             const currentCapacity = 100000; // Default capacity
             if (chunkData.length > currentCapacity * 0.8) {
                 const newCapacity = Math.ceil(chunkData.length * 1.5);
-                this.vectorIndex.resizeIndex(newCapacity);
+                await this.vectorIndex.resizeIndex(newCapacity);
                 console.log(`Resized index capacity to ${newCapacity} for ${chunkData.length} chunks`);
             }
             // Update model version if provided
@@ -279,7 +280,7 @@ export class IndexManager {
             const currentCapacity = 100000;
             if (chunkData.length > currentCapacity * 0.8) {
                 const newCapacity = Math.ceil(chunkData.length * 1.5);
-                this.vectorIndex.resizeIndex(newCapacity);
+                await this.vectorIndex.resizeIndex(newCapacity);
                 console.log(`Resized index capacity to ${newCapacity}`);
             }
             // Re-generate embeddings for all chunks
@@ -294,7 +295,7 @@ export class IndexManager {
                 id: this.hashEmbeddingId(embedding.embedding_id),
                 vector: embedding.vector
             }));
-            this.vectorIndex.addVectors(vectors);
+            await this.vectorIndex.addVectors(vectors);
             console.log(`Added ${vectors.length} vectors to rebuilt index`);
             // Update model version
             await this.updateModelVersion(embeddingEngine.getModelVersion());
@@ -414,12 +415,12 @@ export class IndexManager {
                     // Create text-only index
                     this.textIndex = new VectorIndex(`${this.indexPath}.text`, this.vectorIndexOptions);
                     await this.textIndex.initialize();
-                    this.textIndex.addVectors(indexData.textVectors);
+                    await this.textIndex.addVectors(indexData.textVectors);
                     console.log(`✓ Text index created with ${indexData.textVectors.length} vectors`);
                     // Create image-only index
                     this.imageIndex = new VectorIndex(`${this.indexPath}.image`, this.vectorIndexOptions);
                     await this.imageIndex.initialize();
-                    this.imageIndex.addVectors(indexData.imageVectors);
+                    await this.imageIndex.addVectors(indexData.imageVectors);
                     console.log(`✓ Image index created with ${indexData.imageVectors.length} vectors`);
                     console.log('✓ Specialized indexes ready for content type filtering');
                 }
@@ -475,8 +476,9 @@ export class IndexManager {
     }
     /**
      * Search for similar vectors
+     * Now async due to worker-based VectorIndex implementation
      */
-    search(queryVector, k = 5, contentType) {
+    async search(queryVector, k = 5, contentType) {
         if (!this.isInitialized) {
             throw new Error('Index manager not initialized');
         }
@@ -499,7 +501,7 @@ export class IndexManager {
             // No specialized indexes (text-only mode) - ignore contentType and use combined index
             targetIndex = this.vectorIndex;
         }
-        const results = targetIndex.search(queryVector, k);
+        const results = await targetIndex.search(queryVector, k);
         // Convert numeric IDs back to embedding IDs
         const embeddingIds = results.neighbors.map(id => this.unhashEmbeddingId(id));
         return {
@@ -514,7 +516,7 @@ export class IndexManager {
         if (!this.db) {
             throw new Error('Database not initialized');
         }
-        const totalVectors = this.vectorIndex.getCurrentCount();
+        const totalVectors = await this.vectorIndex.getCurrentCount();
         try {
             const systemInfo = await getSystemInfo(this.db);
             const modelVersion = systemInfo?.modelVersion || null;
@@ -586,13 +588,80 @@ export class IndexManager {
         return embeddingId;
     }
     /**
-     * Close database connection
+     * Close database connection and cleanup vector index worker
      */
     async close() {
         if (this.db) {
             await this.db.close();
             this.db = null;
         }
+        // Clean up vector index worker to free WebAssembly memory
+        if (this.vectorIndex && typeof this.vectorIndex.cleanup === 'function') {
+            await this.vectorIndex.cleanup();
+        }
+        // Also clean up specialized indexes
+        if (this.textIndex && typeof this.textIndex.cleanup === 'function') {
+            await this.textIndex.cleanup();
+        }
+        if (this.imageIndex && typeof this.imageIndex.cleanup === 'function') {
+            await this.imageIndex.cleanup();
+        }
+    }
+    /**
+     * Reset the vector index by clearing all vectors while keeping the index structure.
+     * This is a safer alternative to file deletion that avoids file locking issues on Windows.
+     *
+     * The reset operation:
+     * 1. Clears in-memory HNSW index
+     * 2. Clears in-memory vector storage and ID mappings
+     * 3. Reinitializes an empty index with the same parameters
+     * 4. Saves the empty index to disk (overwrites existing file)
+     *
+     * @returns Promise that resolves when reset is complete
+     */
+    async reset() {
+        console.log('🔄 Starting index reset...');
+        const startTime = Date.now();
+        try {
+            // Clear in-memory mappings
+            const previousVectorCount = await this.vectorIndex.getCurrentCount();
+            this.hashToEmbeddingId.clear();
+            this.embeddingIdToHash.clear();
+            // Clear grouped embeddings if any
+            this.groupedEmbeddings = undefined;
+            // Clear specialized indexes if they exist
+            if (this.textIndex) {
+                this.textIndex = undefined;
+            }
+            if (this.imageIndex) {
+                this.imageIndex = undefined;
+            }
+            // Reset the vector index (clears all vectors and reinitializes empty HNSW graph)
+            console.log('  Resetting HNSW index...');
+            await this.vectorIndex.reset();
+            // Save the empty index to disk (this overwrites the existing file)
+            console.log('  Saving empty index to disk...');
+            await this.vectorIndex.saveIndex();
+            const resetTimeMs = Date.now() - startTime;
+            const currentCount = await this.vectorIndex.getCurrentCount();
+            console.log(`✓ Index reset complete in ${resetTimeMs}ms`);
+            console.log(`  Vectors cleared: ${previousVectorCount}`);
+            console.log(`  Current vector count: ${currentCount}`);
+        }
+        catch (error) {
+            const resetTimeMs = Date.now() - startTime;
+            console.error(`❌ Index reset failed after ${resetTimeMs}ms:`, error);
+            throw new Error(`Failed to reset index: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    }
+    /**
+     * Check if the index has any vectors
+     * @returns true if the index contains vectors, false if empty
+     * Now async due to worker-based VectorIndex implementation
+     */
+    async hasVectors() {
+        const count = await this.vectorIndex.getCurrentCount();
+        return count > 0;
     }
 }
 //# sourceMappingURL=index-manager.js.map

package/dist/esm/index.d.ts

@@ -59,7 +59,8 @@ export { CrossEncoderReranker, createTextRerankFunction } from './text/reranker.
 export { countTokens } from './text/tokenizer.js';
 export type { RerankingStrategyType, RerankingConfig } from './core/reranking-config.js';
 export { validateRerankingStrategy, validateRerankingConfig, getDefaultRerankingConfig, isStrategySupported, getSupportedStrategies, RerankingConfigBuilder, DEFAULT_TEXT_RERANKING_CONFIG, DEFAULT_MULTIMODAL_RERANKING_CONFIG } from './core/reranking-config.js';
-export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, type DatabaseConnection } from './core/db.js';
+export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, resetDatabase, hasDatabaseData, type DatabaseConnection, type DatabaseResetOptions, type DatabaseResetResult } from './core/db.js';
+export { KnowledgeBaseManager, type KnowledgeBaseResetOptions, type KnowledgeBaseResetResult } from './core/knowledge-base-manager.js';
 export { IndexManager } from './index-manager.js';
 export { VectorIndex } from './core/vector-index.js';
 export { config, getModelDefaults, type CoreConfig, type ExtensibleConfig, type ModelDefaults, EXIT_CODES, ConfigurationError, getDefaultModelCachePath, handleUnrecoverableError, logError } from './core/config.js';

package/dist/esm/index.js

@@ -83,7 +83,9 @@ export { validateRerankingStrategy, validateRerankingConfig, getDefaultReranking
 // CORE INFRASTRUCTURE (FOR ADVANCED USERS)
 // =============================================================================
 // Database operations
-export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds } from './core/db.js';
+export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, resetDatabase, hasDatabaseData } from './core/db.js';
+// Knowledge Base Manager (for reset operations)
+export { KnowledgeBaseManager } from './core/knowledge-base-manager.js';
 // Vector index management
 export { IndexManager } from './index-manager.js';
 export { VectorIndex } from './core/vector-index.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rag-lite-ts",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "description": "Local-first TypeScript retrieval engine with Chameleon Multimodal Architecture for semantic search over text and image content",
   "type": "module",
   "main": "./dist/cjs/index.js",
@@ -11,6 +11,7 @@
   },
   "exports": {
     ".": {
+      "types": "./dist/esm/index.d.ts",
       "import": {
         "types": "./dist/esm/index.d.ts",
         "default": "./dist/esm/index.js"
@@ -18,10 +19,10 @@
       "require": {
         "types": "./dist/cjs/index.d.ts",
         "default": "./dist/cjs/index.js"
-      },
-      "types": "./dist/esm/index.d.ts"
+      }
     },
     "./mcp": {
+      "types": "./dist/esm/mcp-server.d.ts",
       "import": {
         "types": "./dist/esm/mcp-server.d.ts",
         "default": "./dist/esm/mcp-server.js"
@@ -29,8 +30,7 @@
       "require": {
         "types": "./dist/cjs/mcp-server.d.ts",
         "default": "./dist/cjs/mcp-server.js"
-      },
-      "types": "./dist/esm/mcp-server.d.ts"
+      }
     }
   },
   "files": [
@@ -39,6 +39,10 @@
     "dist/esm/**/*.d.ts",
     "dist/cjs/**/*.d.ts",
     "!dist/**/*.map",
+    "ui/backend/dist/**",
+    "ui/frontend/dist/**",
+    "!ui/**/src/**",
+    "!ui/**/node_modules/**",
     "README.md",
     "LICENSE"
   ],
@@ -47,7 +51,10 @@
     "build:esm": "tsc -p tsconfig.esm.json",
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "build:test": "tsc --project tsconfig.test.json",
-    "clean": "rimraf dist",
+    "build:ui:backend": "cd ui/backend && npm run build",
+    "build:ui:frontend": "cd ui/frontend && npm run build",
+    "build:ui": "npm run build:ui:backend && npm run build:ui:frontend",
+    "clean": "rimraf dist ui/backend/dist ui/frontend/dist",
     "dev": "tsc --watch",
     "test": "npm run build:test && node --expose-gc --test --test-concurrency=1 dist/__tests__/core dist/__tests__/text dist/__tests__/preprocessors dist/__tests__/cli dist/__tests__/factories",
     "test:verbose": "npm run build:test && node --expose-gc --test --test-concurrency=1 --test-reporter=tap dist/__tests__/core dist/__tests__/text dist/__tests__/preprocessors dist/__tests__/cli dist/__tests__/factories",
@@ -63,7 +70,7 @@
     "test:integration:verbose": "npm run build && npm run build:test && node --expose-gc --test --test-concurrency=1 --test-reporter=tap dist/__tests__/integration",
     "test:all": "npm run build:test && node --expose-gc --test --test-concurrency=1 dist/__tests__",
     "test:all:verbose": "npm run build:test && node --expose-gc --test --test-concurrency=1 --test-reporter=tap dist/__tests__",
-    "prepublishOnly": "npm run clean && npm run build"
+    "prepublishOnly": "npm run clean && npm run build && npm run build:ui"
   },
   "keywords": [
     "rag",