rag-lite-ts 2.2.0 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +88 -5
  2. package/dist/cjs/cli/indexer.js +73 -15
  3. package/dist/cjs/cli/search.js +77 -2
  4. package/dist/cjs/cli/ui-server.d.ts +5 -0
  5. package/dist/cjs/cli/ui-server.js +152 -0
  6. package/dist/cjs/cli.js +53 -7
  7. package/dist/cjs/core/abstract-generator.d.ts +97 -0
  8. package/dist/cjs/core/abstract-generator.js +222 -0
  9. package/dist/cjs/core/binary-index-format.js +53 -10
  10. package/dist/cjs/core/db.d.ts +56 -0
  11. package/dist/cjs/core/db.js +105 -0
  12. package/dist/cjs/core/generator-registry.d.ts +114 -0
  13. package/dist/cjs/core/generator-registry.js +280 -0
  14. package/dist/cjs/core/index.d.ts +4 -0
  15. package/dist/cjs/core/index.js +11 -0
  16. package/dist/cjs/core/ingestion.js +3 -0
  17. package/dist/cjs/core/knowledge-base-manager.d.ts +109 -0
  18. package/dist/cjs/core/knowledge-base-manager.js +256 -0
  19. package/dist/cjs/core/lazy-dependency-loader.d.ts +43 -0
  20. package/dist/cjs/core/lazy-dependency-loader.js +111 -2
  21. package/dist/cjs/core/prompt-templates.d.ts +138 -0
  22. package/dist/cjs/core/prompt-templates.js +225 -0
  23. package/dist/cjs/core/response-generator.d.ts +132 -0
  24. package/dist/cjs/core/response-generator.js +69 -0
  25. package/dist/cjs/core/search-pipeline.js +1 -1
  26. package/dist/cjs/core/search.d.ts +72 -1
  27. package/dist/cjs/core/search.js +80 -7
  28. package/dist/cjs/core/types.d.ts +1 -0
  29. package/dist/cjs/core/vector-index-messages.d.ts +52 -0
  30. package/dist/cjs/core/vector-index-messages.js +5 -0
  31. package/dist/cjs/core/vector-index-worker.d.ts +6 -0
  32. package/dist/cjs/core/vector-index-worker.js +314 -0
  33. package/dist/cjs/core/vector-index.d.ts +45 -10
  34. package/dist/cjs/core/vector-index.js +279 -218
  35. package/dist/cjs/factories/generator-factory.d.ts +88 -0
  36. package/dist/cjs/factories/generator-factory.js +151 -0
  37. package/dist/cjs/factories/index.d.ts +1 -0
  38. package/dist/cjs/factories/index.js +5 -0
  39. package/dist/cjs/factories/ingestion-factory.js +3 -7
  40. package/dist/cjs/factories/search-factory.js +11 -0
  41. package/dist/cjs/index-manager.d.ts +23 -3
  42. package/dist/cjs/index-manager.js +84 -15
  43. package/dist/cjs/index.d.ts +11 -1
  44. package/dist/cjs/index.js +19 -1
  45. package/dist/cjs/text/generators/causal-lm-generator.d.ts +65 -0
  46. package/dist/cjs/text/generators/causal-lm-generator.js +197 -0
  47. package/dist/cjs/text/generators/index.d.ts +10 -0
  48. package/dist/cjs/text/generators/index.js +10 -0
  49. package/dist/cjs/text/generators/instruct-generator.d.ts +62 -0
  50. package/dist/cjs/text/generators/instruct-generator.js +192 -0
  51. package/dist/esm/cli/indexer.js +73 -15
  52. package/dist/esm/cli/search.js +77 -2
  53. package/dist/esm/cli/ui-server.d.ts +5 -0
  54. package/dist/esm/cli/ui-server.js +152 -0
  55. package/dist/esm/cli.js +53 -7
  56. package/dist/esm/core/abstract-generator.d.ts +97 -0
  57. package/dist/esm/core/abstract-generator.js +222 -0
  58. package/dist/esm/core/binary-index-format.js +53 -10
  59. package/dist/esm/core/db.d.ts +56 -0
  60. package/dist/esm/core/db.js +105 -0
  61. package/dist/esm/core/generator-registry.d.ts +114 -0
  62. package/dist/esm/core/generator-registry.js +280 -0
  63. package/dist/esm/core/index.d.ts +4 -0
  64. package/dist/esm/core/index.js +11 -0
  65. package/dist/esm/core/ingestion.js +3 -0
  66. package/dist/esm/core/knowledge-base-manager.d.ts +109 -0
  67. package/dist/esm/core/knowledge-base-manager.js +256 -0
  68. package/dist/esm/core/lazy-dependency-loader.d.ts +43 -0
  69. package/dist/esm/core/lazy-dependency-loader.js +111 -2
  70. package/dist/esm/core/prompt-templates.d.ts +138 -0
  71. package/dist/esm/core/prompt-templates.js +225 -0
  72. package/dist/esm/core/response-generator.d.ts +132 -0
  73. package/dist/esm/core/response-generator.js +69 -0
  74. package/dist/esm/core/search-pipeline.js +1 -1
  75. package/dist/esm/core/search.d.ts +72 -1
  76. package/dist/esm/core/search.js +80 -7
  77. package/dist/esm/core/types.d.ts +1 -0
  78. package/dist/esm/core/vector-index-messages.d.ts +52 -0
  79. package/dist/esm/core/vector-index-messages.js +5 -0
  80. package/dist/esm/core/vector-index-worker.d.ts +6 -0
  81. package/dist/esm/core/vector-index-worker.js +314 -0
  82. package/dist/esm/core/vector-index.d.ts +45 -10
  83. package/dist/esm/core/vector-index.js +279 -218
  84. package/dist/esm/factories/generator-factory.d.ts +88 -0
  85. package/dist/esm/factories/generator-factory.js +151 -0
  86. package/dist/esm/factories/index.d.ts +1 -0
  87. package/dist/esm/factories/index.js +5 -0
  88. package/dist/esm/factories/ingestion-factory.js +3 -7
  89. package/dist/esm/factories/search-factory.js +11 -0
  90. package/dist/esm/index-manager.d.ts +23 -3
  91. package/dist/esm/index-manager.js +84 -15
  92. package/dist/esm/index.d.ts +11 -1
  93. package/dist/esm/index.js +19 -1
  94. package/dist/esm/text/generators/causal-lm-generator.d.ts +65 -0
  95. package/dist/esm/text/generators/causal-lm-generator.js +197 -0
  96. package/dist/esm/text/generators/index.d.ts +10 -0
  97. package/dist/esm/text/generators/index.js +10 -0
  98. package/dist/esm/text/generators/instruct-generator.d.ts +62 -0
  99. package/dist/esm/text/generators/instruct-generator.js +192 -0
  100. package/package.json +14 -7
package/dist/cjs/core/vector-index.js CHANGED
@@ -1,39 +1,22 @@
1
1
  /**
2
2
  * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
3
3
  * Model-agnostic. No transformer or modality-specific logic.
4
+ *
5
+ * Worker-based implementation to prevent WebAssembly memory accumulation.
4
6
  */
7
+ import { Worker } from 'worker_threads';
5
8
  import { existsSync } from 'fs';
6
- import { JSDOM } from 'jsdom';
7
- import { ErrorCategory, ErrorSeverity, safeExecute } from './error-handler.js';
9
+ import { fileURLToPath } from 'url';
10
+ import { dirname, join, resolve } from 'path';
11
+ import { handleError, ErrorCategory, ErrorSeverity, createError } from './error-handler.js';
8
12
  import { createMissingFileError, createDimensionMismatchError } from './actionable-error-messages.js';
9
- import { BinaryIndexFormat } from './binary-index-format.js';
10
- // Set up browser-like environment for hnswlib-wasm
11
- if (typeof window === 'undefined') {
12
- const dom = new JSDOM('<!DOCTYPE html><html><body></body></html>', {
13
- url: 'http://localhost',
14
- pretendToBeVisual: true,
15
- resources: 'usable'
16
- });
17
- // Type assertion to avoid TypeScript issues with global polyfills
18
- global.window = dom.window;
19
- global.document = dom.window.document;
20
- global.XMLHttpRequest = dom.window.XMLHttpRequest;
21
- // Disable IndexedDB to prevent hnswlib-wasm from trying to use it
22
- global.indexedDB = undefined;
23
- // Override indexedDB on the window object to return undefined
24
- Object.defineProperty(dom.window, 'indexedDB', {
25
- value: undefined,
26
- writable: false,
27
- configurable: true
28
- });
29
- }
30
13
  export class VectorIndex {
31
- index = null;
32
- hnswlib = null;
14
+ worker = null;
33
15
  indexPath;
34
16
  options;
35
- currentSize = 0;
36
- vectorStorage = new Map(); // For persistence
17
+ messageQueue = new Map();
18
+ messageId = 0;
19
+ isInitialized = false;
37
20
  constructor(indexPath, options) {
38
21
  this.indexPath = indexPath;
39
22
  this.options = {
@@ -44,62 +27,159 @@ export class VectorIndex {
44
27
  };
45
28
  }
46
29
  /**
47
- * Initialize the HNSW index with cosine similarity using hnswlib-wasm
30
+ * Get the path to the worker script
31
+ * Always uses compiled .js files - workers cannot execute TypeScript directly
48
32
  */
49
- async initialize() {
50
- await safeExecute(async () => {
51
- // Load the hnswlib module
52
- if (!this.hnswlib) {
53
- // Temporarily suppress stderr output during hnswlib loading to avoid IndexedDB warnings
54
- const originalStderrWrite = process.stderr.write;
55
- const originalConsoleError = console.error;
56
- process.stderr.write = function (chunk, encoding, callback) {
57
- const message = chunk.toString();
58
- // Suppress specific IndexedDB/IDBFS related errors and WebAssembly errors
59
- if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
60
- message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
61
- message.includes('jsFS Error') || message.includes('syncing FS') ||
62
- message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
63
- message.includes('abort') || message.includes('assert') ||
64
- message.includes('hnswlib-wasm/dist/hnswlib')) {
65
- if (callback)
66
- callback();
67
- return true;
68
- }
69
- return originalStderrWrite.call(this, chunk, encoding, callback);
70
- };
71
- console.error = (...args) => {
72
- const message = args.join(' ');
73
- if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
74
- message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
75
- message.includes('jsFS Error') || message.includes('syncing FS') ||
76
- message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
77
- message.includes('abort') || message.includes('assert') ||
78
- message.includes('hnswlib-wasm/dist/hnswlib')) {
79
- return;
80
- }
81
- originalConsoleError.apply(console, args);
82
- };
83
- try {
84
- const hnswlibModule = await import('hnswlib-wasm/dist/hnswlib.js');
85
- const { loadHnswlib } = hnswlibModule;
86
- this.hnswlib = await loadHnswlib();
33
+ getWorkerPath() {
34
+ const currentFile = fileURLToPath(import.meta.url);
35
+ const currentDir = dirname(currentFile);
36
+ // Always prefer .js (compiled output)
37
+ const jsPath = resolve(join(currentDir, 'vector-index-worker.js'));
38
+ // Check if .js exists in current directory (compiled)
39
+ if (existsSync(jsPath)) {
40
+ return jsPath;
41
+ }
42
+ // Helper function to find project root
43
+ const findProjectRoot = () => {
44
+ let dir = currentDir;
45
+ // Look for dist/ in the path
46
+ const distMatch = dir.match(/^(.+?)[\\/]dist[\\/]/);
47
+ if (distMatch) {
48
+ return distMatch[1];
49
+ }
50
+ // If no dist/, try going up from src/core
51
+ const srcMatch = dir.match(/^(.+?)[\\/]src[\\/]core/);
52
+ if (srcMatch) {
53
+ return srcMatch[1];
54
+ }
55
+ // If in node_modules, extract package root
56
+ const nodeModulesMatch = dir.match(/^(.+?)[\\/]node_modules/);
57
+ if (nodeModulesMatch) {
58
+ return nodeModulesMatch[1];
59
+ }
60
+ return null;
61
+ };
62
+ const projectRoot = findProjectRoot();
63
+ if (projectRoot) {
64
+ // Try ESM first (preferred for ES modules)
65
+ const distEsmPath = resolve(join(projectRoot, 'dist', 'esm', 'core', 'vector-index-worker.js'));
66
+ if (existsSync(distEsmPath)) {
67
+ return distEsmPath;
68
+ }
69
+ // Try CJS as fallback
70
+ const distCjsPath = resolve(join(projectRoot, 'dist', 'cjs', 'core', 'vector-index-worker.js'));
71
+ if (existsSync(distCjsPath)) {
72
+ return distCjsPath;
73
+ }
74
+ }
75
+ // If running from node_modules (installed package), try dist paths
76
+ if (currentDir.includes('node_modules')) {
77
+ const packageRoot = currentDir.split('node_modules')[0];
78
+ const distEsmPath = resolve(join(packageRoot, 'node_modules', 'rag-lite-ts', 'dist', 'esm', 'core', 'vector-index-worker.js'));
79
+ const distCjsPath = resolve(join(packageRoot, 'node_modules', 'rag-lite-ts', 'dist', 'cjs', 'core', 'vector-index-worker.js'));
80
+ if (existsSync(distEsmPath)) {
81
+ return distEsmPath;
82
+ }
83
+ if (existsSync(distCjsPath)) {
84
+ return distCjsPath;
85
+ }
86
+ }
87
+ // Final fallback - will fail with clear error
88
+ throw new Error(`Worker file not found. Expected: ${jsPath}\n` +
89
+ 'Please run "npm run build" to compile the vector-index-worker.ts file.\n' +
90
+ `Current directory: ${currentDir}\n` +
91
+ `Project root: ${projectRoot || 'not found'}\n` +
92
+ `Checked paths: ${jsPath}${projectRoot ? `, ${join(projectRoot, 'dist', 'esm', 'core', 'vector-index-worker.js')}, ${join(projectRoot, 'dist', 'cjs', 'core', 'vector-index-worker.js')}` : ''}`);
93
+ }
94
+ /**
95
+ * Ensure worker is created and ready
96
+ */
97
+ async ensureWorker() {
98
+ if (this.worker) {
99
+ return;
100
+ }
101
+ const workerPath = this.getWorkerPath();
102
+ this.worker = new Worker(workerPath);
103
+ // Set up message handler
104
+ this.worker.on('message', (response) => {
105
+ const handler = this.messageQueue.get(response.id);
106
+ if (handler) {
107
+ this.messageQueue.delete(response.id);
108
+ if (response.type === 'error') {
109
+ handler.reject(new Error(response.error || 'Unknown error'));
87
110
  }
88
- finally {
89
- // Restore original output streams
90
- process.stderr.write = originalStderrWrite;
91
- console.error = originalConsoleError;
111
+ else {
112
+ handler.resolve(response.payload);
92
113
  }
93
114
  }
94
- // Create new HNSW index (third parameter is autoSaveFilename, but we'll handle persistence manually)
95
- this.index = new this.hnswlib.HierarchicalNSW('cosine', this.options.dimensions, '');
96
- this.index.initIndex(this.options.maxElements, this.options.M || 16, this.options.efConstruction || 200, this.options.seed || 100);
97
- this.currentSize = 0;
98
- console.log(`Initialized HNSW index with ${this.options.dimensions} dimensions using hnswlib-wasm`);
99
- }, 'Vector Index Initialization', {
100
- category: ErrorCategory.INDEX,
101
- severity: ErrorSeverity.FATAL
102
115
  });
116
+ // Handle worker errors
117
+ this.worker.on('error', (error) => {
118
+ console.error('VectorIndex worker error:', error);
119
+ // Reject all pending requests
120
+ for (const [id, handler] of this.messageQueue.entries()) {
121
+ handler.reject(error);
122
+ }
123
+ this.messageQueue.clear();
124
+ });
125
+ // Handle worker exit
126
+ this.worker.on('exit', (code) => {
127
+ if (code !== 0) {
128
+ console.error(`VectorIndex worker exited with code ${code}`);
129
+ }
130
+ // Reject all pending requests
131
+ for (const [id, handler] of this.messageQueue.entries()) {
132
+ handler.reject(new Error(`Worker exited with code ${code}`));
133
+ }
134
+ this.messageQueue.clear();
135
+ this.worker = null;
136
+ this.isInitialized = false;
137
+ });
138
+ }
139
+ /**
140
+ * Send a message to the worker and wait for response
141
+ */
142
+ async sendMessage(type, payload) {
143
+ await this.ensureWorker();
144
+ return new Promise((resolve, reject) => {
145
+ const id = this.messageId++;
146
+ this.messageQueue.set(id, { resolve, reject });
147
+ const request = { id, type, payload };
148
+ this.worker.postMessage(request);
149
+ });
150
+ }
151
+ /**
152
+ * Convert Float32Array to ArrayBuffer for transfer
153
+ */
154
+ float32ArrayToBuffer(vector) {
155
+ const buffer = vector.buffer.slice(vector.byteOffset, vector.byteOffset + vector.byteLength);
156
+ // Ensure we return ArrayBuffer, not SharedArrayBuffer
157
+ return buffer instanceof ArrayBuffer ? buffer : new ArrayBuffer(0);
158
+ }
159
+ /**
160
+ * Initialize the HNSW index with cosine similarity using hnswlib-wasm
161
+ */
162
+ async initialize() {
163
+ try {
164
+ const payload = {
165
+ dimensions: this.options.dimensions,
166
+ maxElements: this.options.maxElements,
167
+ M: this.options.M,
168
+ efConstruction: this.options.efConstruction,
169
+ seed: this.options.seed,
170
+ indexPath: this.indexPath // Pass indexPath to worker for saveIndex operations
171
+ };
172
+ await this.sendMessage('init', payload);
173
+ this.isInitialized = true;
174
+ console.log(`Initialized HNSW index with ${this.options.dimensions} dimensions using hnswlib-wasm (worker)`);
175
+ }
176
+ catch (error) {
177
+ handleError(createError.index(`Failed to initialize vector index: ${error instanceof Error ? error.message : String(error)}`), 'Vector Index Initialization', {
178
+ category: ErrorCategory.INDEX,
179
+ severity: ErrorSeverity.FATAL
180
+ });
181
+ throw error;
182
+ }
103
183
  }
104
184
  /**
105
185
  * Load existing index from file using hnswlib-wasm
@@ -111,80 +191,12 @@ export class VectorIndex {
111
191
  });
112
192
  }
113
193
  try {
114
- // Load the hnswlib module
115
- if (!this.hnswlib) {
116
- // Temporarily suppress stderr output during hnswlib loading to avoid IndexedDB warnings
117
- const originalStderrWrite = process.stderr.write;
118
- const originalConsoleError = console.error;
119
- process.stderr.write = function (chunk, encoding, callback) {
120
- const message = chunk.toString();
121
- // Suppress specific IndexedDB/IDBFS related errors and WebAssembly errors
122
- if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
123
- message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
124
- message.includes('jsFS Error') || message.includes('syncing FS') ||
125
- message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
126
- message.includes('abort') || message.includes('assert') ||
127
- message.includes('hnswlib-wasm/dist/hnswlib')) {
128
- if (callback)
129
- callback();
130
- return true;
131
- }
132
- return originalStderrWrite.call(this, chunk, encoding, callback);
133
- };
134
- console.error = (...args) => {
135
- const message = args.join(' ');
136
- if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
137
- message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
138
- message.includes('jsFS Error') || message.includes('syncing FS') ||
139
- message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
140
- message.includes('abort') || message.includes('assert') ||
141
- message.includes('hnswlib-wasm/dist/hnswlib')) {
142
- return;
143
- }
144
- originalConsoleError.apply(console, args);
145
- };
146
- try {
147
- const hnswlibModule = await import('hnswlib-wasm/dist/hnswlib.js');
148
- const { loadHnswlib } = hnswlibModule;
149
- this.hnswlib = await loadHnswlib();
150
- }
151
- finally {
152
- // Restore original output streams
153
- process.stderr.write = originalStderrWrite;
154
- console.error = originalConsoleError;
155
- }
156
- }
157
- // Create new HNSW index (third parameter is autoSaveFilename, but we'll handle persistence manually)
158
- this.index = new this.hnswlib.HierarchicalNSW('cosine', this.options.dimensions, '');
159
- // Load from binary format
160
- const data = await BinaryIndexFormat.load(this.indexPath);
161
- // Validate dimensions
162
- if (data.dimensions !== this.options.dimensions) {
163
- console.log(`⚠️ Dimension mismatch detected:`);
164
- console.log(` Stored dimensions: ${data.dimensions}`);
165
- console.log(` Expected dimensions: ${this.options.dimensions}`);
166
- console.log(` Number of vectors: ${data.vectors.length}`);
167
- if (data.vectors.length > 0) {
168
- console.log(` Actual vector length: ${data.vectors[0].vector.length}`);
169
- }
170
- throw createDimensionMismatchError(this.options.dimensions, data.dimensions, 'vector index loading', { operationContext: 'VectorIndex.loadIndex' });
171
- }
172
- // Update options from stored data
173
- this.options.maxElements = data.maxElements;
174
- this.options.M = data.M;
175
- this.options.efConstruction = data.efConstruction;
176
- this.options.seed = data.seed;
177
- // Initialize HNSW index
178
- this.index.initIndex(this.options.maxElements, this.options.M, this.options.efConstruction, this.options.seed);
179
- // Clear and repopulate vector storage
180
- this.vectorStorage.clear();
181
- // Add all stored vectors to HNSW index
182
- for (const item of data.vectors) {
183
- this.index.addPoint(item.vector, item.id, false);
184
- this.vectorStorage.set(item.id, item.vector);
185
- }
186
- this.currentSize = data.currentSize;
187
- console.log(`✓ Loaded HNSW index with ${this.currentSize} vectors from ${this.indexPath}`);
194
+ const payload = {
195
+ indexPath: this.indexPath
196
+ };
197
+ const result = await this.sendMessage('loadIndex', payload);
198
+ this.isInitialized = true;
199
+ console.log(`✓ Loaded HNSW index with ${result.count} vectors from ${this.indexPath} (worker)`);
188
200
  }
189
201
  catch (error) {
190
202
  throw new Error(`Failed to load index from ${this.indexPath}: ${error}`);
@@ -194,26 +206,13 @@ export class VectorIndex {
194
206
  * Save index to binary format
195
207
  */
196
208
  async saveIndex() {
197
- if (!this.index) {
209
+ if (!this.isInitialized) {
198
210
  throw new Error('Index not initialized');
199
211
  }
200
212
  try {
201
- // Collect all vectors from storage
202
- const vectors = Array.from(this.vectorStorage.entries()).map(([id, vector]) => ({
203
- id,
204
- vector
205
- }));
206
- // Save to binary format
207
- await BinaryIndexFormat.save(this.indexPath, {
208
- dimensions: this.options.dimensions,
209
- maxElements: this.options.maxElements,
210
- M: this.options.M || 16,
211
- efConstruction: this.options.efConstruction || 200,
212
- seed: this.options.seed || 100,
213
- currentSize: this.currentSize,
214
- vectors
215
- });
216
- console.log(`✓ Saved HNSW index with ${this.currentSize} vectors to ${this.indexPath}`);
213
+ const result = await this.sendMessage('saveIndex');
214
+ const actualSize = result.count;
215
+ console.log(`✓ Saved HNSW index with ${actualSize} vectors (${(actualSize * this.options.dimensions * 4 / 1024).toFixed(2)} KB of vector data) to ${this.indexPath} (worker)`);
217
216
  }
218
217
  catch (error) {
219
218
  throw new Error(`Failed to save index to ${this.indexPath}: ${error}`);
@@ -221,84 +220,91 @@ export class VectorIndex {
221
220
  }
222
221
  /**
223
222
  * Add a single vector to the HNSW index
223
+ * Now async due to worker-based implementation
224
224
  */
225
- addVector(embeddingId, vector) {
226
- if (!this.index) {
225
+ async addVector(embeddingId, vector) {
226
+ if (!this.isInitialized) {
227
227
  throw new Error('Index not initialized');
228
228
  }
229
229
  if (vector.length !== this.options.dimensions) {
230
230
  throw createDimensionMismatchError(this.options.dimensions, vector.length, 'vector addition', { operationContext: 'VectorIndex.addVector' });
231
231
  }
232
- try {
233
- this.index.addPoint(vector, embeddingId, false);
234
- // Store vector for persistence
235
- this.vectorStorage.set(embeddingId, new Float32Array(vector));
236
- this.currentSize++;
237
- }
238
- catch (error) {
239
- throw new Error(`Failed to add vector ${embeddingId}: ${error}`);
240
- }
232
+ const payload = {
233
+ id: embeddingId,
234
+ vector: this.float32ArrayToBuffer(vector),
235
+ dimensions: vector.length
236
+ };
237
+ await this.sendMessage('addVector', payload);
241
238
  }
242
239
  /**
243
240
  * Add multiple vectors to the index in batch
241
+ * Now async due to worker-based implementation
244
242
  */
245
- addVectors(vectors) {
246
- for (const { id, vector } of vectors) {
247
- this.addVector(id, vector);
243
+ async addVectors(vectors) {
244
+ if (!this.isInitialized) {
245
+ throw new Error('Index not initialized');
248
246
  }
247
+ const payload = {
248
+ vectors: vectors.map(v => ({
249
+ id: v.id,
250
+ vector: this.float32ArrayToBuffer(v.vector),
251
+ dimensions: v.vector.length
252
+ }))
253
+ };
254
+ await this.sendMessage('addVectors', payload);
249
255
  }
250
256
  /**
251
257
  * Search for k nearest neighbors using hnswlib-wasm
258
+ * Now async due to worker-based implementation
252
259
  */
253
- search(queryVector, k = 5) {
254
- if (!this.index) {
260
+ async search(queryVector, k = 5) {
261
+ if (!this.isInitialized) {
255
262
  throw new Error('Index not initialized');
256
263
  }
257
264
  if (queryVector.length !== this.options.dimensions) {
258
265
  throw createDimensionMismatchError(this.options.dimensions, queryVector.length, 'vector search', { operationContext: 'VectorIndex.search' });
259
266
  }
260
- if (this.currentSize === 0) {
267
+ const payload = {
268
+ queryVector: this.float32ArrayToBuffer(queryVector),
269
+ dimensions: queryVector.length,
270
+ k
271
+ };
272
+ const result = await this.sendMessage('search', payload);
273
+ // Check if empty result
274
+ if (result.neighbors.length === 0 && result.distances.length === 0) {
261
275
  return { neighbors: [], distances: [] };
262
276
  }
263
- try {
264
- const result = this.index.searchKnn(queryVector, Math.min(k, this.currentSize), undefined);
265
- return {
266
- neighbors: result.neighbors,
267
- distances: result.distances
268
- };
269
- }
270
- catch (error) {
271
- throw new Error(`Search failed: ${error}`);
272
- }
277
+ return result;
273
278
  }
274
279
  /**
275
280
  * Get current number of vectors in the index
281
+ * Now async due to worker-based implementation
276
282
  */
277
- getCurrentCount() {
278
- return this.currentSize;
283
+ async getCurrentCount() {
284
+ if (!this.isInitialized) {
285
+ return 0;
286
+ }
287
+ const result = await this.sendMessage('getCurrentCount');
288
+ return result.count;
279
289
  }
280
290
  /**
281
291
  * Check if index exists on disk
282
292
  */
283
293
  indexExists() {
294
+ // This can be synchronous since it's just a file system check
284
295
  return existsSync(this.indexPath);
285
296
  }
286
297
  /**
287
298
  * Set search parameters for query time
299
+ * Now async due to worker-based implementation
288
300
  */
289
- setEf(ef) {
290
- if (!this.index) {
301
+ async setEf(ef) {
302
+ if (!this.isInitialized) {
291
303
  throw new Error('Index not initialized');
292
304
  }
305
+ const payload = { ef };
293
306
  try {
294
- // hnswlib-wasm might not have setEf method, check if it exists
295
- if (typeof this.index.setEfSearch === 'function') {
296
- this.index.setEfSearch(ef);
297
- console.log(`Set efSearch to ${ef}`);
298
- }
299
- else {
300
- console.log(`setEfSearch not available in hnswlib-wasm`);
301
- }
307
+ await this.sendMessage('setEf', payload);
302
308
  }
303
309
  catch (error) {
304
310
  console.log(`Failed to set ef: ${error}`);
@@ -306,22 +312,29 @@ export class VectorIndex {
306
312
  }
307
313
  /**
308
314
  * Resize index to accommodate more vectors
315
+ * Now async due to worker-based implementation
309
316
  */
310
- resizeIndex(newMaxElements) {
311
- if (!this.index) {
317
+ async resizeIndex(newMaxElements) {
318
+ if (!this.isInitialized) {
312
319
  throw new Error('Index not initialized');
313
320
  }
314
321
  if (newMaxElements <= this.options.maxElements) {
315
322
  throw new Error(`New max elements (${newMaxElements}) must be greater than current (${this.options.maxElements})`);
316
323
  }
317
- try {
318
- this.index.resizeIndex(newMaxElements);
319
- this.options.maxElements = newMaxElements;
320
- console.log(`Resized index to accommodate ${newMaxElements} vectors`);
321
- }
322
- catch (error) {
323
- throw new Error(`Failed to resize index: ${error}`);
324
- }
324
+ const payload = { newMaxElements };
325
+ await this.sendMessage('resizeIndex', payload);
326
+ this.options.maxElements = newMaxElements;
327
+ console.log(`Resized index to accommodate ${newMaxElements} vectors`);
328
+ }
329
+ /**
330
+ * Reset the vector index to an empty state.
331
+ * Clears all vectors from the HNSW graph and vectorStorage.
332
+ * The index parameters (dimensions, M, efConstruction) are preserved.
333
+ */
334
+ async reset() {
335
+ console.log('🔄 VectorIndex: Resetting to empty state...');
336
+ await this.sendMessage('reset');
337
+ console.log('✓ VectorIndex reset: cleared all vectors');
325
338
  }
326
339
  /**
327
340
  * Get index options (for external access to configuration)
@@ -329,5 +342,53 @@ export class VectorIndex {
329
342
  getOptions() {
330
343
  return { ...this.options };
331
344
  }
345
+ /**
346
+ * Cleanup: terminate worker and free all WebAssembly memory
347
+ */
348
+ async cleanup() {
349
+ if (this.worker) {
350
+ const workerToTerminate = this.worker;
351
+ // Clear state first to prevent new operations
352
+ this.worker = null;
353
+ this.isInitialized = false;
354
+ try {
355
+ // Send cleanup message (worker will acknowledge) with timeout
356
+ // Use the worker directly since we've cleared this.worker
357
+ const cleanupPromise = new Promise((resolve, reject) => {
358
+ const id = this.messageId++;
359
+ const timeout = setTimeout(() => {
360
+ this.messageQueue.delete(id);
361
+ reject(new Error('Cleanup timeout'));
362
+ }, 1000);
363
+ this.messageQueue.set(id, {
364
+ resolve: () => {
365
+ clearTimeout(timeout);
366
+ resolve();
367
+ },
368
+ reject: (error) => {
369
+ clearTimeout(timeout);
370
+ reject(error);
371
+ }
372
+ });
373
+ workerToTerminate.postMessage({ id, type: 'cleanup', payload: undefined });
374
+ });
375
+ await cleanupPromise;
376
+ }
377
+ catch (error) {
378
+ // Ignore errors during cleanup - worker might already be terminating
379
+ }
380
+ finally {
381
+ // Clear message queue
382
+ this.messageQueue.clear();
383
+ // Terminate worker - this frees ALL WebAssembly memory
384
+ try {
385
+ await workerToTerminate.terminate();
386
+ }
387
+ catch (error) {
388
+ // Ignore termination errors
389
+ }
390
+ }
391
+ }
392
+ }
332
393
  }
333
394
  //# sourceMappingURL=vector-index.js.map