verso-db 0.1.5 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/CHANGELOG.md +13 -0
  2. package/README.md +13 -7
  3. package/dist/BinaryHeap.d.ts +11 -1
  4. package/dist/BinaryHeap.d.ts.map +1 -1
  5. package/dist/BinaryHeap.js +138 -0
  6. package/dist/BinaryHeap.js.map +1 -0
  7. package/dist/Collection.d.ts +30 -4
  8. package/dist/Collection.d.ts.map +1 -1
  9. package/dist/Collection.js +1186 -0
  10. package/dist/Collection.js.map +1 -0
  11. package/dist/HNSWIndex.d.ts +59 -0
  12. package/dist/HNSWIndex.d.ts.map +1 -1
  13. package/dist/HNSWIndex.js +2818 -0
  14. package/dist/HNSWIndex.js.map +1 -0
  15. package/dist/MaxBinaryHeap.d.ts +2 -64
  16. package/dist/MaxBinaryHeap.d.ts.map +1 -1
  17. package/dist/MaxBinaryHeap.js +5 -0
  18. package/dist/MaxBinaryHeap.js.map +1 -0
  19. package/dist/SearchWorker.d.ts +57 -4
  20. package/dist/SearchWorker.d.ts.map +1 -1
  21. package/dist/SearchWorker.js +573 -0
  22. package/dist/SearchWorker.js.map +1 -0
  23. package/dist/VectorDB.d.ts.map +1 -1
  24. package/dist/VectorDB.js +246 -0
  25. package/dist/VectorDB.js.map +1 -0
  26. package/dist/WorkerPool.d.ts +32 -2
  27. package/dist/WorkerPool.d.ts.map +1 -1
  28. package/dist/WorkerPool.js +266 -0
  29. package/dist/WorkerPool.js.map +1 -0
  30. package/dist/backends/JsDistanceBackend.d.ts.map +1 -1
  31. package/dist/backends/JsDistanceBackend.js +163 -0
  32. package/dist/backends/JsDistanceBackend.js.map +1 -0
  33. package/dist/encoding/DeltaEncoder.d.ts +2 -2
  34. package/dist/encoding/DeltaEncoder.d.ts.map +1 -1
  35. package/dist/encoding/DeltaEncoder.js +199 -0
  36. package/dist/encoding/DeltaEncoder.js.map +1 -0
  37. package/dist/errors.js +97 -0
  38. package/dist/errors.js.map +1 -0
  39. package/dist/index.d.ts +3 -3
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +61 -42
  42. package/dist/index.js.map +1 -9
  43. package/dist/presets.js +205 -0
  44. package/dist/presets.js.map +1 -0
  45. package/dist/quantization/ScalarQuantizer.d.ts +0 -34
  46. package/dist/quantization/ScalarQuantizer.d.ts.map +1 -1
  47. package/dist/quantization/ScalarQuantizer.js +346 -0
  48. package/dist/quantization/ScalarQuantizer.js.map +1 -0
  49. package/dist/storage/BatchWriter.js +351 -0
  50. package/dist/storage/BatchWriter.js.map +1 -0
  51. package/dist/storage/BunStorageBackend.d.ts +7 -3
  52. package/dist/storage/BunStorageBackend.d.ts.map +1 -1
  53. package/dist/storage/BunStorageBackend.js +182 -0
  54. package/dist/storage/BunStorageBackend.js.map +1 -0
  55. package/dist/storage/MemoryBackend.js +109 -0
  56. package/dist/storage/MemoryBackend.js.map +1 -0
  57. package/dist/storage/OPFSBackend.d.ts.map +1 -1
  58. package/dist/storage/OPFSBackend.js +325 -0
  59. package/dist/storage/OPFSBackend.js.map +1 -0
  60. package/dist/storage/StorageBackend.js +12 -0
  61. package/dist/storage/StorageBackend.js.map +1 -0
  62. package/dist/storage/WriteAheadLog.js +321 -0
  63. package/dist/storage/WriteAheadLog.js.map +1 -0
  64. package/dist/storage/createStorageBackend.d.ts +4 -0
  65. package/dist/storage/createStorageBackend.d.ts.map +1 -1
  66. package/dist/storage/createStorageBackend.js +119 -0
  67. package/dist/storage/createStorageBackend.js.map +1 -0
  68. package/{src/storage/index.ts → dist/storage/index.js} +7 -27
  69. package/dist/storage/index.js.map +1 -0
  70. package/dist/storage/nodeFsRuntime.d.ts +14 -0
  71. package/dist/storage/nodeFsRuntime.d.ts.map +1 -0
  72. package/dist/storage/nodeFsRuntime.js +105 -0
  73. package/dist/storage/nodeFsRuntime.js.map +1 -0
  74. package/package.json +9 -7
  75. package/src/BinaryHeap.ts +0 -136
  76. package/src/Collection.ts +0 -1262
  77. package/src/HNSWIndex.ts +0 -2894
  78. package/src/MaxBinaryHeap.ts +0 -181
  79. package/src/SearchWorker.ts +0 -264
  80. package/src/VectorDB.ts +0 -319
  81. package/src/WorkerPool.ts +0 -222
  82. package/src/backends/JsDistanceBackend.ts +0 -171
  83. package/src/encoding/DeltaEncoder.ts +0 -236
  84. package/src/errors.ts +0 -110
  85. package/src/index.ts +0 -106
  86. package/src/presets.ts +0 -229
  87. package/src/quantization/ScalarQuantizer.ts +0 -487
  88. package/src/storage/BatchWriter.ts +0 -420
  89. package/src/storage/BunStorageBackend.ts +0 -199
  90. package/src/storage/MemoryBackend.ts +0 -122
  91. package/src/storage/OPFSBackend.ts +0 -348
  92. package/src/storage/StorageBackend.ts +0 -74
  93. package/src/storage/WriteAheadLog.ts +0 -379
  94. package/src/storage/createStorageBackend.ts +0 -137
@@ -0,0 +1,246 @@
1
+ import { Collection } from './Collection.js';
2
+ import { BunStorageBackend } from './storage/BunStorageBackend.js';
3
+ import { MemoryBackend } from './storage/MemoryBackend.js';
4
+ import { OPFSBackend } from './storage/OPFSBackend.js';
5
+ import { detectEnvironment } from './storage/createStorageBackend.js';
6
+ import { CollectionExistsError, CollectionNotFoundError, VectorDBError } from './errors.js';
7
+ const MANIFEST_KEY = 'manifest.json';
8
+ const VALID_METRICS = new Set(['cosine', 'euclidean', 'dot_product']);
9
+ export class VectorDB {
10
+ collections;
11
+ storageBackend;
12
+ initialized = false;
13
+ initPromise = null;
14
+ createCollectionLocks = new Map();
15
+ constructor(config) {
16
+ this.collections = new Map();
17
+ if (config?.storageBackend) {
18
+ this.storageBackend = config.storageBackend;
19
+ }
20
+ else {
21
+ this.storageBackend = this.createDefaultStorageBackend(config?.storagePath);
22
+ }
23
+ }
24
+ createDefaultStorageBackend(storagePath) {
25
+ const env = detectEnvironment();
26
+ if (env === 'bun' || env === 'node') {
27
+ return new BunStorageBackend(storagePath || './vectordb_data');
28
+ }
29
+ if (env === 'browser' && OPFSBackend.isAvailable()) {
30
+ return new OPFSBackend();
31
+ }
32
+ return new MemoryBackend();
33
+ }
34
+ /**
35
+ * Get the storage backend used by this VectorDB instance.
36
+ * Useful for custom storage operations or debugging.
37
+ */
38
+ getStorageBackend() {
39
+ return this.storageBackend;
40
+ }
41
+ /**
42
+ * Initialize the VectorDB - creates storage directory and loads existing collections.
43
+ * Called automatically on first operation if not called explicitly.
44
+ * Safe to call concurrently — only runs initialization once.
45
+ */
46
+ async init() {
47
+ if (this.initialized)
48
+ return;
49
+ if (!this.initPromise) {
50
+ this.initPromise = this.doInit().catch((err) => {
51
+ // Allow retry after transient initialization failures.
52
+ this.initPromise = null;
53
+ this.initialized = false;
54
+ throw err;
55
+ });
56
+ }
57
+ return this.initPromise;
58
+ }
59
+ async doInit() {
60
+ // Use storage backend to create root directory
61
+ await this.storageBackend.mkdir('');
62
+ // Load existing collections from manifest
63
+ await this.loadManifest();
64
+ this.initialized = true;
65
+ }
66
+ /**
67
+ * Load collection metadata from the manifest file and restore collections.
68
+ */
69
+ async loadManifest() {
70
+ const data = await this.storageBackend.read(MANIFEST_KEY);
71
+ if (!data)
72
+ return;
73
+ let manifest;
74
+ try {
75
+ const text = new TextDecoder().decode(data);
76
+ manifest = JSON.parse(text);
77
+ }
78
+ catch {
79
+ // Corrupt manifest JSON — continue with empty collections
80
+ return;
81
+ }
82
+ for (const [name, entry] of Object.entries(manifest)) {
83
+ // Skip collections already loaded
84
+ if (this.collections.has(name))
85
+ continue;
86
+ try {
87
+ this.validateCollectionName(name);
88
+ }
89
+ catch {
90
+ console.warn(`Skipping collection '${name}' with invalid name in manifest`);
91
+ continue;
92
+ }
93
+ // Validate manifest entry types to catch corrupted manifests early
94
+ if (typeof entry.dimension !== 'number' || entry.dimension <= 0 || !Number.isInteger(entry.dimension) ||
95
+ typeof entry.metric !== 'string' || !VALID_METRICS.has(entry.metric) ||
96
+ typeof entry.M !== 'number' || entry.M <= 0 || !Number.isInteger(entry.M) ||
97
+ typeof entry.efConstruction !== 'number' || entry.efConstruction <= 0 || !Number.isInteger(entry.efConstruction)) {
98
+ console.warn(`Skipping collection '${name}' with invalid manifest entry`);
99
+ continue;
100
+ }
101
+ const config = {
102
+ dimension: entry.dimension,
103
+ metric: entry.metric,
104
+ M: entry.M,
105
+ efConstruction: entry.efConstruction,
106
+ };
107
+ try {
108
+ const collection = new Collection(name, config, this.storageBackend, { autoPersist: true });
109
+ await collection.init();
110
+ this.collections.set(name, collection);
111
+ }
112
+ catch (e) {
113
+ // Per-collection init failure — skip this collection but continue loading others
114
+ console.warn(`Failed to load collection '${name}':`, e);
115
+ }
116
+ }
117
+ }
118
+ /**
119
+ * Save collection metadata to the manifest file.
120
+ */
121
+ async saveManifest() {
122
+ const manifest = {};
123
+ for (const [name, collection] of this.collections) {
124
+ manifest[name] = {
125
+ dimension: collection.getDimension(),
126
+ metric: collection.getMetric(),
127
+ M: collection.getM(),
128
+ efConstruction: collection.getEfConstruction(),
129
+ };
130
+ }
131
+ const data = new TextEncoder().encode(JSON.stringify(manifest));
132
+ await this.storageBackend.write(MANIFEST_KEY, data);
133
+ }
134
+ async withCollectionCreateLock(name, fn) {
135
+ const previous = this.createCollectionLocks.get(name) ?? Promise.resolve();
136
+ let release;
137
+ const lock = new Promise((resolve) => {
138
+ release = resolve;
139
+ });
140
+ this.createCollectionLocks.set(name, lock);
141
+ await previous;
142
+ try {
143
+ return await fn();
144
+ }
145
+ finally {
146
+ release();
147
+ if (this.createCollectionLocks.get(name) === lock) {
148
+ this.createCollectionLocks.delete(name);
149
+ }
150
+ }
151
+ }
152
+ async createCollection(name, config) {
153
+ this.validateCollectionName(name);
154
+ // Ensure VectorDB is initialized
155
+ await this.init();
156
+ return this.withCollectionCreateLock(name, async () => {
157
+ if (this.collections.has(name)) {
158
+ throw new CollectionExistsError(name);
159
+ }
160
+ // Create collection directory using storage backend
161
+ await this.storageBackend.mkdir(name);
162
+ const collection = new Collection(name, config, this.storageBackend, { autoPersist: true });
163
+ await collection.init();
164
+ this.collections.set(name, collection);
165
+ // Persist manifest
166
+ await this.saveManifest();
167
+ return collection;
168
+ });
169
+ }
170
+ async getCollection(name) {
171
+ await this.init();
172
+ return this.collections.get(name);
173
+ }
174
+ async listCollections() {
175
+ await this.init();
176
+ return [...this.collections.keys()];
177
+ }
178
+ async deleteCollection(name) {
179
+ await this.init();
180
+ const collection = this.collections.get(name);
181
+ if (!collection) {
182
+ throw new CollectionNotFoundError(name);
183
+ }
184
+ // Destroy without saving — we're deleting the collection
185
+ await collection.destroy(false);
186
+ this.collections.delete(name);
187
+ // Remove all files under the collection prefix, including versioned snapshots.
188
+ const keys = await this.storageBackend.list(name);
189
+ for (const key of keys) {
190
+ await this.storageBackend.delete(key).catch(() => { });
191
+ }
192
+ // Update manifest
193
+ await this.saveManifest();
194
+ }
195
+ validateCollectionName(name) {
196
+ if (!name || name.trim().length === 0) {
197
+ throw new VectorDBError('Collection name must be a non-empty string', 'VALIDATION_ERROR');
198
+ }
199
+ if (name.length > 255) {
200
+ throw new VectorDBError(`Collection name must be 255 characters or fewer, got ${name.length}`, 'VALIDATION_ERROR');
201
+ }
202
+ if (/[\/\\.\x00]/.test(name)) {
203
+ throw new VectorDBError(`Collection name '${name}' contains invalid characters (/, \\, ., or null)`, 'VALIDATION_ERROR');
204
+ }
205
+ // Reject control characters (U+0001–U+001F, U+007F)
206
+ // eslint-disable-next-line no-control-regex
207
+ if (/[\x01-\x1f\x7f]/.test(name)) {
208
+ throw new VectorDBError('Collection name contains control characters', 'VALIDATION_ERROR');
209
+ }
210
+ // Reject leading/trailing whitespace (checked after control chars since
211
+ // tab/etc. are control chars and should get the more specific message)
212
+ if (name !== name.trim()) {
213
+ throw new VectorDBError('Collection name must not have leading or trailing whitespace', 'VALIDATION_ERROR');
214
+ }
215
+ }
216
+ async close() {
217
+ // Close all collections — continue on individual failures so remaining
218
+ // collections still get saved and cleanup always completes.
219
+ const errors = [];
220
+ for (const collection of this.collections.values()) {
221
+ try {
222
+ await collection.destroy();
223
+ }
224
+ catch (e) {
225
+ errors.push(e instanceof Error ? e : new Error(String(e)));
226
+ }
227
+ }
228
+ // Persist the manifest so collection metadata survives re-open
229
+ if (this.initialized) {
230
+ try {
231
+ await this.saveManifest();
232
+ }
233
+ catch (e) {
234
+ errors.push(e instanceof Error ? e : new Error(String(e)));
235
+ }
236
+ }
237
+ this.collections.clear();
238
+ this.createCollectionLocks.clear();
239
+ this.initialized = false;
240
+ this.initPromise = null;
241
+ if (errors.length > 0) {
242
+ throw new VectorDBError(`Failed to close ${errors.length} collection(s): ${errors.map(e => e.message).join('; ')}`, 'STORAGE_ERROR');
243
+ }
244
+ }
245
+ }
246
+ //# sourceMappingURL=VectorDB.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"VectorDB.js","sourceRoot":"","sources":["../src/VectorDB.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AA8BzF,MAAM,YAAY,GAAG,eAAe,CAAC;AACrC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;AAEtE,MAAM,OAAO,QAAQ;IACX,WAAW,CAA0B;IACrC,cAAc,CAAiB;IAC/B,WAAW,GAAY,KAAK,CAAC;IAC7B,WAAW,GAAyB,IAAI,CAAC;IACzC,qBAAqB,GAA+B,IAAI,GAAG,EAAE,CAAC;IAEtE,YAAY,MAAuB;QACjC,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAC7B,IAAI,MAAM,EAAE,cAAc,EAAE,CAAC;YAC3B,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,WAAoB;QACtD,MAAM,GAAG,GAAG,iBAAiB,EAAE,CAAC;QAEhC,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;YACpC,OAAO,IAAI,iBAAiB,CAAC,WAAW,IAAI,iBAAiB,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,GAAG,KAAK,SAAS,IAAI,WAAW,CAAC,WAAW,EAAE,EAAE,CAAC;YACnD,OAAO,IAAI,WAAW,EAAE,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,aAAa,EAAE,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,iBAAiB;QACf,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC7C,uDAAuD;gBACvD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;gBACzB,MAAM,GAAG,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,MAAM;QAClB,+CAA+C;QAC/C,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAEpC,0CAA0C;QAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,IAAI,QAAuC,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5C,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,0DAA0D;YAC1D,OAAO;QACT,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrD,kCAAkC;YAClC,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAEzC,IAAI,CAAC;gBACH,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,IAAI,CAAC,wBAAwB,IAAI,iCAAiC,CAAC,CAAC;gBAC5E,SAAS;YACX,CAAC;YAED,mEAAmE;YACnE,IACE,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC;gBACjG,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;gBACpE,OAAO,KAAK,CAAC,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;gBACzE,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ,IAAI,KAAK,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC,EAChH,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,wBAAwB,IAAI,+BAA+B,CAAC,CAAC;gBAC1E,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAqB;gBAC/B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,CAAC,EAAE,KAAK,CAAC,CAAC;gBACV,cAAc,EAAE,KAAK,CAAC,cAAc;aACrC,CAAC;YAEF,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5F,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;gBACxB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,iFAAiF;gBACjF,OAAO,CAAC,IAAI,CAAC,8BAA8B,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,MAAM,QAAQ,GAAkC,EAAE,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAClD,QAAQ,CAAC,IAAI,CAAC,GAAG;gBACf,SAAS,EAAE,UAAU,CAAC,YAAY,EAAE;gBACpC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE;gBAC9B,CAAC,EAAE,UAAU,CAAC,IAAI,EAAE;gBACpB,cAAc,EAAE,UAAU,CAAC,iBAAiB,EAAE;aAC/C,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QAChE,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IACtD,CAAC;IAEO,KAAK,CAAC,wBAAwB,CAAI,IAAY,EAAE,EAAoB;QAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3E,IAAI,OAAoB,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACzC,OAAO,GAAG,OAAO,CAAC;QACpB,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAE3C,MAAM,QAAQ,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,OAAO,EAAE,CAAC;YACV,IAAI,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gBAClD,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAY,EAAE,MAAwB;QAC3D,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAElC,iCAAiC;QACjC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAElB,OAAO,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;YACpD,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,qBAAqB,CAAC,IAAI,CAAC,CAAC;YACxC,CAAC;YAED,oDAAoD;YACpD,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEtC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5F,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAEvC,mBAAmB;YACnB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,IAAY;QAC9B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAY;QACjC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QAED,yDAAyD;QACzD,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE9B,+EAA+E;QAC/E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,kBAAkB;QAClB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC5B,CAAC;IAEO,sBAAsB,CAAC,IAAY;QACzC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,aAAa,CAAC,4CAA4C,EAAE,kBAAkB,CAAC,CAAC;QAC5F,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACtB,MAAM,IAAI,aAAa,CACrB,wDAAwD,IAAI,CAAC,MAAM,EAAE,EACrE,kBAAkB,CACnB,CAAC;QACJ,CAAC;QACD,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,aAAa,CACrB,oBAAoB,IAAI,mDAAmD,EAC3E,kBAAkB,CACnB,CAAC;QACJ,CAAC;QACD,oDAAoD;QACpD,4CAA4C;QAC5C,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,aAAa,CACrB,6CAA6C,EAC7C,kBAAkB,CACnB,CAAC;QACJ,CAAC;QACD,wEAAwE;QACxE,uEAAuE;QACvE,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACzB,MAAM,IAAI,aAAa,CAAC,8DAA8D,EAAE,kBAAkB,CAAC,CAAC;QAC9G,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,uEAAuE;QACvE,4DAA4D;QAC5D,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YACnD,IAAI,CAAC;gBACH,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;YAC7B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,+DAA+D;QAC/D,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC5B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,aAAa,CACrB,mBAAmB,MAAM,CAAC,MAAM,mBAAmB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAC1F,eAAe,CAChB,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}
@@ -19,8 +19,8 @@ export declare class WorkerPool {
19
19
  constructor(numWorkers?: number | undefined);
20
20
  /**
21
21
  * Initialize workers with shared index data.
22
- * Each worker receives a copy of the flat vectors and graph structure
23
- * and maintains its own independent search state.
22
+ * When the index uses SharedArrayBuffer, workers receive zero-copy
23
+ * views into the same vector memory. Otherwise, data is copied.
24
24
  *
25
25
  * @param index The HNSW index to distribute to workers
26
26
  */
@@ -38,6 +38,20 @@ export declare class WorkerPool {
38
38
  id: number;
39
39
  distance: number;
40
40
  }>>;
41
+ /**
42
+ * Quantized search using worker pool.
43
+ * Workers perform int8 candidate scan + float32 rescore.
44
+ *
45
+ * @param query Query vector
46
+ * @param k Number of results
47
+ * @param candidateMultiplier Multiplier for rescore candidates
48
+ * @param efSearch Search effort parameter
49
+ * @returns Array of {id, distance} results
50
+ */
51
+ searchQuantized(query: Float32Array, k: number, candidateMultiplier?: number, efSearch?: number): Promise<Array<{
52
+ id: number;
53
+ distance: number;
54
+ }>>;
41
55
  /**
42
56
  * Batch search: dispatch multiple queries in parallel across workers.
43
57
  *
@@ -50,6 +64,22 @@ export declare class WorkerPool {
50
64
  id: number;
51
65
  distance: number;
52
66
  }>>>;
67
+ /**
68
+ * Batch quantized search across workers.
69
+ */
70
+ searchBatchQuantized(queries: Float32Array[], k: number, candidateMultiplier?: number, efSearch?: number): Promise<Array<Array<{
71
+ id: number;
72
+ distance: number;
73
+ }>>>;
74
+ /**
75
+ * Send incremental graph updates to all workers.
76
+ * Workers update their neighbor lists without full re-initialization.
77
+ * New vectors are already visible via SharedArrayBuffer.
78
+ */
79
+ broadcastGraphUpdate(newNodes: Array<{
80
+ id: number;
81
+ neighbors: number[][];
82
+ }>, entryPointId?: number, maxLevel?: number): void;
53
83
  /**
54
84
  * Terminate all workers and clean up.
55
85
  */
@@ -1 +1 @@
1
- {"version":3,"file":"WorkerPool.d.ts","sourceRoot":"","sources":["../src/WorkerPool.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AASxC,qBAAa,UAAU;IAYT,OAAO,CAAC,UAAU,CAAC;IAX/B,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,aAAa,CAA0B;IAE/C;;;;OAIG;gBACiB,UAAU,CAAC,EAAE,MAAM,YAAA;IAQvC;;;;;;OAMG;IACG,IAAI,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAyF3C;;;;;;;;OAQG;IACG,MAAM,CACV,KAAK,EAAE,YAAY,EACnB,CAAC,EAAE,MAAM,EACT,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA4BnD;;;;;;;OAOG;IACG,WAAW,CACf,OAAO,EAAE,YAAY,EAAE,EACvB,CAAC,EAAE,MAAM,EACT,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;IAc1D;;OAEG;IACH,OAAO,IAAI,IAAI;IAcf;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;CACF"}
1
+ {"version":3,"file":"WorkerPool.d.ts","sourceRoot":"","sources":["../src/WorkerPool.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,qBAAa,UAAU;IAYT,OAAO,CAAC,UAAU,CAAC;IAX/B,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,aAAa,CAA0B;IAE/C;;;;OAIG;gBACiB,UAAU,CAAC,EAAE,MAAM,YAAA;IAQvC;;;;;;OAMG;IACG,IAAI,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAmG3C;;;;;;;;OAQG;IACG,MAAM,CACV,KAAK,EAAE,YAAY,EACnB,CAAC,EAAE,MAAM,EACT,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA4BnD;;;;;;;;;OASG;IACG,eAAe,CACnB,KAAK,EAAE,YAAY,EACnB,CAAC,EAAE,MAAM,EACT,mBAAmB,GAAE,MAAU,EAC/B,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA8BnD;;;;;;;OAOG;IACG,WAAW,CACf,OAAO,EAAE,YAAY,EAAE,EACvB,CAAC,EAAE,MAAM,EACT,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;IAc1D;;OAEG;IACG,oBAAoB,CACxB,OAAO,EAAE,YAAY,EAAE,EACvB,CAAC,EAAE,MAAM,EACT,mBAAmB,GAAE,MAAU,EAC/B,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;IAa1D;;;;OAIG;IACH,oBAAoB,CAClB,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,EAAE,CAAA;KAAE,CAAC,EACtD,YAAY,CAAC,EAAE,MAAM,EACrB,QAAQ,CAAC,EAAE,MAAM,GAChB,IAAI;IAgBP;;OAEG;IACH,OAAO,IAAI,IAAI;IAcf;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;CACF"}
@@ -0,0 +1,266 @@
1
+ /**
2
+ * Worker pool for parallel query processing.
3
+ * Dispatches queries to workers via round-robin for concurrent execution.
4
+ * Falls back to sequential processing when workers are unavailable.
5
+ */
6
+ import { VectorDBError } from './errors.js';
7
+ export class WorkerPool {
8
+ numWorkers;
9
+ workers = [];
10
+ nextQueryId = 0;
11
+ roundRobinIndex = 0;
12
+ initialized = false;
13
+ fallbackIndex = null;
14
+ /**
15
+ * Create a worker pool for parallel HNSW search.
16
+ *
17
+ * @param numWorkers Number of workers (default: available hardware concurrency or 4)
18
+ */
19
+ constructor(numWorkers) {
20
+ this.numWorkers = numWorkers;
21
+ if (!numWorkers) {
22
+ this.numWorkers = typeof navigator !== 'undefined'
23
+ ? (navigator.hardwareConcurrency ?? 4)
24
+ : 4;
25
+ }
26
+ }
27
+ /**
28
+ * Initialize workers with shared index data.
29
+ * When the index uses SharedArrayBuffer, workers receive zero-copy
30
+ * views into the same vector memory. Otherwise, data is copied.
31
+ *
32
+ * @param index The HNSW index to distribute to workers
33
+ */
34
+ async init(index) {
35
+ this.fallbackIndex = index;
36
+ // Check if Worker is available
37
+ if (typeof Worker === 'undefined') {
38
+ // Workers not available, will use fallback
39
+ this.initialized = true;
40
+ return;
41
+ }
42
+ // Enable SharedArrayBuffer on the index for zero-copy vector sharing
43
+ index.enableSharedMemory();
44
+ const sharedData = index.getSharedSearchData();
45
+ if (!sharedData) {
46
+ // Index can't produce shared data, use fallback
47
+ this.initialized = true;
48
+ return;
49
+ }
50
+ const workerModulePath = import.meta.url.endsWith('.ts')
51
+ ? './SearchWorker.ts'
52
+ : './SearchWorker.js';
53
+ const workerUrl = new URL(workerModulePath, import.meta.url).href;
54
+ const initPromises = [];
55
+ for (let i = 0; i < this.numWorkers; i++) {
56
+ try {
57
+ const worker = new Worker(workerUrl, { type: 'module' });
58
+ const handle = {
59
+ worker,
60
+ readyResolve: null,
61
+ pendingResolve: new Map(),
62
+ pendingReject: new Map(),
63
+ };
64
+ worker.onmessage = (event) => {
65
+ const msg = event.data;
66
+ if (msg.type === 'ready') {
67
+ if (handle.readyResolve) {
68
+ handle.readyResolve();
69
+ handle.readyResolve = null;
70
+ }
71
+ }
72
+ else if (msg.type === 'result') {
73
+ const resolve = handle.pendingResolve.get(msg.queryId);
74
+ if (resolve) {
75
+ handle.pendingResolve.delete(msg.queryId);
76
+ handle.pendingReject.delete(msg.queryId);
77
+ resolve(msg.results);
78
+ }
79
+ }
80
+ else if (msg.type === 'error') {
81
+ const reject = handle.pendingReject.get(msg.queryId);
82
+ if (reject) {
83
+ handle.pendingResolve.delete(msg.queryId);
84
+ handle.pendingReject.delete(msg.queryId);
85
+ reject(new Error(msg.error));
86
+ }
87
+ }
88
+ };
89
+ worker.onerror = (event) => {
90
+ for (const reject of handle.pendingReject.values()) {
91
+ reject(new Error(`Worker error: ${event.message}`));
92
+ }
93
+ handle.pendingResolve.clear();
94
+ handle.pendingReject.clear();
95
+ };
96
+ const readyPromise = new Promise((resolve) => {
97
+ handle.readyResolve = resolve;
98
+ });
99
+ worker.postMessage({
100
+ type: 'init',
101
+ ...sharedData,
102
+ });
103
+ this.workers.push(handle);
104
+ initPromises.push(readyPromise);
105
+ }
106
+ catch {
107
+ // Worker creation failed, will use fallback for remaining
108
+ break;
109
+ }
110
+ }
111
+ if (initPromises.length > 0) {
112
+ const timeout = new Promise((_, reject) => {
113
+ const timer = setTimeout(() => reject(new Error('WorkerPool initialization timed out after 10s')), 10000);
114
+ if (typeof timer === 'object' && 'unref' in timer) {
115
+ timer.unref();
116
+ }
117
+ });
118
+ await Promise.race([Promise.all(initPromises), timeout]);
119
+ }
120
+ this.initialized = true;
121
+ }
122
+ /**
123
+ * Search for k nearest neighbors using worker pool.
124
+ * Dispatches to next available worker via round-robin.
125
+ *
126
+ * @param query Query vector
127
+ * @param k Number of results
128
+ * @param efSearch Search effort parameter
129
+ * @returns Array of {id, distance} results
130
+ */
131
+ async search(query, k, efSearch) {
132
+ if (!this.initialized) {
133
+ throw new VectorDBError('WorkerPool not initialized. Call init() first.', 'VALIDATION_ERROR');
134
+ }
135
+ // Fallback to sequential if no workers available
136
+ if (this.workers.length === 0 && this.fallbackIndex) {
137
+ return this.fallbackIndex.searchKNN(query, k, efSearch);
138
+ }
139
+ const queryId = this.nextQueryId++;
140
+ const workerIdx = this.roundRobinIndex % this.workers.length;
141
+ this.roundRobinIndex++;
142
+ const handle = this.workers[workerIdx];
143
+ return new Promise((resolve, reject) => {
144
+ handle.pendingResolve.set(queryId, resolve);
145
+ handle.pendingReject.set(queryId, reject);
146
+ handle.worker.postMessage({
147
+ type: 'search',
148
+ queryId,
149
+ query,
150
+ k,
151
+ efSearch: efSearch || Math.max(k * 2, 50),
152
+ });
153
+ });
154
+ }
155
+ /**
156
+ * Quantized search using worker pool.
157
+ * Workers perform int8 candidate scan + float32 rescore.
158
+ *
159
+ * @param query Query vector
160
+ * @param k Number of results
161
+ * @param candidateMultiplier Multiplier for rescore candidates
162
+ * @param efSearch Search effort parameter
163
+ * @returns Array of {id, distance} results
164
+ */
165
+ async searchQuantized(query, k, candidateMultiplier = 3, efSearch) {
166
+ if (!this.initialized) {
167
+ throw new VectorDBError('WorkerPool not initialized. Call init() first.', 'VALIDATION_ERROR');
168
+ }
169
+ // Fallback to sequential
170
+ if (this.workers.length === 0 && this.fallbackIndex) {
171
+ return this.fallbackIndex.searchKNNQuantized(query, k, candidateMultiplier, efSearch);
172
+ }
173
+ const queryId = this.nextQueryId++;
174
+ const workerIdx = this.roundRobinIndex % this.workers.length;
175
+ this.roundRobinIndex++;
176
+ const handle = this.workers[workerIdx];
177
+ return new Promise((resolve, reject) => {
178
+ handle.pendingResolve.set(queryId, resolve);
179
+ handle.pendingReject.set(queryId, reject);
180
+ handle.worker.postMessage({
181
+ type: 'search',
182
+ queryId,
183
+ query,
184
+ k,
185
+ efSearch: efSearch || Math.max(k * candidateMultiplier * 2, 50),
186
+ quantized: true,
187
+ candidateMultiplier,
188
+ });
189
+ });
190
+ }
191
+ /**
192
+ * Batch search: dispatch multiple queries in parallel across workers.
193
+ *
194
+ * @param queries Array of query vectors
195
+ * @param k Number of results per query
196
+ * @param efSearch Search effort parameter
197
+ * @returns Array of results, one per query
198
+ */
199
+ async searchBatch(queries, k, efSearch) {
200
+ if (!this.initialized) {
201
+ throw new VectorDBError('WorkerPool not initialized. Call init() first.', 'VALIDATION_ERROR');
202
+ }
203
+ // Fallback to sequential
204
+ if (this.workers.length === 0 && this.fallbackIndex) {
205
+ return queries.map(q => this.fallbackIndex.searchKNN(q, k, efSearch));
206
+ }
207
+ const promises = queries.map(q => this.search(q, k, efSearch));
208
+ return Promise.all(promises);
209
+ }
210
+ /**
211
+ * Batch quantized search across workers.
212
+ */
213
+ async searchBatchQuantized(queries, k, candidateMultiplier = 3, efSearch) {
214
+ if (!this.initialized) {
215
+ throw new VectorDBError('WorkerPool not initialized. Call init() first.', 'VALIDATION_ERROR');
216
+ }
217
+ if (this.workers.length === 0 && this.fallbackIndex) {
218
+ return queries.map(q => this.fallbackIndex.searchKNNQuantized(q, k, candidateMultiplier, efSearch));
219
+ }
220
+ const promises = queries.map(q => this.searchQuantized(q, k, candidateMultiplier, efSearch));
221
+ return Promise.all(promises);
222
+ }
223
+ /**
224
+ * Send incremental graph updates to all workers.
225
+ * Workers update their neighbor lists without full re-initialization.
226
+ * New vectors are already visible via SharedArrayBuffer.
227
+ */
228
+ broadcastGraphUpdate(newNodes, entryPointId, maxLevel) {
229
+ for (const handle of this.workers) {
230
+ handle.worker.postMessage({
231
+ type: 'graphUpdate',
232
+ newNodes,
233
+ });
234
+ if (entryPointId !== undefined && maxLevel !== undefined) {
235
+ handle.worker.postMessage({
236
+ type: 'updateEntryPoint',
237
+ entryPointId,
238
+ maxLevel,
239
+ });
240
+ }
241
+ }
242
+ }
243
+ /**
244
+ * Terminate all workers and clean up.
245
+ */
246
+ destroy() {
247
+ for (const handle of this.workers) {
248
+ handle.worker.terminate();
249
+ for (const reject of handle.pendingReject.values()) {
250
+ reject(new Error('Worker pool destroyed'));
251
+ }
252
+ handle.pendingResolve.clear();
253
+ handle.pendingReject.clear();
254
+ }
255
+ this.workers = [];
256
+ this.initialized = false;
257
+ this.fallbackIndex = null;
258
+ }
259
+ /**
260
+ * Get the number of active workers.
261
+ */
262
+ get workerCount() {
263
+ return this.workers.length;
264
+ }
265
+ }
266
+ //# sourceMappingURL=WorkerPool.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"WorkerPool.js","sourceRoot":"","sources":["../src/WorkerPool.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AASzC,MAAM,OAAO,UAAU;IAYD;IAXZ,OAAO,GAAmB,EAAE,CAAC;IAC7B,WAAW,GAAW,CAAC,CAAC;IACxB,eAAe,GAAW,CAAC,CAAC;IAC5B,WAAW,GAAY,KAAK,CAAC;IAC7B,aAAa,GAAqB,IAAI,CAAC;IAE/C;;;;OAIG;IACH,YAAoB,UAAmB;QAAnB,eAAU,GAAV,UAAU,CAAS;QACrC,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,IAAI,CAAC,UAAU,GAAG,OAAO,SAAS,KAAK,WAAW;gBAChD,CAAC,CAAC,CAAC,SAAS,CAAC,mBAAmB,IAAI,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC,CAAC;QACR,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,CAAC,KAAgB;QACzB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;QAE3B,+BAA+B;QAC/B,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,2CAA2C;YAC3C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO;QACT,CAAC;QAED,qEAAqE;QACrE,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAE3B,MAAM,UAAU,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;QAC/C,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,gDAAgD;YAChD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO;QACT,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;YACtD,CAAC,CAAC,mBAAmB;YACrB,CAAC,CAAC,mBAAmB,CAAC;QACxB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;QAElE,MAAM,YAAY,GAAoB,EAAE,CAAC;QAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACzD,MAAM,MAAM,GAAiB;oBAC3B,MAAM;oBACN,YAAY,EAAE,IAAI;oBAClB,cAAc,EAAE,IAAI,GAAG,EAAE;oBACzB,aAAa,EAAE,IAAI,GAAG,EAAE;iBACzB,CAAC;gBAEF,MAAM,CAAC,SAAS,GAAG,CAAC,KAAmB,EAAE,EAAE;oBACzC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC;oBACvB,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;wBACzB,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;4BACxB,MAAM,CAAC,YAAY,EAAE,CAAC;4BACtB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;wBAC7B,CAAC;oBACH,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACjC,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;wBACvD,IAAI,OAAO,EAAE,CAAC;4BACZ,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;4BAC1C,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;4BACzC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;wBACvB,CAAC;oBACH,CAAC;yBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;wBAChC,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;wBACrD,IAAI,MAAM,EAAE,CAAC;4BACX,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;4BAC1C,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;4BACzC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;wBAC/B,CAAC;oBACH,CAAC;gBACH,CAAC,CAAC;gBAEF,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;oBACzB,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;wBACnD,MAAM,CAAC,IAAI,KAAK,CAAC,iBAAiB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACtD,CAAC;oBACD,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;oBAC9B,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;gBAC/B,CAAC,CAAC;gBAEF,MAAM,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;oBACjD,MAAM,CAAC,YAAY,GAAG,OAAO,CAAC;gBAChC,CAAC,CAAC,CAAC;gBAEH,MAAM,CAAC,WAAW,CAAC;oBACjB,IAAI,EAAE,MAAM;oBACZ,GAAG,UAAU;iBACd,CAAC,CAAC;gBAEH,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC1B,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAClC,CAAC;YAAC,MAAM,CAAC;gBACP,0DAA0D;gBAC1D,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC1G,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;oBACjD,KAAa,CAAC,KAAK,EAAE,CAAC;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,MAAM,CACV,KAAmB,EACnB,CAAS,EACT,QAAiB;QAEjB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,aAAa,CAAC,gDAAgD,EAAE,kBAAkB,CAAC,CAAC;QAChG,CAAC;QAED,iDAAiD;QACjD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC7D,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,OAAO,IAAI,OAAO,CAA0C,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC9E,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5C,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;gBACxB,IAAI,EAAE,QAAQ;gBACd,OAAO;gBACP,KAAK;gBACL,CAAC;gBACD,QAAQ,EAAE,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;aAC1C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,eAAe,CACnB,KAAmB,EACnB,CAAS,EACT,sBAA8B,CAAC,EAC/B,QAAiB;QAEjB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,aAAa,CAAC,gDAAgD,EAAE,kBAAkB,CAAC,CAAC;QAChG,CAAC;QAED,yBAAyB;QACzB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC,EAAE,mBAAmB,EAAE,QAAQ,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC7D,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,OAAO,IAAI,OAAO,CAA0C,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC9E,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5C,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;gBACxB,IAAI,EAAE,QAAQ;gBACd,OAAO;gBACP,KAAK;gBACL,CAAC;gBACD,QAAQ,EAAE,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,mBAAmB,GAAG,CAAC,EAAE,EAAE,CAAC;gBAC/D,SAAS,EAAE,IAAI;gBACf,mBAAmB;aACpB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,WAAW,CACf,OAAuB,EACvB,CAAS,EACT,QAAiB;QAEjB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,aAAa,CAAC,gDAAgD,EAAE,kBAAkB,CAAC,CAAC;QAChG,CAAC;QAED,yBAAyB;QACzB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACpD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,aAAc,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CACxB,OAAuB,EACvB,CAAS,EACT,sBAA8B,CAAC,EAC/B,QAAiB;QAEjB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,aAAa,CAAC,gDAAgD,EAAE,kBAAkB,CAAC,CAAC;QAChG,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACpD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,aAAc,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC,EAAE,mBAAmB,EAAE,QAAQ,CAAC,CAAC,CAAC;QACvG,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,EAAE,mBAAmB,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC7F,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAClB,QAAsD,EACtD,YAAqB,EACrB,QAAiB;QAEjB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;gBACxB,IAAI,EAAE,aAAa;gBACnB,QAAQ;aACT,CAAC,CAAC;YACH,IAAI,YAAY,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBACzD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;oBACxB,IAAI,EAAE,kBAAkB;oBACxB,YAAY;oBACZ,QAAQ;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO;QACL,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;gBACnD,MAAM,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC;YAC7C,CAAC;YACD,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;CACF"}
@@ -1 +1 @@
1
- {"version":3,"file":"JsDistanceBackend.d.ts","sourceRoot":"","sources":["../../src/backends/JsDistanceBackend.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAYH,wBAAgB,cAAc,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CA2BvE;AAED,wBAAgB,aAAa,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAoCtE;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,CAoCxD;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,aAAa,UAAQ,EAAE,aAAa,UAAQ,GAAG,MAAM,CAuCzH"}
1
+ {"version":3,"file":"JsDistanceBackend.d.ts","sourceRoot":"","sources":["../../src/backends/JsDistanceBackend.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAcH,wBAAgB,cAAc,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CA2BvE;AAED,wBAAgB,aAAa,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAoCtE;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,CAoCxD;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,aAAa,UAAQ,EAAE,aAAa,UAAQ,GAAG,MAAM,CAuCzH"}