verso-db 0.1.5 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/CHANGELOG.md +13 -0
  2. package/README.md +13 -7
  3. package/dist/BinaryHeap.d.ts +11 -1
  4. package/dist/BinaryHeap.d.ts.map +1 -1
  5. package/dist/BinaryHeap.js +138 -0
  6. package/dist/BinaryHeap.js.map +1 -0
  7. package/dist/Collection.d.ts +30 -4
  8. package/dist/Collection.d.ts.map +1 -1
  9. package/dist/Collection.js +1186 -0
  10. package/dist/Collection.js.map +1 -0
  11. package/dist/HNSWIndex.d.ts +59 -0
  12. package/dist/HNSWIndex.d.ts.map +1 -1
  13. package/dist/HNSWIndex.js +2818 -0
  14. package/dist/HNSWIndex.js.map +1 -0
  15. package/dist/MaxBinaryHeap.d.ts +2 -64
  16. package/dist/MaxBinaryHeap.d.ts.map +1 -1
  17. package/dist/MaxBinaryHeap.js +5 -0
  18. package/dist/MaxBinaryHeap.js.map +1 -0
  19. package/dist/SearchWorker.d.ts +57 -4
  20. package/dist/SearchWorker.d.ts.map +1 -1
  21. package/dist/SearchWorker.js +573 -0
  22. package/dist/SearchWorker.js.map +1 -0
  23. package/dist/VectorDB.d.ts.map +1 -1
  24. package/dist/VectorDB.js +246 -0
  25. package/dist/VectorDB.js.map +1 -0
  26. package/dist/WorkerPool.d.ts +32 -2
  27. package/dist/WorkerPool.d.ts.map +1 -1
  28. package/dist/WorkerPool.js +266 -0
  29. package/dist/WorkerPool.js.map +1 -0
  30. package/dist/backends/JsDistanceBackend.d.ts.map +1 -1
  31. package/dist/backends/JsDistanceBackend.js +163 -0
  32. package/dist/backends/JsDistanceBackend.js.map +1 -0
  33. package/dist/encoding/DeltaEncoder.d.ts +2 -2
  34. package/dist/encoding/DeltaEncoder.d.ts.map +1 -1
  35. package/dist/encoding/DeltaEncoder.js +199 -0
  36. package/dist/encoding/DeltaEncoder.js.map +1 -0
  37. package/dist/errors.js +97 -0
  38. package/dist/errors.js.map +1 -0
  39. package/dist/index.d.ts +3 -3
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +61 -42
  42. package/dist/index.js.map +1 -9
  43. package/dist/presets.js +205 -0
  44. package/dist/presets.js.map +1 -0
  45. package/dist/quantization/ScalarQuantizer.d.ts +0 -34
  46. package/dist/quantization/ScalarQuantizer.d.ts.map +1 -1
  47. package/dist/quantization/ScalarQuantizer.js +346 -0
  48. package/dist/quantization/ScalarQuantizer.js.map +1 -0
  49. package/dist/storage/BatchWriter.js +351 -0
  50. package/dist/storage/BatchWriter.js.map +1 -0
  51. package/dist/storage/BunStorageBackend.d.ts +7 -3
  52. package/dist/storage/BunStorageBackend.d.ts.map +1 -1
  53. package/dist/storage/BunStorageBackend.js +182 -0
  54. package/dist/storage/BunStorageBackend.js.map +1 -0
  55. package/dist/storage/MemoryBackend.js +109 -0
  56. package/dist/storage/MemoryBackend.js.map +1 -0
  57. package/dist/storage/OPFSBackend.d.ts.map +1 -1
  58. package/dist/storage/OPFSBackend.js +325 -0
  59. package/dist/storage/OPFSBackend.js.map +1 -0
  60. package/dist/storage/StorageBackend.js +12 -0
  61. package/dist/storage/StorageBackend.js.map +1 -0
  62. package/dist/storage/WriteAheadLog.js +321 -0
  63. package/dist/storage/WriteAheadLog.js.map +1 -0
  64. package/dist/storage/createStorageBackend.d.ts +4 -0
  65. package/dist/storage/createStorageBackend.d.ts.map +1 -1
  66. package/dist/storage/createStorageBackend.js +119 -0
  67. package/dist/storage/createStorageBackend.js.map +1 -0
  68. package/{src/storage/index.ts → dist/storage/index.js} +7 -27
  69. package/dist/storage/index.js.map +1 -0
  70. package/dist/storage/nodeFsRuntime.d.ts +14 -0
  71. package/dist/storage/nodeFsRuntime.d.ts.map +1 -0
  72. package/dist/storage/nodeFsRuntime.js +105 -0
  73. package/dist/storage/nodeFsRuntime.js.map +1 -0
  74. package/package.json +9 -7
  75. package/src/BinaryHeap.ts +0 -136
  76. package/src/Collection.ts +0 -1262
  77. package/src/HNSWIndex.ts +0 -2894
  78. package/src/MaxBinaryHeap.ts +0 -181
  79. package/src/SearchWorker.ts +0 -264
  80. package/src/VectorDB.ts +0 -319
  81. package/src/WorkerPool.ts +0 -222
  82. package/src/backends/JsDistanceBackend.ts +0 -171
  83. package/src/encoding/DeltaEncoder.ts +0 -236
  84. package/src/errors.ts +0 -110
  85. package/src/index.ts +0 -106
  86. package/src/presets.ts +0 -229
  87. package/src/quantization/ScalarQuantizer.ts +0 -487
  88. package/src/storage/BatchWriter.ts +0 -420
  89. package/src/storage/BunStorageBackend.ts +0 -199
  90. package/src/storage/MemoryBackend.ts +0 -122
  91. package/src/storage/OPFSBackend.ts +0 -348
  92. package/src/storage/StorageBackend.ts +0 -74
  93. package/src/storage/WriteAheadLog.ts +0 -379
  94. package/src/storage/createStorageBackend.ts +0 -137
@@ -0,0 +1,351 @@
1
+ /**
2
+ * Batch Write Coalescing for Storage Backends
3
+ *
4
+ * Buffers writes in memory and flushes them in batches to reduce I/O operations.
5
+ * This is especially beneficial for:
6
+ * - OPFS where each write has overhead
7
+ * - IndexedDB where transactions have cost
8
+ * - Network storage where round-trips are expensive
9
+ *
10
+ * Features:
11
+ * - Configurable flush thresholds (count and size)
12
+ * - Automatic flushing when thresholds are reached
13
+ * - Explicit flush for durability guarantees
14
+ * - Coalesces multiple writes to the same key
15
+ */
16
+ /**
17
+ * BatchWriter - Coalesces multiple writes into batched flushes
18
+ */
19
+ export class BatchWriter {
20
+ backend;
21
+ pendingWrites = new Map();
22
+ pendingAppends = new Map();
23
+ pendingBytes = 0;
24
+ maxPendingWrites;
25
+ maxPendingBytes;
26
+ autoFlushInterval;
27
+ flushTimer = null;
28
+ isFlushing = false;
29
+ flushPromise = null;
30
+ constructor(backend, options = {}) {
31
+ this.backend = backend;
32
+ this.maxPendingWrites = options.maxPendingWrites ?? 100;
33
+ this.maxPendingBytes = options.maxPendingBytes ?? 1024 * 1024; // 1MB
34
+ this.autoFlushInterval = options.autoFlushInterval ?? 0;
35
+ if (this.autoFlushInterval > 0) {
36
+ this.startAutoFlush();
37
+ }
38
+ }
39
+ /**
40
+ * Write data to a key (buffered)
41
+ * Multiple writes to the same key will coalesce to the last value
42
+ */
43
+ async write(key, data) {
44
+ const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
45
+ const bytesCopy = new Uint8Array(bytes.length);
46
+ bytesCopy.set(bytes);
47
+ // If there's already a pending write, subtract its size
48
+ const existing = this.pendingWrites.get(key);
49
+ if (existing) {
50
+ this.pendingBytes -= existing.data.length;
51
+ }
52
+ // Clear any pending appends for this key (write overwrites)
53
+ const existingAppends = this.pendingAppends.get(key);
54
+ if (existingAppends) {
55
+ for (const append of existingAppends) {
56
+ this.pendingBytes -= append.length;
57
+ }
58
+ this.pendingAppends.delete(key);
59
+ }
60
+ this.pendingWrites.set(key, {
61
+ key,
62
+ data: bytesCopy,
63
+ append: false
64
+ });
65
+ this.pendingBytes += bytesCopy.length;
66
+ await this.checkThresholds();
67
+ }
68
+ /**
69
+ * Append data to a key (buffered)
70
+ * Multiple appends to the same key will be concatenated
71
+ */
72
+ async append(key, data) {
73
+ const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
74
+ const bytesCopy = new Uint8Array(bytes.length);
75
+ bytesCopy.set(bytes);
76
+ // If there's a pending write, append to it instead
77
+ const existingWrite = this.pendingWrites.get(key);
78
+ if (existingWrite) {
79
+ // Concatenate with existing write
80
+ const newData = new Uint8Array(existingWrite.data.length + bytesCopy.length);
81
+ newData.set(existingWrite.data);
82
+ newData.set(bytesCopy, existingWrite.data.length);
83
+ this.pendingBytes -= existingWrite.data.length;
84
+ this.pendingBytes += newData.length;
85
+ existingWrite.data = newData;
86
+ }
87
+ else {
88
+ // Add to pending appends
89
+ let appends = this.pendingAppends.get(key);
90
+ if (!appends) {
91
+ appends = [];
92
+ this.pendingAppends.set(key, appends);
93
+ }
94
+ appends.push(bytesCopy);
95
+ this.pendingBytes += bytesCopy.length;
96
+ }
97
+ await this.checkThresholds();
98
+ }
99
+ /**
100
+ * Delete a key (buffered)
101
+ * Clears any pending writes/appends for this key
102
+ */
103
+ async delete(key) {
104
+ // Clear pending operations
105
+ const existing = this.pendingWrites.get(key);
106
+ if (existing) {
107
+ this.pendingBytes -= existing.data.length;
108
+ this.pendingWrites.delete(key);
109
+ }
110
+ const existingAppends = this.pendingAppends.get(key);
111
+ if (existingAppends) {
112
+ for (const append of existingAppends) {
113
+ this.pendingBytes -= append.length;
114
+ }
115
+ this.pendingAppends.delete(key);
116
+ }
117
+ // Perform delete immediately (can't batch deletes effectively)
118
+ await this.backend.delete(key);
119
+ }
120
+ /**
121
+ * Read data from a key
122
+ * Returns pending data if available, otherwise reads from backend
123
+ */
124
+ async read(key) {
125
+ // Check for pending write
126
+ const pending = this.pendingWrites.get(key);
127
+ if (pending) {
128
+ const buffer = pending.data.buffer.slice(pending.data.byteOffset, pending.data.byteOffset + pending.data.length);
129
+ return buffer;
130
+ }
131
+ // Check for pending appends - need to combine with existing data.
132
+ // Re-validate pending state after backend read to avoid returning stale
133
+ // appends that may have already been flushed.
134
+ while (true) {
135
+ const appends = this.pendingAppends.get(key);
136
+ if (!appends || appends.length === 0) {
137
+ // No pending data, read from backend
138
+ return this.backend.read(key);
139
+ }
140
+ const appendCount = appends.length;
141
+ const existing = await this.backend.read(key);
142
+ // A write supersedes appends and should take precedence if it appeared
143
+ // while we were waiting on backend I/O.
144
+ const latestWrite = this.pendingWrites.get(key);
145
+ if (latestWrite) {
146
+ const buffer = latestWrite.data.buffer.slice(latestWrite.data.byteOffset, latestWrite.data.byteOffset + latestWrite.data.length);
147
+ return buffer;
148
+ }
149
+ const latestAppends = this.pendingAppends.get(key);
150
+ if (!latestAppends || latestAppends !== appends || latestAppends.length !== appendCount) {
151
+ // Pending append state changed (flushed/new appends/etc.), retry with
152
+ // a fresh snapshot to avoid duplicate or stale data.
153
+ continue;
154
+ }
155
+ const existingBytes = existing ? new Uint8Array(existing) : new Uint8Array(0);
156
+ // Calculate total size
157
+ let totalSize = existingBytes.length;
158
+ for (let i = 0; i < appendCount; i++) {
159
+ totalSize += latestAppends[i].length;
160
+ }
161
+ // Combine
162
+ const result = new Uint8Array(totalSize);
163
+ result.set(existingBytes);
164
+ let offset = existingBytes.length;
165
+ for (let i = 0; i < appendCount; i++) {
166
+ const append = latestAppends[i];
167
+ result.set(append, offset);
168
+ offset += append.length;
169
+ }
170
+ return result.buffer;
171
+ }
172
+ }
173
+ /**
174
+ * Check if thresholds are exceeded and flush if needed
175
+ */
176
+ async checkThresholds() {
177
+ const totalWrites = this.pendingWrites.size + this.pendingAppends.size;
178
+ if (totalWrites >= this.maxPendingWrites || this.pendingBytes >= this.maxPendingBytes) {
179
+ await this.flush();
180
+ }
181
+ }
182
+ /**
183
+ * Flush all pending writes to the backend
184
+ */
185
+ async flush() {
186
+ // If already flushing, wait for it to complete then re-check
187
+ if (this.isFlushing && this.flushPromise) {
188
+ await this.flushPromise;
189
+ // Re-check: writes may have arrived during the previous flush
190
+ if (this.pendingWrites.size === 0 && this.pendingAppends.size === 0) {
191
+ return;
192
+ }
193
+ }
194
+ if (this.pendingWrites.size === 0 && this.pendingAppends.size === 0) {
195
+ return;
196
+ }
197
+ this.isFlushing = true;
198
+ this.flushPromise = this.doFlush();
199
+ try {
200
+ await this.flushPromise;
201
+ }
202
+ finally {
203
+ this.isFlushing = false;
204
+ this.flushPromise = null;
205
+ }
206
+ }
207
+ async doFlush() {
208
+ // Capture current pending writes and clear
209
+ const writes = Array.from(this.pendingWrites.entries());
210
+ const appendEntries = Array.from(this.pendingAppends.entries());
211
+ this.pendingWrites.clear();
212
+ this.pendingAppends.clear();
213
+ this.pendingBytes = 0;
214
+ // Execute writes and appends sequentially per-key to avoid partial-failure
215
+ // ambiguity with Promise.all (where we can't tell which appends succeeded,
216
+ // risking duplicate data on retry since append is not idempotent).
217
+ const failedWrites = [];
218
+ const failedAppends = [];
219
+ let firstError = null;
220
+ for (const [key, write] of writes) {
221
+ try {
222
+ await this.backend.write(write.key, write.data);
223
+ }
224
+ catch (err) {
225
+ if (!firstError)
226
+ firstError = err;
227
+ failedWrites.push([key, write]);
228
+ }
229
+ }
230
+ for (const [key, chunks] of appendEntries) {
231
+ // Concatenate all appends for this key
232
+ let totalSize = 0;
233
+ for (const chunk of chunks) {
234
+ totalSize += chunk.length;
235
+ }
236
+ const combined = new Uint8Array(totalSize);
237
+ let offset = 0;
238
+ for (const chunk of chunks) {
239
+ combined.set(chunk, offset);
240
+ offset += chunk.length;
241
+ }
242
+ try {
243
+ await this.backend.append(key, combined);
244
+ }
245
+ catch (err) {
246
+ if (!firstError)
247
+ firstError = err;
248
+ failedAppends.push([key, chunks]);
249
+ }
250
+ }
251
+ if (firstError) {
252
+ // Only restore entries that actually failed — successful ones are persisted.
253
+ let restoredBytes = 0;
254
+ for (const [key, write] of failedWrites) {
255
+ if (!this.pendingWrites.has(key)) {
256
+ this.pendingWrites.set(key, write);
257
+ restoredBytes += write.data.length;
258
+ }
259
+ }
260
+ for (const [key, chunks] of failedAppends) {
261
+ // If a newer pending write exists for this key, it supersedes old appends.
262
+ // Restoring those stale appends would violate write ordering and corrupt data.
263
+ if (this.pendingWrites.has(key)) {
264
+ continue;
265
+ }
266
+ const existing = this.pendingAppends.get(key);
267
+ if (existing) {
268
+ // Prepend old chunks before any new ones
269
+ this.pendingAppends.set(key, [...chunks, ...existing]);
270
+ }
271
+ else {
272
+ this.pendingAppends.set(key, chunks);
273
+ }
274
+ for (const chunk of chunks) {
275
+ restoredBytes += chunk.length;
276
+ }
277
+ }
278
+ this.pendingBytes += restoredBytes;
279
+ throw firstError;
280
+ }
281
+ }
282
+ /**
283
+ * Get statistics about pending writes
284
+ */
285
+ getStats() {
286
+ return {
287
+ pendingWrites: this.pendingWrites.size,
288
+ pendingAppends: this.pendingAppends.size,
289
+ pendingBytes: this.pendingBytes,
290
+ maxPendingWrites: this.maxPendingWrites,
291
+ maxPendingBytes: this.maxPendingBytes
292
+ };
293
+ }
294
+ /**
295
+ * Check if there are pending writes
296
+ */
297
+ hasPendingWrites() {
298
+ return this.pendingWrites.size > 0 || this.pendingAppends.size > 0;
299
+ }
300
+ /**
301
+ * Start auto-flush timer
302
+ */
303
+ startAutoFlush() {
304
+ if (this.flushTimer)
305
+ return;
306
+ this.flushTimer = setInterval(async () => {
307
+ if (this.hasPendingWrites()) {
308
+ try {
309
+ await this.flush();
310
+ }
311
+ catch {
312
+ // Preserve pending data and retry on next interval/explicit flush.
313
+ }
314
+ }
315
+ }, this.autoFlushInterval);
316
+ // Allow the process to exit even if the timer is still running.
317
+ // Without this, the timer keeps the event loop alive indefinitely.
318
+ if (typeof this.flushTimer === 'object' && 'unref' in this.flushTimer) {
319
+ this.flushTimer.unref();
320
+ }
321
+ }
322
+ /**
323
+ * Stop auto-flush timer
324
+ */
325
+ stopAutoFlush() {
326
+ if (this.flushTimer) {
327
+ clearInterval(this.flushTimer);
328
+ this.flushTimer = null;
329
+ }
330
+ }
331
+ /**
332
+ * Close the batch writer, flushing any pending writes
333
+ */
334
+ async close() {
335
+ this.stopAutoFlush();
336
+ await this.flush();
337
+ }
338
+ /**
339
+ * Get the underlying storage backend
340
+ */
341
+ getBackend() {
342
+ return this.backend;
343
+ }
344
+ }
345
+ /**
346
+ * Create a batch writer that wraps an existing storage backend
347
+ */
348
+ export function createBatchWriter(backend, options) {
349
+ return new BatchWriter(backend, options);
350
+ }
351
+ //# sourceMappingURL=BatchWriter.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"BatchWriter.js","sourceRoot":"","sources":["../../src/storage/BatchWriter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAmBH;;GAEG;AACH,MAAM,OAAO,WAAW;IACd,OAAO,CAAiB;IACxB,aAAa,GAA8B,IAAI,GAAG,EAAE,CAAC;IACrD,cAAc,GAA8B,IAAI,GAAG,EAAE,CAAC;IACtD,YAAY,GAAW,CAAC,CAAC;IACzB,gBAAgB,CAAS;IACzB,eAAe,CAAS;IACxB,iBAAiB,CAAS;IAC1B,UAAU,GAAyC,IAAI,CAAC;IACxD,UAAU,GAAY,KAAK,CAAC;IAC5B,YAAY,GAAyB,IAAI,CAAC;IAElD,YAAY,OAAuB,EAAE,UAA8B,EAAE;QACnE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,GAAG,CAAC;QACxD,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;QACrE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,IAA8B;QACrD,MAAM,KAAK,GAAG,IAAI,YAAY,WAAW,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAErB,wDAAwD;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,YAAY,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,4DAA4D;QAC5D,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,eAAe,EAAE,CAAC;YACpB,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;gBACrC,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;YACrC,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE;YAC1B,GAAG;YACH,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC,MAAM,CAAC;QAEtC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,IAA8B;QACtD,MAAM,KAAK,GAAG,IAAI,YAAY,WAAW,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAErB,mDAAmD;QACnD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,aAAa,EAAE,CAAC;YAClB,kCAAkC;YAClC,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAC7E,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClD,IAAI,CAAC,YAAY,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC;YAC/C,IAAI,CAAC,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;YACpC,aAAa,CAAC,IAAI,GAAG,OAAO,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,IAAI,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC,MAAM,CAAC;QACxC,CAAC;QAED,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,YAAY,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC1C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,eAAe,EAAE,CAAC;YACpB,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;gBACrC,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;YACrC,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,+DAA+D;QAC/D,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI,CAAC,GAAW;QACpB,0BAA0B;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CACtC,OAAO,CAAC,IAAI,CAAC,UAAU,EACvB,OAAO,CAAC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAC9C,CAAC;YACF,OAAO,MAAqB,CAAC;QAC/B,CAAC;QAED,kEAAkE;QAClE,wEAAwE;QACxE,8CAA8C;QAC9C,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,qCAAqC;gBACrC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;YAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAE9C,uEAAuE;YACvE,wCAAwC;YACxC,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAC1C,WAAW,CAAC,IAAI,CAAC,UAAU,EAC3B,WAAW,CAAC,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CACtD,CAAC;gBACF,OAAO,MAAqB,CAAC;YAC/B,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,CAAC,aAAa,IAAI,aAAa,KAAK,OAAO,IAAI,aAAa,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACxF,sEAAsE;gBACtE,qDAAqD;gBACrD,SAAS;YACX,CAAC;YAED,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;YAE9E,uBAAuB;YACvB,IAAI,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,SAAS,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACvC,CAAC;YAED,UAAU;YACV,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YACzC,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAC1B,IAAI,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;YAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC3B,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;YAC1B,CAAC;YAED,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe;QAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;QAEvE,IAAI,WAAW,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACtF,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,6DAA6D;QAC7D,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,YAAY,CAAC;YACxB,8DAA8D;YAC9D,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACpE,OAAO;YACT,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACpE,OAAO;QACT,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,YAAY,CAAC;QAC1B,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACxD,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC;QAEhE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAEtB,2EAA2E;QAC3E,2EAA2E;QAC3E,mEAAmE;QACnE,MAAM,YAAY,GAAkC,EAAE,CAAC;QACvD,MAAM,aAAa,GAAkC,EAAE,CAAC;QACxD,IAAI,UAAU,GAAY,IAAI,CAAC;QAE/B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAClD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,UAAU;oBAAE,UAAU,GAAG,GAAG,CAAC;gBAClC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;YAC1C,uCAAuC;YACvC,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;YAC5B,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAC3C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,UAAU;oBAAE,UAAU,GAAG,GAAG,CAAC;gBAClC,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAED,IAAI,UAAU,EAAE,CAAC;YACf,6EAA6E;YAC7E,IAAI,aAAa,GAAG,CAAC,CAAC;YACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,YAAY,EAAE,CAAC;gBACxC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBACnC,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;gBACrC,CAAC;YACH,CAAC;YACD,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;gBAC1C,2EAA2E;gBAC3E,+EAA+E;gBAC/E,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChC,SAAS;gBACX,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9C,IAAI,QAAQ,EAAE,CAAC;oBACb,yCAAyC;oBACzC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC;gBACzD,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBACvC,CAAC;gBACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC;gBAChC,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,IAAI,aAAa,CAAC;YACnC,MAAM,UAAU,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QAON,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;YACtC,cAAc,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI;YACxC,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,eAAe,EAAE,IAAI,CAAC,eAAe;SACtC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC,CAAC;IACrE,CAAC;IAED;;OAEG;IACK,cAAc;QACpB,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAE5B,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;gBAAC,MAAM,CAAC;oBACP,mEAAmE;gBACrE,CAAC;YACH,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAE3B,gEAAgE;QAChE,mEAAmE;QACnE,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACrE,IAAI,CAAC,UAAkB,CAAC,KAAK,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAAuB,EACvB,OAA4B;IAE5B,OAAO,IAAI,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAC3C,CAAC"}
@@ -1,7 +1,8 @@
1
1
  /**
2
- * Bun Storage Backend
2
+ * Bun/Node file-system storage backend
3
3
  *
4
- * High-performance file system storage using Bun's native APIs.
4
+ * High-performance file system storage using Bun native APIs when available,
5
+ * with a Node.js filesystem fallback for runtime compatibility.
5
6
  * Features:
6
7
  * - Auto-initializing (no manual init() required)
7
8
  * - Automatic memory mapping for large files
@@ -12,12 +13,15 @@ import type { StorageBackend } from './StorageBackend';
12
13
  export declare class BunStorageBackend implements StorageBackend {
13
14
  readonly type = "bun";
14
15
  private basePath;
16
+ private nodeRuntimePromise;
15
17
  private dirCache;
16
18
  /**
17
19
  * Create a new Bun storage backend
18
20
  * @param basePath Base directory for all storage operations
19
21
  */
20
22
  constructor(basePath?: string);
23
+ private isBrowserRuntime;
24
+ private getNodeRuntime;
21
25
  /**
22
26
  * Ensure directory exists, using cache to avoid redundant mkdir calls
23
27
  */
@@ -52,7 +56,7 @@ export declare class BunStorageBackend implements StorageBackend {
52
56
  * Note: Callers should check exists() before calling stream(),
53
57
  * as Bun.file().stream() on a non-existent file will error at read time.
54
58
  */
55
- stream(key: string): ReadableStream<Uint8Array> | null;
59
+ stream(key: string): Promise<ReadableStream<Uint8Array> | null>;
56
60
  /**
57
61
  * Get the base path
58
62
  */
@@ -1 +1 @@
1
- {"version":3,"file":"BunStorageBackend.d.ts","sourceRoot":"","sources":["../../src/storage/BunStorageBackend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAEvD,qBAAa,iBAAkB,YAAW,cAAc;IACtD,QAAQ,CAAC,IAAI,SAAS;IACtB,OAAO,CAAC,QAAQ,CAAS;IAGzB,OAAO,CAAC,QAAQ,CAA0B;IAE1C;;;OAGG;gBACS,QAAQ,GAAE,MAA0B;IAIhD;;OAEG;YACW,SAAS;IAkBvB;;;OAGG;IACH,OAAO,CAAC,WAAW;IASnB;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAW9C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAajE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAWlE,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAMrC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAoBxC,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B;;OAEG;IACG,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWxC;;;;OAIG;IACH,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAKtD;;OAEG;IACH,WAAW,IAAI,MAAM;CAGtB"}
1
+ {"version":3,"file":"BunStorageBackend.d.ts","sourceRoot":"","sources":["../../src/storage/BunStorageBackend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAGvD,qBAAa,iBAAkB,YAAW,cAAc;IACtD,QAAQ,CAAC,IAAI,SAAS;IACtB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,kBAAkB,CAA0D;IAGpF,OAAO,CAAC,QAAQ,CAA0B;IAE1C;;;OAGG;gBACS,QAAQ,GAAE,MAA0B;IAIhD,OAAO,CAAC,gBAAgB;YAIV,cAAc;IAU5B;;OAEG;YACW,SAAS;IAmBvB;;;OAGG;YACW,WAAW;IAKzB;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAKrB,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAe9C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAYjE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAWlE,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWrC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAMxC,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAa3C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B;;OAEG;IACG,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWxC;;;;OAIG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IASrE;;OAEG;IACH,WAAW,IAAI,MAAM;CAGtB"}
@@ -0,0 +1,182 @@
1
+ /**
2
+ * Bun/Node file-system storage backend
3
+ *
4
+ * High-performance file system storage using Bun native APIs when available,
5
+ * with a Node.js filesystem fallback for runtime compatibility.
6
+ * Features:
7
+ * - Auto-initializing (no manual init() required)
8
+ * - Automatic memory mapping for large files
9
+ * - Efficient async I/O with O(1) append
10
+ * - Native TypedArray support
11
+ */
12
+ import { StorageError } from '../errors.js';
13
+ export class BunStorageBackend {
14
+ type = 'bun';
15
+ basePath;
16
+ nodeRuntimePromise = null;
17
+ // Cache of directories we've already ensured exist - avoids redundant mkdir calls
18
+ // Profiling showed 5-10% write overhead from mkdir on every write
19
+ dirCache = new Set();
20
+ /**
21
+ * Create a new Bun storage backend
22
+ * @param basePath Base directory for all storage operations
23
+ */
24
+ constructor(basePath = './vectordb_data') {
25
+ this.basePath = basePath;
26
+ }
27
+ isBrowserRuntime() {
28
+ return typeof window !== 'undefined' && typeof navigator !== 'undefined';
29
+ }
30
+ async getNodeRuntime() {
31
+ if (this.isBrowserRuntime()) {
32
+ throw new StorageError('init', 'BunStorageBackend is only available in Bun or Node.js runtimes');
33
+ }
34
+ if (!this.nodeRuntimePromise) {
35
+ this.nodeRuntimePromise = import('./nodeFsRuntime.js');
36
+ }
37
+ return this.nodeRuntimePromise;
38
+ }
39
+ /**
40
+ * Ensure directory exists, using cache to avoid redundant mkdir calls
41
+ */
42
+ async ensureDir(dir) {
43
+ if (this.dirCache.has(dir))
44
+ return;
45
+ try {
46
+ const runtime = await this.getNodeRuntime();
47
+ await runtime.ensureDir(dir);
48
+ this.dirCache.add(dir);
49
+ }
50
+ catch (err) {
51
+ // EEXIST is fine - directory already exists
52
+ if (err instanceof Error && err.code === 'EEXIST') {
53
+ this.dirCache.add(dir);
54
+ return;
55
+ }
56
+ // Re-throw real errors (permissions, disk full, etc.) so callers get
57
+ // a clear failure instead of a confusing "file not found" on the
58
+ // subsequent write attempt.
59
+ throw err;
60
+ }
61
+ }
62
+ /**
63
+ * Get the full path for a key.
64
+ * Validates that the resolved path stays within basePath to prevent path traversal.
65
+ */
66
+ async getFullPath(key) {
67
+ const runtime = await this.getNodeRuntime();
68
+ return runtime.resolveStoragePath(this.basePath, key);
69
+ }
70
+ /**
71
+ * Ensure base directory exists (optional - operations auto-initialize)
72
+ * @deprecated No longer required - write/append create directories automatically
73
+ */
74
+ async init() {
75
+ const runtime = await this.getNodeRuntime();
76
+ await runtime.ensureDir(runtime.resolveBasePath(this.basePath)).catch(() => { });
77
+ }
78
+ async read(key) {
79
+ const fullPath = await this.getFullPath(key);
80
+ if (typeof Bun !== 'undefined') {
81
+ const file = Bun.file(fullPath);
82
+ // Bun.file().size is 0 for non-existent files — avoids TOCTOU race with exists()
83
+ if (file.size === 0) {
84
+ return null;
85
+ }
86
+ return file.arrayBuffer();
87
+ }
88
+ const runtime = await this.getNodeRuntime();
89
+ return runtime.read(fullPath);
90
+ }
91
+ async write(key, data) {
92
+ const runtime = await this.getNodeRuntime();
93
+ const fullPath = await this.getFullPath(key);
94
+ // Ensure parent directory exists (cached to avoid redundant syscalls)
95
+ await this.ensureDir(runtime.dirname(fullPath));
96
+ // Atomic write: write to temp file then rename.
97
+ // If the process crashes mid-write, the original file is untouched.
98
+ await runtime.writeAtomic(fullPath, data);
99
+ }
100
+ async append(key, data) {
101
+ const runtime = await this.getNodeRuntime();
102
+ const fullPath = await this.getFullPath(key);
103
+ // Ensure parent directory exists (cached to avoid redundant syscalls)
104
+ await this.ensureDir(runtime.dirname(fullPath));
105
+ // Use true O(1) append instead of O(n) read-modify-write
106
+ await runtime.append(fullPath, data);
107
+ }
108
+ async delete(key) {
109
+ const fullPath = await this.getFullPath(key);
110
+ const runtime = await this.getNodeRuntime();
111
+ await runtime.deleteFile(fullPath);
112
+ }
113
+ async exists(key) {
114
+ const fullPath = await this.getFullPath(key);
115
+ if (typeof Bun !== 'undefined') {
116
+ const file = Bun.file(fullPath);
117
+ return file.exists();
118
+ }
119
+ const runtime = await this.getNodeRuntime();
120
+ return runtime.exists(fullPath);
121
+ }
122
+ async list(prefix) {
123
+ const runtime = await this.getNodeRuntime();
124
+ const searchPath = prefix ? await this.getFullPath(prefix) : runtime.resolveBasePath(this.basePath);
125
+ return runtime.list(this.basePath, searchPath);
126
+ }
127
+ async mkdir(dirPath) {
128
+ const runtime = await this.getNodeRuntime();
129
+ const fullPath = await this.getFullPath(dirPath);
130
+ try {
131
+ await runtime.ensureDir(fullPath);
132
+ }
133
+ catch (err) {
134
+ if (err instanceof Error && err.code === 'EEXIST') {
135
+ return;
136
+ }
137
+ throw err;
138
+ }
139
+ }
140
+ /**
141
+ * Delete all data in the storage directory
142
+ */
143
+ async clear() {
144
+ const runtime = await this.getNodeRuntime();
145
+ const resolvedBase = await runtime.clear(this.basePath);
146
+ // Clear directory cache since directories were deleted
147
+ this.dirCache.clear();
148
+ this.dirCache.add(resolvedBase);
149
+ }
150
+ /**
151
+ * Get file size without reading the entire file
152
+ */
153
+ async size(key) {
154
+ const fullPath = await this.getFullPath(key);
155
+ if (typeof Bun !== 'undefined') {
156
+ // Bun.file().size returns 0 for non-existent files — no exists() check needed
157
+ return Bun.file(fullPath).size;
158
+ }
159
+ const runtime = await this.getNodeRuntime();
160
+ return runtime.size(fullPath);
161
+ }
162
+ /**
163
+ * Read a file as a stream (for large files).
164
+ * Note: Callers should check exists() before calling stream(),
165
+ * as Bun.file().stream() on a non-existent file will error at read time.
166
+ */
167
+ async stream(key) {
168
+ const fullPath = await this.getFullPath(key);
169
+ if (typeof Bun !== 'undefined') {
170
+ return Bun.file(fullPath).stream();
171
+ }
172
+ const runtime = await this.getNodeRuntime();
173
+ return runtime.stream(fullPath);
174
+ }
175
+ /**
176
+ * Get the base path
177
+ */
178
+ getBasePath() {
179
+ return this.basePath;
180
+ }
181
+ }
182
+ //# sourceMappingURL=BunStorageBackend.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"BunStorageBackend.js","sourceRoot":"","sources":["../../src/storage/BunStorageBackend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,KAAK,CAAC;IACd,QAAQ,CAAS;IACjB,kBAAkB,GAAqD,IAAI,CAAC;IACpF,kFAAkF;IAClF,kEAAkE;IAC1D,QAAQ,GAAgB,IAAI,GAAG,EAAE,CAAC;IAE1C;;;OAGG;IACH,YAAY,WAAmB,iBAAiB;QAC9C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAEO,gBAAgB;QACtB,OAAO,OAAO,MAAM,KAAK,WAAW,IAAI,OAAO,SAAS,KAAK,WAAW,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,YAAY,CAAC,MAAM,EAAE,gEAAgE,CAAC,CAAC;QACnG,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC7B,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QACnC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5C,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,4CAA4C;YAC5C,IAAI,GAAG,YAAY,KAAK,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7E,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACvB,OAAO;YACT,CAAC;YACD,qEAAqE;YACrE,iEAAiE;YACjE,4BAA4B;YAC5B,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACxD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAClF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAW;QACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,OAAO,GAAG,KAAK,WAAW,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,iFAAiF;YACjF,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,IAA8B;QACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAE7C,sEAAsE;QACtE,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAEhD,gDAAgD;QAChD,oEAAoE;QACpE,MAAM,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,IAA8B;QACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAE7C,sEAAsE;QACtE,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAEhD,yDAAyD;QACzD,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,OAAO,GAAG,KAAK,WAAW,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAe;QACxB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpG,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACjD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,IAAI,GAAG,YAAY,KAAK,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7E,OAAO;YACT,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,uDAAuD;QACvD,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,GAAW;QACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,OAAO,GAAG,KAAK,WAAW,EAAE,CAAC;YAC/B,8EAA8E;YAC9E,OAAO,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;QACjC,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,OAAO,GAAG,KAAK,WAAW,EAAE,CAAC;YAC/B,OAAO,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;QACrC,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;CACF"}