verso-db 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +13 -7
- package/dist/BinaryHeap.d.ts +11 -1
- package/dist/BinaryHeap.d.ts.map +1 -1
- package/dist/BinaryHeap.js +138 -0
- package/dist/BinaryHeap.js.map +1 -0
- package/dist/Collection.d.ts +30 -4
- package/dist/Collection.d.ts.map +1 -1
- package/dist/Collection.js +1186 -0
- package/dist/Collection.js.map +1 -0
- package/dist/HNSWIndex.d.ts +59 -0
- package/dist/HNSWIndex.d.ts.map +1 -1
- package/dist/HNSWIndex.js +2818 -0
- package/dist/HNSWIndex.js.map +1 -0
- package/dist/MaxBinaryHeap.d.ts +2 -64
- package/dist/MaxBinaryHeap.d.ts.map +1 -1
- package/dist/MaxBinaryHeap.js +5 -0
- package/dist/MaxBinaryHeap.js.map +1 -0
- package/dist/SearchWorker.d.ts +57 -4
- package/dist/SearchWorker.d.ts.map +1 -1
- package/dist/SearchWorker.js +573 -0
- package/dist/SearchWorker.js.map +1 -0
- package/dist/VectorDB.d.ts.map +1 -1
- package/dist/VectorDB.js +246 -0
- package/dist/VectorDB.js.map +1 -0
- package/dist/WorkerPool.d.ts +32 -2
- package/dist/WorkerPool.d.ts.map +1 -1
- package/dist/WorkerPool.js +266 -0
- package/dist/WorkerPool.js.map +1 -0
- package/dist/backends/JsDistanceBackend.d.ts.map +1 -1
- package/dist/backends/JsDistanceBackend.js +163 -0
- package/dist/backends/JsDistanceBackend.js.map +1 -0
- package/dist/encoding/DeltaEncoder.d.ts +2 -2
- package/dist/encoding/DeltaEncoder.d.ts.map +1 -1
- package/dist/encoding/DeltaEncoder.js +199 -0
- package/dist/encoding/DeltaEncoder.js.map +1 -0
- package/dist/errors.js +97 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +61 -42
- package/dist/index.js.map +1 -9
- package/dist/presets.js +205 -0
- package/dist/presets.js.map +1 -0
- package/dist/quantization/ScalarQuantizer.d.ts +0 -34
- package/dist/quantization/ScalarQuantizer.d.ts.map +1 -1
- package/dist/quantization/ScalarQuantizer.js +346 -0
- package/dist/quantization/ScalarQuantizer.js.map +1 -0
- package/dist/storage/BatchWriter.js +351 -0
- package/dist/storage/BatchWriter.js.map +1 -0
- package/dist/storage/BunStorageBackend.d.ts +7 -3
- package/dist/storage/BunStorageBackend.d.ts.map +1 -1
- package/dist/storage/BunStorageBackend.js +182 -0
- package/dist/storage/BunStorageBackend.js.map +1 -0
- package/dist/storage/MemoryBackend.js +109 -0
- package/dist/storage/MemoryBackend.js.map +1 -0
- package/dist/storage/OPFSBackend.d.ts.map +1 -1
- package/dist/storage/OPFSBackend.js +325 -0
- package/dist/storage/OPFSBackend.js.map +1 -0
- package/dist/storage/StorageBackend.js +12 -0
- package/dist/storage/StorageBackend.js.map +1 -0
- package/dist/storage/WriteAheadLog.js +321 -0
- package/dist/storage/WriteAheadLog.js.map +1 -0
- package/dist/storage/createStorageBackend.d.ts +4 -0
- package/dist/storage/createStorageBackend.d.ts.map +1 -1
- package/dist/storage/createStorageBackend.js +119 -0
- package/dist/storage/createStorageBackend.js.map +1 -0
- package/{src/storage/index.ts → dist/storage/index.js} +7 -27
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/nodeFsRuntime.d.ts +14 -0
- package/dist/storage/nodeFsRuntime.d.ts.map +1 -0
- package/dist/storage/nodeFsRuntime.js +105 -0
- package/dist/storage/nodeFsRuntime.js.map +1 -0
- package/package.json +9 -7
- package/src/BinaryHeap.ts +0 -136
- package/src/Collection.ts +0 -1262
- package/src/HNSWIndex.ts +0 -2894
- package/src/MaxBinaryHeap.ts +0 -181
- package/src/SearchWorker.ts +0 -264
- package/src/VectorDB.ts +0 -319
- package/src/WorkerPool.ts +0 -222
- package/src/backends/JsDistanceBackend.ts +0 -171
- package/src/encoding/DeltaEncoder.ts +0 -236
- package/src/errors.ts +0 -110
- package/src/index.ts +0 -106
- package/src/presets.ts +0 -229
- package/src/quantization/ScalarQuantizer.ts +0 -487
- package/src/storage/BatchWriter.ts +0 -420
- package/src/storage/BunStorageBackend.ts +0 -199
- package/src/storage/MemoryBackend.ts +0 -122
- package/src/storage/OPFSBackend.ts +0 -348
- package/src/storage/StorageBackend.ts +0 -74
- package/src/storage/WriteAheadLog.ts +0 -379
- package/src/storage/createStorageBackend.ts +0 -137
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batch Write Coalescing for Storage Backends
|
|
3
|
+
*
|
|
4
|
+
* Buffers writes in memory and flushes them in batches to reduce I/O operations.
|
|
5
|
+
* This is especially beneficial for:
|
|
6
|
+
* - OPFS where each write has overhead
|
|
7
|
+
* - IndexedDB where transactions have cost
|
|
8
|
+
* - Network storage where round-trips are expensive
|
|
9
|
+
*
|
|
10
|
+
* Features:
|
|
11
|
+
* - Configurable flush thresholds (count and size)
|
|
12
|
+
* - Automatic flushing when thresholds are reached
|
|
13
|
+
* - Explicit flush for durability guarantees
|
|
14
|
+
* - Coalesces multiple writes to the same key
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* BatchWriter - Coalesces multiple writes into batched flushes
|
|
18
|
+
*/
|
|
19
|
+
export class BatchWriter {
|
|
20
|
+
backend;
|
|
21
|
+
pendingWrites = new Map();
|
|
22
|
+
pendingAppends = new Map();
|
|
23
|
+
pendingBytes = 0;
|
|
24
|
+
maxPendingWrites;
|
|
25
|
+
maxPendingBytes;
|
|
26
|
+
autoFlushInterval;
|
|
27
|
+
flushTimer = null;
|
|
28
|
+
isFlushing = false;
|
|
29
|
+
flushPromise = null;
|
|
30
|
+
constructor(backend, options = {}) {
|
|
31
|
+
this.backend = backend;
|
|
32
|
+
this.maxPendingWrites = options.maxPendingWrites ?? 100;
|
|
33
|
+
this.maxPendingBytes = options.maxPendingBytes ?? 1024 * 1024; // 1MB
|
|
34
|
+
this.autoFlushInterval = options.autoFlushInterval ?? 0;
|
|
35
|
+
if (this.autoFlushInterval > 0) {
|
|
36
|
+
this.startAutoFlush();
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Write data to a key (buffered)
|
|
41
|
+
* Multiple writes to the same key will coalesce to the last value
|
|
42
|
+
*/
|
|
43
|
+
async write(key, data) {
|
|
44
|
+
const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
|
|
45
|
+
const bytesCopy = new Uint8Array(bytes.length);
|
|
46
|
+
bytesCopy.set(bytes);
|
|
47
|
+
// If there's already a pending write, subtract its size
|
|
48
|
+
const existing = this.pendingWrites.get(key);
|
|
49
|
+
if (existing) {
|
|
50
|
+
this.pendingBytes -= existing.data.length;
|
|
51
|
+
}
|
|
52
|
+
// Clear any pending appends for this key (write overwrites)
|
|
53
|
+
const existingAppends = this.pendingAppends.get(key);
|
|
54
|
+
if (existingAppends) {
|
|
55
|
+
for (const append of existingAppends) {
|
|
56
|
+
this.pendingBytes -= append.length;
|
|
57
|
+
}
|
|
58
|
+
this.pendingAppends.delete(key);
|
|
59
|
+
}
|
|
60
|
+
this.pendingWrites.set(key, {
|
|
61
|
+
key,
|
|
62
|
+
data: bytesCopy,
|
|
63
|
+
append: false
|
|
64
|
+
});
|
|
65
|
+
this.pendingBytes += bytesCopy.length;
|
|
66
|
+
await this.checkThresholds();
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Append data to a key (buffered)
|
|
70
|
+
* Multiple appends to the same key will be concatenated
|
|
71
|
+
*/
|
|
72
|
+
async append(key, data) {
|
|
73
|
+
const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
|
|
74
|
+
const bytesCopy = new Uint8Array(bytes.length);
|
|
75
|
+
bytesCopy.set(bytes);
|
|
76
|
+
// If there's a pending write, append to it instead
|
|
77
|
+
const existingWrite = this.pendingWrites.get(key);
|
|
78
|
+
if (existingWrite) {
|
|
79
|
+
// Concatenate with existing write
|
|
80
|
+
const newData = new Uint8Array(existingWrite.data.length + bytesCopy.length);
|
|
81
|
+
newData.set(existingWrite.data);
|
|
82
|
+
newData.set(bytesCopy, existingWrite.data.length);
|
|
83
|
+
this.pendingBytes -= existingWrite.data.length;
|
|
84
|
+
this.pendingBytes += newData.length;
|
|
85
|
+
existingWrite.data = newData;
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
// Add to pending appends
|
|
89
|
+
let appends = this.pendingAppends.get(key);
|
|
90
|
+
if (!appends) {
|
|
91
|
+
appends = [];
|
|
92
|
+
this.pendingAppends.set(key, appends);
|
|
93
|
+
}
|
|
94
|
+
appends.push(bytesCopy);
|
|
95
|
+
this.pendingBytes += bytesCopy.length;
|
|
96
|
+
}
|
|
97
|
+
await this.checkThresholds();
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Delete a key (buffered)
|
|
101
|
+
* Clears any pending writes/appends for this key
|
|
102
|
+
*/
|
|
103
|
+
async delete(key) {
|
|
104
|
+
// Clear pending operations
|
|
105
|
+
const existing = this.pendingWrites.get(key);
|
|
106
|
+
if (existing) {
|
|
107
|
+
this.pendingBytes -= existing.data.length;
|
|
108
|
+
this.pendingWrites.delete(key);
|
|
109
|
+
}
|
|
110
|
+
const existingAppends = this.pendingAppends.get(key);
|
|
111
|
+
if (existingAppends) {
|
|
112
|
+
for (const append of existingAppends) {
|
|
113
|
+
this.pendingBytes -= append.length;
|
|
114
|
+
}
|
|
115
|
+
this.pendingAppends.delete(key);
|
|
116
|
+
}
|
|
117
|
+
// Perform delete immediately (can't batch deletes effectively)
|
|
118
|
+
await this.backend.delete(key);
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Read data from a key
|
|
122
|
+
* Returns pending data if available, otherwise reads from backend
|
|
123
|
+
*/
|
|
124
|
+
async read(key) {
|
|
125
|
+
// Check for pending write
|
|
126
|
+
const pending = this.pendingWrites.get(key);
|
|
127
|
+
if (pending) {
|
|
128
|
+
const buffer = pending.data.buffer.slice(pending.data.byteOffset, pending.data.byteOffset + pending.data.length);
|
|
129
|
+
return buffer;
|
|
130
|
+
}
|
|
131
|
+
// Check for pending appends - need to combine with existing data.
|
|
132
|
+
// Re-validate pending state after backend read to avoid returning stale
|
|
133
|
+
// appends that may have already been flushed.
|
|
134
|
+
while (true) {
|
|
135
|
+
const appends = this.pendingAppends.get(key);
|
|
136
|
+
if (!appends || appends.length === 0) {
|
|
137
|
+
// No pending data, read from backend
|
|
138
|
+
return this.backend.read(key);
|
|
139
|
+
}
|
|
140
|
+
const appendCount = appends.length;
|
|
141
|
+
const existing = await this.backend.read(key);
|
|
142
|
+
// A write supersedes appends and should take precedence if it appeared
|
|
143
|
+
// while we were waiting on backend I/O.
|
|
144
|
+
const latestWrite = this.pendingWrites.get(key);
|
|
145
|
+
if (latestWrite) {
|
|
146
|
+
const buffer = latestWrite.data.buffer.slice(latestWrite.data.byteOffset, latestWrite.data.byteOffset + latestWrite.data.length);
|
|
147
|
+
return buffer;
|
|
148
|
+
}
|
|
149
|
+
const latestAppends = this.pendingAppends.get(key);
|
|
150
|
+
if (!latestAppends || latestAppends !== appends || latestAppends.length !== appendCount) {
|
|
151
|
+
// Pending append state changed (flushed/new appends/etc.), retry with
|
|
152
|
+
// a fresh snapshot to avoid duplicate or stale data.
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
const existingBytes = existing ? new Uint8Array(existing) : new Uint8Array(0);
|
|
156
|
+
// Calculate total size
|
|
157
|
+
let totalSize = existingBytes.length;
|
|
158
|
+
for (let i = 0; i < appendCount; i++) {
|
|
159
|
+
totalSize += latestAppends[i].length;
|
|
160
|
+
}
|
|
161
|
+
// Combine
|
|
162
|
+
const result = new Uint8Array(totalSize);
|
|
163
|
+
result.set(existingBytes);
|
|
164
|
+
let offset = existingBytes.length;
|
|
165
|
+
for (let i = 0; i < appendCount; i++) {
|
|
166
|
+
const append = latestAppends[i];
|
|
167
|
+
result.set(append, offset);
|
|
168
|
+
offset += append.length;
|
|
169
|
+
}
|
|
170
|
+
return result.buffer;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Check if thresholds are exceeded and flush if needed
|
|
175
|
+
*/
|
|
176
|
+
async checkThresholds() {
|
|
177
|
+
const totalWrites = this.pendingWrites.size + this.pendingAppends.size;
|
|
178
|
+
if (totalWrites >= this.maxPendingWrites || this.pendingBytes >= this.maxPendingBytes) {
|
|
179
|
+
await this.flush();
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Flush all pending writes to the backend
|
|
184
|
+
*/
|
|
185
|
+
async flush() {
|
|
186
|
+
// If already flushing, wait for it to complete then re-check
|
|
187
|
+
if (this.isFlushing && this.flushPromise) {
|
|
188
|
+
await this.flushPromise;
|
|
189
|
+
// Re-check: writes may have arrived during the previous flush
|
|
190
|
+
if (this.pendingWrites.size === 0 && this.pendingAppends.size === 0) {
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
if (this.pendingWrites.size === 0 && this.pendingAppends.size === 0) {
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
this.isFlushing = true;
|
|
198
|
+
this.flushPromise = this.doFlush();
|
|
199
|
+
try {
|
|
200
|
+
await this.flushPromise;
|
|
201
|
+
}
|
|
202
|
+
finally {
|
|
203
|
+
this.isFlushing = false;
|
|
204
|
+
this.flushPromise = null;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
async doFlush() {
|
|
208
|
+
// Capture current pending writes and clear
|
|
209
|
+
const writes = Array.from(this.pendingWrites.entries());
|
|
210
|
+
const appendEntries = Array.from(this.pendingAppends.entries());
|
|
211
|
+
this.pendingWrites.clear();
|
|
212
|
+
this.pendingAppends.clear();
|
|
213
|
+
this.pendingBytes = 0;
|
|
214
|
+
// Execute writes and appends sequentially per-key to avoid partial-failure
|
|
215
|
+
// ambiguity with Promise.all (where we can't tell which appends succeeded,
|
|
216
|
+
// risking duplicate data on retry since append is not idempotent).
|
|
217
|
+
const failedWrites = [];
|
|
218
|
+
const failedAppends = [];
|
|
219
|
+
let firstError = null;
|
|
220
|
+
for (const [key, write] of writes) {
|
|
221
|
+
try {
|
|
222
|
+
await this.backend.write(write.key, write.data);
|
|
223
|
+
}
|
|
224
|
+
catch (err) {
|
|
225
|
+
if (!firstError)
|
|
226
|
+
firstError = err;
|
|
227
|
+
failedWrites.push([key, write]);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
for (const [key, chunks] of appendEntries) {
|
|
231
|
+
// Concatenate all appends for this key
|
|
232
|
+
let totalSize = 0;
|
|
233
|
+
for (const chunk of chunks) {
|
|
234
|
+
totalSize += chunk.length;
|
|
235
|
+
}
|
|
236
|
+
const combined = new Uint8Array(totalSize);
|
|
237
|
+
let offset = 0;
|
|
238
|
+
for (const chunk of chunks) {
|
|
239
|
+
combined.set(chunk, offset);
|
|
240
|
+
offset += chunk.length;
|
|
241
|
+
}
|
|
242
|
+
try {
|
|
243
|
+
await this.backend.append(key, combined);
|
|
244
|
+
}
|
|
245
|
+
catch (err) {
|
|
246
|
+
if (!firstError)
|
|
247
|
+
firstError = err;
|
|
248
|
+
failedAppends.push([key, chunks]);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (firstError) {
|
|
252
|
+
// Only restore entries that actually failed — successful ones are persisted.
|
|
253
|
+
let restoredBytes = 0;
|
|
254
|
+
for (const [key, write] of failedWrites) {
|
|
255
|
+
if (!this.pendingWrites.has(key)) {
|
|
256
|
+
this.pendingWrites.set(key, write);
|
|
257
|
+
restoredBytes += write.data.length;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
for (const [key, chunks] of failedAppends) {
|
|
261
|
+
// If a newer pending write exists for this key, it supersedes old appends.
|
|
262
|
+
// Restoring those stale appends would violate write ordering and corrupt data.
|
|
263
|
+
if (this.pendingWrites.has(key)) {
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
const existing = this.pendingAppends.get(key);
|
|
267
|
+
if (existing) {
|
|
268
|
+
// Prepend old chunks before any new ones
|
|
269
|
+
this.pendingAppends.set(key, [...chunks, ...existing]);
|
|
270
|
+
}
|
|
271
|
+
else {
|
|
272
|
+
this.pendingAppends.set(key, chunks);
|
|
273
|
+
}
|
|
274
|
+
for (const chunk of chunks) {
|
|
275
|
+
restoredBytes += chunk.length;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
this.pendingBytes += restoredBytes;
|
|
279
|
+
throw firstError;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Get statistics about pending writes
|
|
284
|
+
*/
|
|
285
|
+
getStats() {
|
|
286
|
+
return {
|
|
287
|
+
pendingWrites: this.pendingWrites.size,
|
|
288
|
+
pendingAppends: this.pendingAppends.size,
|
|
289
|
+
pendingBytes: this.pendingBytes,
|
|
290
|
+
maxPendingWrites: this.maxPendingWrites,
|
|
291
|
+
maxPendingBytes: this.maxPendingBytes
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Check if there are pending writes
|
|
296
|
+
*/
|
|
297
|
+
hasPendingWrites() {
|
|
298
|
+
return this.pendingWrites.size > 0 || this.pendingAppends.size > 0;
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Start auto-flush timer
|
|
302
|
+
*/
|
|
303
|
+
startAutoFlush() {
|
|
304
|
+
if (this.flushTimer)
|
|
305
|
+
return;
|
|
306
|
+
this.flushTimer = setInterval(async () => {
|
|
307
|
+
if (this.hasPendingWrites()) {
|
|
308
|
+
try {
|
|
309
|
+
await this.flush();
|
|
310
|
+
}
|
|
311
|
+
catch {
|
|
312
|
+
// Preserve pending data and retry on next interval/explicit flush.
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
}, this.autoFlushInterval);
|
|
316
|
+
// Allow the process to exit even if the timer is still running.
|
|
317
|
+
// Without this, the timer keeps the event loop alive indefinitely.
|
|
318
|
+
if (typeof this.flushTimer === 'object' && 'unref' in this.flushTimer) {
|
|
319
|
+
this.flushTimer.unref();
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Stop auto-flush timer
|
|
324
|
+
*/
|
|
325
|
+
stopAutoFlush() {
|
|
326
|
+
if (this.flushTimer) {
|
|
327
|
+
clearInterval(this.flushTimer);
|
|
328
|
+
this.flushTimer = null;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Close the batch writer, flushing any pending writes
|
|
333
|
+
*/
|
|
334
|
+
async close() {
|
|
335
|
+
this.stopAutoFlush();
|
|
336
|
+
await this.flush();
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Get the underlying storage backend
|
|
340
|
+
*/
|
|
341
|
+
getBackend() {
|
|
342
|
+
return this.backend;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* Create a batch writer that wraps an existing storage backend
|
|
347
|
+
*/
|
|
348
|
+
export function createBatchWriter(backend, options) {
|
|
349
|
+
return new BatchWriter(backend, options);
|
|
350
|
+
}
|
|
351
|
+
//# sourceMappingURL=BatchWriter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BatchWriter.js","sourceRoot":"","sources":["../../src/storage/BatchWriter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAmBH;;GAEG;AACH,MAAM,OAAO,WAAW;IACd,OAAO,CAAiB;IACxB,aAAa,GAA8B,IAAI,GAAG,EAAE,CAAC;IACrD,cAAc,GAA8B,IAAI,GAAG,EAAE,CAAC;IACtD,YAAY,GAAW,CAAC,CAAC;IACzB,gBAAgB,CAAS;IACzB,eAAe,CAAS;IACxB,iBAAiB,CAAS;IAC1B,UAAU,GAAyC,IAAI,CAAC;IACxD,UAAU,GAAY,KAAK,CAAC;IAC5B,YAAY,GAAyB,IAAI,CAAC;IAElD,YAAY,OAAuB,EAAE,UAA8B,EAAE;QACnE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,GAAG,CAAC;QACxD,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;QACrE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,IAA8B;QACrD,MAAM,KAAK,GAAG,IAAI,YAAY,WAAW,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAErB,wDAAwD;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,YAAY,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,4DAA4D;QAC5D,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,eAAe,EAAE,CAAC;YACpB,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;gBACrC,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;YACrC,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE;YAC1B,GAAG;YACH,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC,MAAM,CAAC;QAEtC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,IAA8B;QACtD,MAAM,KAAK,GAAG,IAAI,YAAY,WAAW,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAErB,mDAAmD;QACnD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,aAAa,EAAE,CAAC;YAClB,kCAAkC;YAClC,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAC7E,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClD,IAAI,CAAC,YAAY,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC;YAC/C,IAAI,CAAC,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;YACpC,aAAa,CAAC,IAAI,GAAG,OAAO,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,IAAI,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC,MAAM,CAAC;QACxC,CAAC;QAED,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,YAAY,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC1C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,eAAe,EAAE,CAAC;YACpB,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;gBACrC,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;YACrC,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,+DAA+D;QAC/D,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI,CAAC,GAAW;QACpB,0BAA0B;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CACtC,OAAO,CAAC,IAAI,CAAC,UAAU,EACvB,OAAO,CAAC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAC9C,CAAC;YACF,OAAO,MAAqB,CAAC;QAC/B,CAAC;QAED,kEAAkE;QAClE,wEAAwE;QACxE,8CAA8C;QAC9C,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,qCAAqC;gBACrC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;YAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAE9C,uEAAuE;YACvE,wCAAwC;YACxC,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAC1C,WAAW,CAAC,IAAI,CAAC,UAAU,EAC3B,WAAW,CAAC,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CACtD,CAAC;gBACF,OAAO,MAAqB,CAAC;YAC/B,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,CAAC,aAAa,IAAI,aAAa,KAAK,OAAO,IAAI,aAAa,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACxF,sEAAsE;gBACtE,qDAAqD;gBACrD,SAAS;YACX,CAAC;YAED,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;YAE9E,uBAAuB;YACvB,IAAI,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,SAAS,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACvC,CAAC;YAED,UAAU;YACV,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YACzC,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAC1B,IAAI,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;YAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC3B,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;YAC1B,CAAC;YAED,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe;QAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;QAEvE,IAAI,WAAW,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACtF,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,6DAA6D;QAC7D,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,YAAY,CAAC;YACxB,8DAA8D;YAC9D,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACpE,OAAO;YACT,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACpE,OAAO;QACT,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,YAAY,CAAC;QAC1B,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACxD,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC;QAEhE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAEtB,2EAA2E;QAC3E,2EAA2E;QAC3E,mEAAmE;QACnE,MAAM,YAAY,GAAkC,EAAE,CAAC;QACvD,MAAM,aAAa,GAAkC,EAAE,CAAC;QACxD,IAAI,UAAU,GAAY,IAAI,CAAC;QAE/B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAClD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,UAAU;oBAAE,UAAU,GAAG,GAAG,CAAC;gBAClC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;YAC1C,uCAAuC;YACvC,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;YAC5B,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAC3C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,UAAU;oBAAE,UAAU,GAAG,GAAG,CAAC;gBAClC,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAED,IAAI,UAAU,EAAE,CAAC;YACf,6EAA6E;YAC7E,IAAI,aAAa,GAAG,CAAC,CAAC;YACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,YAAY,EAAE,CAAC;gBACxC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBACnC,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;gBACrC,CAAC;YACH,CAAC;YACD,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;gBAC1C,2EAA2E;gBAC3E,+EAA+E;gBAC/E,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChC,SAAS;gBACX,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9C,IAAI,QAAQ,EAAE,CAAC;oBACb,yCAAyC;oBACzC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC;gBACzD,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBACvC,CAAC;gBACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC;gBAChC,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,IAAI,aAAa,CAAC;YACnC,MAAM,UAAU,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QAON,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;YACtC,cAAc,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI;YACxC,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,eAAe,EAAE,IAAI,CAAC,eAAe;SACtC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC,CAAC;IACrE,CAAC;IAED;;OAEG;IACK,cAAc;QACpB,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAE5B,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;gBAAC,MAAM,CAAC;oBACP,mEAAmE;gBACrE,CAAC;YACH,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAE3B,gEAAgE;QAChE,mEAAmE;QACnE,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACrE,IAAI,CAAC,UAAkB,CAAC,KAAK,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAAuB,EACvB,OAA4B;IAE5B,OAAO,IAAI,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAC3C,CAAC"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Bun
|
|
2
|
+
* Bun/Node file-system storage backend
|
|
3
3
|
*
|
|
4
|
-
* High-performance file system storage using Bun
|
|
4
|
+
* High-performance file system storage using Bun native APIs when available,
|
|
5
|
+
* with a Node.js filesystem fallback for runtime compatibility.
|
|
5
6
|
* Features:
|
|
6
7
|
* - Auto-initializing (no manual init() required)
|
|
7
8
|
* - Automatic memory mapping for large files
|
|
@@ -12,12 +13,15 @@ import type { StorageBackend } from './StorageBackend';
|
|
|
12
13
|
export declare class BunStorageBackend implements StorageBackend {
|
|
13
14
|
readonly type = "bun";
|
|
14
15
|
private basePath;
|
|
16
|
+
private nodeRuntimePromise;
|
|
15
17
|
private dirCache;
|
|
16
18
|
/**
|
|
17
19
|
* Create a new Bun storage backend
|
|
18
20
|
* @param basePath Base directory for all storage operations
|
|
19
21
|
*/
|
|
20
22
|
constructor(basePath?: string);
|
|
23
|
+
private isBrowserRuntime;
|
|
24
|
+
private getNodeRuntime;
|
|
21
25
|
/**
|
|
22
26
|
* Ensure directory exists, using cache to avoid redundant mkdir calls
|
|
23
27
|
*/
|
|
@@ -52,7 +56,7 @@ export declare class BunStorageBackend implements StorageBackend {
|
|
|
52
56
|
* Note: Callers should check exists() before calling stream(),
|
|
53
57
|
* as Bun.file().stream() on a non-existent file will error at read time.
|
|
54
58
|
*/
|
|
55
|
-
stream(key: string): ReadableStream<Uint8Array> | null
|
|
59
|
+
stream(key: string): Promise<ReadableStream<Uint8Array> | null>;
|
|
56
60
|
/**
|
|
57
61
|
* Get the base path
|
|
58
62
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BunStorageBackend.d.ts","sourceRoot":"","sources":["../../src/storage/BunStorageBackend.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"BunStorageBackend.d.ts","sourceRoot":"","sources":["../../src/storage/BunStorageBackend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAGvD,qBAAa,iBAAkB,YAAW,cAAc;IACtD,QAAQ,CAAC,IAAI,SAAS;IACtB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,kBAAkB,CAA0D;IAGpF,OAAO,CAAC,QAAQ,CAA0B;IAE1C;;;OAGG;gBACS,QAAQ,GAAE,MAA0B;IAIhD,OAAO,CAAC,gBAAgB;YAIV,cAAc;IAU5B;;OAEG;YACW,SAAS;IAmBvB;;;OAGG;YACW,WAAW;IAKzB;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAKrB,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAe9C,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAYjE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAWlE,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWrC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAMxC,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAa3C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B;;OAEG;IACG,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWxC;;;;OAIG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IASrE;;OAEG;IACH,WAAW,IAAI,MAAM;CAGtB"}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bun/Node file-system storage backend
|
|
3
|
+
*
|
|
4
|
+
* High-performance file system storage using Bun native APIs when available,
|
|
5
|
+
* with a Node.js filesystem fallback for runtime compatibility.
|
|
6
|
+
* Features:
|
|
7
|
+
* - Auto-initializing (no manual init() required)
|
|
8
|
+
* - Automatic memory mapping for large files
|
|
9
|
+
* - Efficient async I/O with O(1) append
|
|
10
|
+
* - Native TypedArray support
|
|
11
|
+
*/
|
|
12
|
+
import { StorageError } from '../errors.js';
|
|
13
|
+
export class BunStorageBackend {
|
|
14
|
+
type = 'bun';
|
|
15
|
+
basePath;
|
|
16
|
+
nodeRuntimePromise = null;
|
|
17
|
+
// Cache of directories we've already ensured exist - avoids redundant mkdir calls
|
|
18
|
+
// Profiling showed 5-10% write overhead from mkdir on every write
|
|
19
|
+
dirCache = new Set();
|
|
20
|
+
/**
|
|
21
|
+
* Create a new Bun storage backend
|
|
22
|
+
* @param basePath Base directory for all storage operations
|
|
23
|
+
*/
|
|
24
|
+
constructor(basePath = './vectordb_data') {
|
|
25
|
+
this.basePath = basePath;
|
|
26
|
+
}
|
|
27
|
+
isBrowserRuntime() {
|
|
28
|
+
return typeof window !== 'undefined' && typeof navigator !== 'undefined';
|
|
29
|
+
}
|
|
30
|
+
async getNodeRuntime() {
|
|
31
|
+
if (this.isBrowserRuntime()) {
|
|
32
|
+
throw new StorageError('init', 'BunStorageBackend is only available in Bun or Node.js runtimes');
|
|
33
|
+
}
|
|
34
|
+
if (!this.nodeRuntimePromise) {
|
|
35
|
+
this.nodeRuntimePromise = import('./nodeFsRuntime.js');
|
|
36
|
+
}
|
|
37
|
+
return this.nodeRuntimePromise;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Ensure directory exists, using cache to avoid redundant mkdir calls
|
|
41
|
+
*/
|
|
42
|
+
async ensureDir(dir) {
|
|
43
|
+
if (this.dirCache.has(dir))
|
|
44
|
+
return;
|
|
45
|
+
try {
|
|
46
|
+
const runtime = await this.getNodeRuntime();
|
|
47
|
+
await runtime.ensureDir(dir);
|
|
48
|
+
this.dirCache.add(dir);
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
// EEXIST is fine - directory already exists
|
|
52
|
+
if (err instanceof Error && err.code === 'EEXIST') {
|
|
53
|
+
this.dirCache.add(dir);
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
// Re-throw real errors (permissions, disk full, etc.) so callers get
|
|
57
|
+
// a clear failure instead of a confusing "file not found" on the
|
|
58
|
+
// subsequent write attempt.
|
|
59
|
+
throw err;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Get the full path for a key.
|
|
64
|
+
* Validates that the resolved path stays within basePath to prevent path traversal.
|
|
65
|
+
*/
|
|
66
|
+
async getFullPath(key) {
|
|
67
|
+
const runtime = await this.getNodeRuntime();
|
|
68
|
+
return runtime.resolveStoragePath(this.basePath, key);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Ensure base directory exists (optional - operations auto-initialize)
|
|
72
|
+
* @deprecated No longer required - write/append create directories automatically
|
|
73
|
+
*/
|
|
74
|
+
async init() {
|
|
75
|
+
const runtime = await this.getNodeRuntime();
|
|
76
|
+
await runtime.ensureDir(runtime.resolveBasePath(this.basePath)).catch(() => { });
|
|
77
|
+
}
|
|
78
|
+
async read(key) {
|
|
79
|
+
const fullPath = await this.getFullPath(key);
|
|
80
|
+
if (typeof Bun !== 'undefined') {
|
|
81
|
+
const file = Bun.file(fullPath);
|
|
82
|
+
// Bun.file().size is 0 for non-existent files — avoids TOCTOU race with exists()
|
|
83
|
+
if (file.size === 0) {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
return file.arrayBuffer();
|
|
87
|
+
}
|
|
88
|
+
const runtime = await this.getNodeRuntime();
|
|
89
|
+
return runtime.read(fullPath);
|
|
90
|
+
}
|
|
91
|
+
async write(key, data) {
|
|
92
|
+
const runtime = await this.getNodeRuntime();
|
|
93
|
+
const fullPath = await this.getFullPath(key);
|
|
94
|
+
// Ensure parent directory exists (cached to avoid redundant syscalls)
|
|
95
|
+
await this.ensureDir(runtime.dirname(fullPath));
|
|
96
|
+
// Atomic write: write to temp file then rename.
|
|
97
|
+
// If the process crashes mid-write, the original file is untouched.
|
|
98
|
+
await runtime.writeAtomic(fullPath, data);
|
|
99
|
+
}
|
|
100
|
+
async append(key, data) {
|
|
101
|
+
const runtime = await this.getNodeRuntime();
|
|
102
|
+
const fullPath = await this.getFullPath(key);
|
|
103
|
+
// Ensure parent directory exists (cached to avoid redundant syscalls)
|
|
104
|
+
await this.ensureDir(runtime.dirname(fullPath));
|
|
105
|
+
// Use true O(1) append instead of O(n) read-modify-write
|
|
106
|
+
await runtime.append(fullPath, data);
|
|
107
|
+
}
|
|
108
|
+
async delete(key) {
|
|
109
|
+
const fullPath = await this.getFullPath(key);
|
|
110
|
+
const runtime = await this.getNodeRuntime();
|
|
111
|
+
await runtime.deleteFile(fullPath);
|
|
112
|
+
}
|
|
113
|
+
async exists(key) {
|
|
114
|
+
const fullPath = await this.getFullPath(key);
|
|
115
|
+
if (typeof Bun !== 'undefined') {
|
|
116
|
+
const file = Bun.file(fullPath);
|
|
117
|
+
return file.exists();
|
|
118
|
+
}
|
|
119
|
+
const runtime = await this.getNodeRuntime();
|
|
120
|
+
return runtime.exists(fullPath);
|
|
121
|
+
}
|
|
122
|
+
async list(prefix) {
|
|
123
|
+
const runtime = await this.getNodeRuntime();
|
|
124
|
+
const searchPath = prefix ? await this.getFullPath(prefix) : runtime.resolveBasePath(this.basePath);
|
|
125
|
+
return runtime.list(this.basePath, searchPath);
|
|
126
|
+
}
|
|
127
|
+
async mkdir(dirPath) {
|
|
128
|
+
const runtime = await this.getNodeRuntime();
|
|
129
|
+
const fullPath = await this.getFullPath(dirPath);
|
|
130
|
+
try {
|
|
131
|
+
await runtime.ensureDir(fullPath);
|
|
132
|
+
}
|
|
133
|
+
catch (err) {
|
|
134
|
+
if (err instanceof Error && err.code === 'EEXIST') {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
throw err;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Delete all data in the storage directory
|
|
142
|
+
*/
|
|
143
|
+
async clear() {
|
|
144
|
+
const runtime = await this.getNodeRuntime();
|
|
145
|
+
const resolvedBase = await runtime.clear(this.basePath);
|
|
146
|
+
// Clear directory cache since directories were deleted
|
|
147
|
+
this.dirCache.clear();
|
|
148
|
+
this.dirCache.add(resolvedBase);
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Get file size without reading the entire file
|
|
152
|
+
*/
|
|
153
|
+
async size(key) {
|
|
154
|
+
const fullPath = await this.getFullPath(key);
|
|
155
|
+
if (typeof Bun !== 'undefined') {
|
|
156
|
+
// Bun.file().size returns 0 for non-existent files — no exists() check needed
|
|
157
|
+
return Bun.file(fullPath).size;
|
|
158
|
+
}
|
|
159
|
+
const runtime = await this.getNodeRuntime();
|
|
160
|
+
return runtime.size(fullPath);
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Read a file as a stream (for large files).
|
|
164
|
+
* Note: Callers should check exists() before calling stream(),
|
|
165
|
+
* as Bun.file().stream() on a non-existent file will error at read time.
|
|
166
|
+
*/
|
|
167
|
+
async stream(key) {
|
|
168
|
+
const fullPath = await this.getFullPath(key);
|
|
169
|
+
if (typeof Bun !== 'undefined') {
|
|
170
|
+
return Bun.file(fullPath).stream();
|
|
171
|
+
}
|
|
172
|
+
const runtime = await this.getNodeRuntime();
|
|
173
|
+
return runtime.stream(fullPath);
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Get the base path
|
|
177
|
+
*/
|
|
178
|
+
getBasePath() {
|
|
179
|
+
return this.basePath;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
//# sourceMappingURL=BunStorageBackend.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BunStorageBackend.js","sourceRoot":"","sources":["../../src/storage/BunStorageBackend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,KAAK,CAAC;IACd,QAAQ,CAAS;IACjB,kBAAkB,GAAqD,IAAI,CAAC;IACpF,kFAAkF;IAClF,kEAAkE;IAC1D,QAAQ,GAAgB,IAAI,GAAG,EAAE,CAAC;IAE1C;;;OAGG;IACH,YAAY,WAAmB,iBAAiB;QAC9C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAEO,gBAAgB;QACtB,OAAO,OAAO,MAAM,KAAK,WAAW,IAAI,OAAO,SAAS,KAAK,WAAW,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,YAAY,CAAC,MAAM,EAAE,gEAAgE,CAAC,CAAC;QACnG,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC7B,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QACnC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5C,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,4CAA4C;YAC5C,IAAI,GAAG,YAAY,KAAK,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7E,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACvB,OAAO;YACT,CAAC;YACD,qEAAqE;YACrE,iEAAiE;YACjE,4BAA4B;YAC5B,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACxD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAClF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAW;QACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,OAAO,GAAG,KAAK,WAAW,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,iFAAiF;YACjF,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,IAA8B;QACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAE7C,sEAAsE;QACtE,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAEhD,gDAAgD;QAChD,oEAAoE;QACpE,MAAM,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,IAA8B;QACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAE7C,sEAAsE;QACtE,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAEhD,yDAAyD;QACzD,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,OAAO,GAAG,KAAK,WAAW,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAe;QACxB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpG,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACjD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,IAAI,GAAG,YAAY,KAAK,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7E,OAAO;YACT,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,uDAAuD;QACvD,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,GAAW;QACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,OAAO,GAAG,KAAK,WAAW,EAAE,CAAC;YAC/B,8EAA8E;YAC9E,OAAO,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;QACjC,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,OAAO,GAAG,KAAK,WAAW,EAAE,CAAC;YAC/B,OAAO,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;QACrC,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;CACF"}
|