reflectdb 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +486 -17
- package/dist/cjs/htmx/index.cjs +1720 -0
- package/dist/cjs/htmx/index.d.cts +760 -0
- package/dist/cjs/server/index.cjs +12 -0
- package/dist/cjs/server/index.d.cts +14 -0
- package/dist/cjs/server/storage/object/index.cjs +2284 -0
- package/dist/cjs/server/storage/object/index.d.cts +578 -0
- package/dist/cjs/transport/sse.cjs +52 -1
- package/dist/cjs/transport/sse.d.cts +34 -0
- package/dist/htmx/index.d.ts +760 -0
- package/dist/htmx/index.js +297 -0
- package/dist/server/drizzle.js +2 -1
- package/dist/server/index.d.ts +14 -0
- package/dist/server/index.js +16 -2
- package/dist/server/storage/object/index.d.ts +578 -0
- package/dist/server/storage/object/index.js +2232 -0
- package/dist/shared/{esm-77ndhmpv.js → esm-5ahpq25j.js} +4 -14
- package/dist/shared/esm-dcs8qa5n.js +263 -0
- package/dist/shared/esm-f11s9zpb.js +15 -0
- package/dist/transport/sse.d.ts +34 -0
- package/dist/transport/sse.js +52 -1
- package/dist/vanilla/index.js +4 -260
- package/package.json +23 -1
|
@@ -0,0 +1,2284 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
5
|
+
function __accessProp(key) {
|
|
6
|
+
return this[key];
|
|
7
|
+
}
|
|
8
|
+
var __toCommonJS = (from) => {
|
|
9
|
+
var entry = (__moduleCache ??= new WeakMap).get(from), desc;
|
|
10
|
+
if (entry)
|
|
11
|
+
return entry;
|
|
12
|
+
entry = __defProp({}, "__esModule", { value: true });
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (var key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(entry, key))
|
|
16
|
+
__defProp(entry, key, {
|
|
17
|
+
get: __accessProp.bind(from, key),
|
|
18
|
+
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
__moduleCache.set(from, entry);
|
|
22
|
+
return entry;
|
|
23
|
+
};
|
|
24
|
+
var __moduleCache;
|
|
25
|
+
var __returnValue = (v) => v;
|
|
26
|
+
function __exportSetter(name, newValue) {
|
|
27
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
28
|
+
}
|
|
29
|
+
var __export = (target, all) => {
|
|
30
|
+
for (var name in all)
|
|
31
|
+
__defProp(target, name, {
|
|
32
|
+
get: all[name],
|
|
33
|
+
enumerable: true,
|
|
34
|
+
configurable: true,
|
|
35
|
+
set: __exportSetter.bind(all, name)
|
|
36
|
+
});
|
|
37
|
+
};
|
|
38
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
39
|
+
|
|
40
|
+
// src/server/storage/object/index.ts
|
|
41
|
+
var exports_object = {};
|
|
42
|
+
__export(exports_object, {
|
|
43
|
+
BackpressureError: () => BackpressureError,
|
|
44
|
+
IncompleteStateError: () => IncompleteStateError,
|
|
45
|
+
MemoryLimitExceededError: () => MemoryLimitExceededError,
|
|
46
|
+
NotWriterError: () => NotWriterError,
|
|
47
|
+
OBJECT_STORAGE_DEFAULTS: () => OBJECT_STORAGE_DEFAULTS,
|
|
48
|
+
PreconditionFailedError: () => PreconditionFailedError,
|
|
49
|
+
createFilesystemDriver: () => createFilesystemDriver,
|
|
50
|
+
createMemoryDriver: () => createMemoryDriver,
|
|
51
|
+
createObjectStorage: () => createObjectStorage,
|
|
52
|
+
createS3Driver: () => createS3Driver,
|
|
53
|
+
resolveObjectStorageConfig: () => resolveObjectStorageConfig,
|
|
54
|
+
roomPrefix: () => roomPrefix
|
|
55
|
+
});
|
|
56
|
+
module.exports = __toCommonJS(exports_object);
|
|
57
|
+
|
|
58
|
+
// src/server/storage/object/types.ts
|
|
59
|
+
class PreconditionFailedError extends Error {
|
|
60
|
+
key;
|
|
61
|
+
constructor(key, message) {
|
|
62
|
+
super(message ?? `Conditional write failed for "${key}" (etag mismatch)`);
|
|
63
|
+
this.name = "PreconditionFailedError";
|
|
64
|
+
this.key = key;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
class BackpressureError extends Error {
|
|
69
|
+
bufferedBytes;
|
|
70
|
+
limitBytes;
|
|
71
|
+
constructor(bufferedBytes, limitBytes) {
|
|
72
|
+
super(`Object storage write buffer is full (${bufferedBytes} > ${limitBytes} bytes). ` + `The object store is not keeping up; the write was rejected so backpressure ` + `reaches the client. Set batch.onBackpressure to "degrade" to accept writes ` + `without a durability guarantee instead.`);
|
|
73
|
+
this.name = "BackpressureError";
|
|
74
|
+
this.bufferedBytes = bufferedBytes;
|
|
75
|
+
this.limitBytes = limitBytes;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
class MemoryLimitExceededError extends Error {
|
|
80
|
+
usedBytes;
|
|
81
|
+
limitBytes;
|
|
82
|
+
constructor(usedBytes, limitBytes) {
|
|
83
|
+
super(`Object storage room state exceeded its memory budget ` + `(${usedBytes} > ${limitBytes} bytes). State is authoritative in memory, so ` + `this is a hard ceiling rather than a slowdown. Raise memory.maxRoomBytes, ` + `shard the room, or wait for memory.onExceeded: "spill".`);
|
|
84
|
+
this.name = "MemoryLimitExceededError";
|
|
85
|
+
this.usedBytes = usedBytes;
|
|
86
|
+
this.limitBytes = limitBytes;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
class IncompleteStateError extends Error {
|
|
91
|
+
roomId;
|
|
92
|
+
key;
|
|
93
|
+
constructor(roomId, key, detail) {
|
|
94
|
+
super(`Object storage: object "${key}" named by the manifest for room "${roomId}" is ` + `missing. ${detail}`);
|
|
95
|
+
this.name = "IncompleteStateError";
|
|
96
|
+
this.roomId = roomId;
|
|
97
|
+
this.key = key;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
class NotWriterError extends Error {
|
|
102
|
+
roomId;
|
|
103
|
+
constructor(roomId, detail) {
|
|
104
|
+
super(`Not the writer for room "${roomId}"${detail ? `: ${detail}` : ""}`);
|
|
105
|
+
this.name = "NotWriterError";
|
|
106
|
+
this.roomId = roomId;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
var OBJECT_STORAGE_DEFAULTS = {
|
|
110
|
+
durability: "durable",
|
|
111
|
+
retentionMs: Number.POSITIVE_INFINITY,
|
|
112
|
+
concurrency: "single-writer",
|
|
113
|
+
batch: {
|
|
114
|
+
maxBytes: 4 * 1024 * 1024,
|
|
115
|
+
minLingerMs: 5,
|
|
116
|
+
maxBufferBytes: 64 * 1024 * 1024,
|
|
117
|
+
onBackpressure: "reject"
|
|
118
|
+
},
|
|
119
|
+
compaction: {
|
|
120
|
+
afterSegments: 200,
|
|
121
|
+
afterBytes: 64 * 1024 * 1024,
|
|
122
|
+
gcGraceMs: 60 * 60 * 1000
|
|
123
|
+
},
|
|
124
|
+
lease: {
|
|
125
|
+
ttlMs: 5 * 60 * 1000,
|
|
126
|
+
renewMs: 2 * 60 * 1000,
|
|
127
|
+
mode: "on-write"
|
|
128
|
+
},
|
|
129
|
+
memory: {
|
|
130
|
+
maxTotalBytes: Number.POSITIVE_INFINITY,
|
|
131
|
+
maxRoomBytes: Number.POSITIVE_INFINITY,
|
|
132
|
+
onExceeded: "reject",
|
|
133
|
+
idleEvictMs: 5 * 60 * 1000
|
|
134
|
+
},
|
|
135
|
+
shutdownFlushMs: 5000
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
// src/server/storage/object/state.ts
|
|
139
|
+
var systemClock = {
|
|
140
|
+
now: () => Date.now(),
|
|
141
|
+
delay: (ms) => new Promise((resolve) => {
|
|
142
|
+
unref(setTimeout(resolve, ms));
|
|
143
|
+
}),
|
|
144
|
+
setTimer: (fn, ms) => {
|
|
145
|
+
const id = setTimeout(fn, ms);
|
|
146
|
+
unref(id);
|
|
147
|
+
return () => {
|
|
148
|
+
clearTimeout(id);
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
};
|
|
152
|
+
function unref(id) {
|
|
153
|
+
id.unref?.();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
class ProcessMemoryBudget {
|
|
157
|
+
limitBytes = Number.POSITIVE_INFINITY;
|
|
158
|
+
usedBytes = 0;
|
|
159
|
+
get used() {
|
|
160
|
+
return this.usedBytes;
|
|
161
|
+
}
|
|
162
|
+
get limit() {
|
|
163
|
+
return this.limitBytes;
|
|
164
|
+
}
|
|
165
|
+
constrain(limit) {
|
|
166
|
+
if (limit < this.limitBytes)
|
|
167
|
+
this.limitBytes = limit;
|
|
168
|
+
}
|
|
169
|
+
add(delta) {
|
|
170
|
+
this.usedBytes += delta;
|
|
171
|
+
if (this.usedBytes < 0)
|
|
172
|
+
this.usedBytes = 0;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
var processMemoryBudget = new ProcessMemoryBudget;
|
|
176
|
+
function approxBytes(value) {
|
|
177
|
+
if (value === undefined)
|
|
178
|
+
return 0;
|
|
179
|
+
return JSON.stringify(value)?.length ?? 0;
|
|
180
|
+
}
|
|
181
|
+
function reservationBytes(opId) {
|
|
182
|
+
return opId.length + 24;
|
|
183
|
+
}
|
|
184
|
+
function recordHlc(record) {
|
|
185
|
+
if (record.k === "reserve")
|
|
186
|
+
return "";
|
|
187
|
+
return record.k === "op" ? record.op.hlc : record.row.hlc;
|
|
188
|
+
}
|
|
189
|
+
var OP_ID_RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
190
|
+
var OP_ID_SWEEP_INTERVAL_MS = 60000;
|
|
191
|
+
|
|
192
|
+
class RoomState {
|
|
193
|
+
memory;
|
|
194
|
+
clock;
|
|
195
|
+
budget;
|
|
196
|
+
rows = new Map;
|
|
197
|
+
ops = [];
|
|
198
|
+
reserved = new Map;
|
|
199
|
+
meta = new Map;
|
|
200
|
+
roomBytes = 0;
|
|
201
|
+
highestHlc = null;
|
|
202
|
+
lastSweepAt = 0;
|
|
203
|
+
constructor(options) {
|
|
204
|
+
this.memory = options.memory;
|
|
205
|
+
this.clock = options.clock ?? systemClock;
|
|
206
|
+
this.budget = options.budget ?? processMemoryBudget;
|
|
207
|
+
this.budget.constrain(this.memory.maxTotalBytes);
|
|
208
|
+
}
|
|
209
|
+
get bytes() {
|
|
210
|
+
return this.roomBytes;
|
|
211
|
+
}
|
|
212
|
+
get maxHlc() {
|
|
213
|
+
return this.highestHlc;
|
|
214
|
+
}
|
|
215
|
+
getRow(table, rowId) {
|
|
216
|
+
const entry = this.rows.get(table)?.get(rowId);
|
|
217
|
+
if (!entry)
|
|
218
|
+
return { row: null, rowHlc: null, colClocks: {} };
|
|
219
|
+
return { row: { ...entry.row }, rowHlc: entry.hlc, colClocks: { ...entry.colClocks } };
|
|
220
|
+
}
|
|
221
|
+
getRowsByIds(table, rowIds) {
|
|
222
|
+
const out = {};
|
|
223
|
+
const tableRows = this.rows.get(table);
|
|
224
|
+
if (!tableRows)
|
|
225
|
+
return out;
|
|
226
|
+
for (const rowId of rowIds) {
|
|
227
|
+
const entry = tableRows.get(rowId);
|
|
228
|
+
if (entry) {
|
|
229
|
+
out[rowId] = {
|
|
230
|
+
row: { ...entry.row },
|
|
231
|
+
rowHlc: entry.hlc,
|
|
232
|
+
colClocks: { ...entry.colClocks }
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return out;
|
|
237
|
+
}
|
|
238
|
+
getRows(table, filter) {
|
|
239
|
+
const rows = [];
|
|
240
|
+
const colClocks = {};
|
|
241
|
+
const tableRows = this.rows.get(table);
|
|
242
|
+
if (!tableRows)
|
|
243
|
+
return { rows, colClocks };
|
|
244
|
+
const filterKeys = filter ? Object.keys(filter) : [];
|
|
245
|
+
for (const [rowId, entry] of tableRows) {
|
|
246
|
+
if (filterKeys.length > 0) {
|
|
247
|
+
let matches = true;
|
|
248
|
+
for (const key of filterKeys) {
|
|
249
|
+
if (entry.row[key] !== filter[key]) {
|
|
250
|
+
matches = false;
|
|
251
|
+
break;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (!matches)
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
rows.push({ ...entry.row });
|
|
258
|
+
colClocks[rowId] = { ...entry.colClocks };
|
|
259
|
+
}
|
|
260
|
+
return { rows, colClocks };
|
|
261
|
+
}
|
|
262
|
+
getOpsSince(since, tables) {
|
|
263
|
+
if (tables.length === 0)
|
|
264
|
+
return [];
|
|
265
|
+
const wanted = new Set(tables);
|
|
266
|
+
const out = [];
|
|
267
|
+
for (let i = this.indexAfter(since);i < this.ops.length; i++) {
|
|
268
|
+
const op = this.ops[i];
|
|
269
|
+
if (wanted.has(op.entry.table))
|
|
270
|
+
out.push({ ...op.entry });
|
|
271
|
+
}
|
|
272
|
+
return out;
|
|
273
|
+
}
|
|
274
|
+
getChangedTablesSince(since, tables) {
|
|
275
|
+
if (tables.length === 0)
|
|
276
|
+
return [];
|
|
277
|
+
const wanted = new Set(tables);
|
|
278
|
+
const changed = new Set;
|
|
279
|
+
for (let i = this.indexAfter(since);i < this.ops.length; i++) {
|
|
280
|
+
const table = this.ops[i].entry.table;
|
|
281
|
+
if (wanted.has(table))
|
|
282
|
+
changed.add(table);
|
|
283
|
+
if (changed.size === wanted.size)
|
|
284
|
+
break;
|
|
285
|
+
}
|
|
286
|
+
return [...changed];
|
|
287
|
+
}
|
|
288
|
+
getOplogHead(tables) {
|
|
289
|
+
if (tables.length === 0)
|
|
290
|
+
return null;
|
|
291
|
+
const wanted = new Set(tables);
|
|
292
|
+
for (let i = this.ops.length - 1;i >= 0; i--) {
|
|
293
|
+
const op = this.ops[i];
|
|
294
|
+
if (wanted.has(op.entry.table))
|
|
295
|
+
return op.entry.hlc;
|
|
296
|
+
}
|
|
297
|
+
return null;
|
|
298
|
+
}
|
|
299
|
+
getMeta(key) {
|
|
300
|
+
return this.meta.get(key) ?? null;
|
|
301
|
+
}
|
|
302
|
+
metaSnapshot() {
|
|
303
|
+
return Object.fromEntries(this.meta);
|
|
304
|
+
}
|
|
305
|
+
putRow(table, rowId, row, colClocks, hlc) {
|
|
306
|
+
let tableRows = this.rows.get(table);
|
|
307
|
+
const previous = tableRows?.get(rowId);
|
|
308
|
+
if (row === null) {
|
|
309
|
+
if (previous && tableRows) {
|
|
310
|
+
this.addBytes(-previous.bytes);
|
|
311
|
+
tableRows.delete(rowId);
|
|
312
|
+
if (tableRows.size === 0)
|
|
313
|
+
this.rows.delete(table);
|
|
314
|
+
}
|
|
315
|
+
this.noteHlc(hlc);
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
const bytes = table.length + rowId.length + hlc.length + approxBytes(row) + approxBytes(colClocks);
|
|
319
|
+
this.addBytes(bytes - (previous?.bytes ?? 0));
|
|
320
|
+
if (!tableRows) {
|
|
321
|
+
tableRows = new Map;
|
|
322
|
+
this.rows.set(table, tableRows);
|
|
323
|
+
}
|
|
324
|
+
tableRows.set(rowId, { row: { ...row }, colClocks: { ...colClocks }, hlc, bytes });
|
|
325
|
+
this.noteHlc(hlc);
|
|
326
|
+
}
|
|
327
|
+
assertCanAdmitApply(mutation, op) {
|
|
328
|
+
const previous = this.rows.get(mutation.table)?.get(mutation.rowId);
|
|
329
|
+
const rowBytes = mutation.row === null ? 0 : mutation.table.length + mutation.rowId.length + mutation.hlc.length + approxBytes(mutation.row) + approxBytes(mutation.colClocks);
|
|
330
|
+
const delta = rowBytes - (previous?.bytes ?? 0) + (op ? approxBytes(op) : 0);
|
|
331
|
+
if (delta > 0)
|
|
332
|
+
this.checkBytes(delta);
|
|
333
|
+
}
|
|
334
|
+
appendOp(entry) {
|
|
335
|
+
const bytes = approxBytes(entry);
|
|
336
|
+
this.addBytes(bytes);
|
|
337
|
+
const item = { entry, bytes };
|
|
338
|
+
const last = this.ops[this.ops.length - 1];
|
|
339
|
+
if (!last || last.entry.hlc <= entry.hlc) {
|
|
340
|
+
this.ops.push(item);
|
|
341
|
+
} else {
|
|
342
|
+
this.ops.splice(this.indexAfter(entry.hlc), 0, item);
|
|
343
|
+
}
|
|
344
|
+
this.noteHlc(entry.hlc);
|
|
345
|
+
}
|
|
346
|
+
deleteOpsBefore(hlc) {
|
|
347
|
+
const cut = this.indexBefore(hlc);
|
|
348
|
+
if (cut === 0)
|
|
349
|
+
return 0;
|
|
350
|
+
const removed = this.ops.slice(0, cut);
|
|
351
|
+
let freed = 0;
|
|
352
|
+
for (const op of removed)
|
|
353
|
+
freed += op.bytes;
|
|
354
|
+
this.ops = this.ops.slice(cut);
|
|
355
|
+
this.addBytes(-freed);
|
|
356
|
+
return removed.length;
|
|
357
|
+
}
|
|
358
|
+
reserveOp(opId) {
|
|
359
|
+
this.sweepReservedOps();
|
|
360
|
+
if (this.reserved.has(opId))
|
|
361
|
+
return false;
|
|
362
|
+
this.reserved.set(opId, this.clock.now());
|
|
363
|
+
this.addBytes(reservationBytes(opId));
|
|
364
|
+
return true;
|
|
365
|
+
}
|
|
366
|
+
reserveOps(opIds) {
|
|
367
|
+
if (opIds.length === 0)
|
|
368
|
+
return [];
|
|
369
|
+
this.sweepReservedOps();
|
|
370
|
+
const now = this.clock.now();
|
|
371
|
+
const fresh = [];
|
|
372
|
+
for (const opId of opIds) {
|
|
373
|
+
if (this.reserved.has(opId))
|
|
374
|
+
continue;
|
|
375
|
+
this.reserved.set(opId, now);
|
|
376
|
+
this.addBytes(reservationBytes(opId));
|
|
377
|
+
fresh.push(opId);
|
|
378
|
+
}
|
|
379
|
+
return fresh;
|
|
380
|
+
}
|
|
381
|
+
setMeta(key, value) {
|
|
382
|
+
this.meta.set(key, value);
|
|
383
|
+
}
|
|
384
|
+
loadMeta(meta) {
|
|
385
|
+
this.meta.clear();
|
|
386
|
+
for (const [key, value] of Object.entries(meta))
|
|
387
|
+
this.meta.set(key, value);
|
|
388
|
+
}
|
|
389
|
+
applyRecord(record) {
|
|
390
|
+
if (record.k === "reserve") {
|
|
391
|
+
this.restoreReservations(record.opIds, record.at);
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
if (record.k === "row" || record.k === "apply") {
|
|
395
|
+
const { table, rowId, row, colClocks, hlc } = record.row;
|
|
396
|
+
this.putRow(table, rowId, row, colClocks, hlc);
|
|
397
|
+
}
|
|
398
|
+
if (record.k === "op" || record.k === "apply") {
|
|
399
|
+
this.appendOp(record.op);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
restoreReservations(opIds, at) {
|
|
403
|
+
const cutoff = this.clock.now() - OP_ID_RETENTION_MS;
|
|
404
|
+
if (at < cutoff)
|
|
405
|
+
return;
|
|
406
|
+
for (const opId of opIds) {
|
|
407
|
+
const existing = this.reserved.get(opId);
|
|
408
|
+
if (existing === undefined) {
|
|
409
|
+
this.reserved.set(opId, at);
|
|
410
|
+
this.addBytes(reservationBytes(opId));
|
|
411
|
+
} else if (at < existing) {
|
|
412
|
+
this.reserved.set(opId, at);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
applyRecords(records) {
|
|
417
|
+
for (const record of records)
|
|
418
|
+
this.applyRecord(record);
|
|
419
|
+
}
|
|
420
|
+
toSnapshot() {
|
|
421
|
+
const rows = [];
|
|
422
|
+
for (const [table, tableRows] of this.rows) {
|
|
423
|
+
for (const [rowId, entry] of tableRows) {
|
|
424
|
+
rows.push({ table, rowId, row: entry.row, colClocks: entry.colClocks, hlc: entry.hlc });
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
const cutoff = this.clock.now() - OP_ID_RETENTION_MS;
|
|
428
|
+
const reservedOps = [];
|
|
429
|
+
for (const [opId, at] of this.reserved) {
|
|
430
|
+
if (at >= cutoff)
|
|
431
|
+
reservedOps.push([opId, at]);
|
|
432
|
+
}
|
|
433
|
+
return { version: 1, hlc: this.highestHlc, rows, reservedOps };
|
|
434
|
+
}
|
|
435
|
+
loadSnapshot(record) {
|
|
436
|
+
this.reset();
|
|
437
|
+
for (const row of record.rows) {
|
|
438
|
+
this.putRow(row.table, row.rowId, row.row, row.colClocks, row.hlc);
|
|
439
|
+
}
|
|
440
|
+
const cutoff = this.clock.now() - OP_ID_RETENTION_MS;
|
|
441
|
+
for (const [opId, at] of record.reservedOps) {
|
|
442
|
+
if (at < cutoff)
|
|
443
|
+
continue;
|
|
444
|
+
this.reserved.set(opId, at);
|
|
445
|
+
this.addBytes(reservationBytes(opId));
|
|
446
|
+
}
|
|
447
|
+
if (record.hlc && (!this.highestHlc || record.hlc > this.highestHlc)) {
|
|
448
|
+
this.highestHlc = record.hlc;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
dispose() {
|
|
452
|
+
this.reset();
|
|
453
|
+
}
|
|
454
|
+
reset() {
|
|
455
|
+
this.addBytes(-this.roomBytes);
|
|
456
|
+
this.rows.clear();
|
|
457
|
+
this.ops = [];
|
|
458
|
+
this.reserved.clear();
|
|
459
|
+
this.highestHlc = null;
|
|
460
|
+
}
|
|
461
|
+
noteHlc(hlc) {
|
|
462
|
+
if (!this.highestHlc || hlc > this.highestHlc)
|
|
463
|
+
this.highestHlc = hlc;
|
|
464
|
+
}
|
|
465
|
+
checkBytes(delta) {
|
|
466
|
+
const roomTotal = this.roomBytes + delta;
|
|
467
|
+
if (roomTotal > this.memory.maxRoomBytes) {
|
|
468
|
+
this.refuse(roomTotal, this.memory.maxRoomBytes);
|
|
469
|
+
}
|
|
470
|
+
const processTotal = this.budget.used + delta;
|
|
471
|
+
if (processTotal > this.budget.limit) {
|
|
472
|
+
this.refuse(processTotal, this.budget.limit);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
addBytes(delta) {
|
|
476
|
+
if (delta > 0)
|
|
477
|
+
this.checkBytes(delta);
|
|
478
|
+
this.roomBytes += delta;
|
|
479
|
+
if (this.roomBytes < 0)
|
|
480
|
+
this.roomBytes = 0;
|
|
481
|
+
this.budget.add(delta);
|
|
482
|
+
}
|
|
483
|
+
refuse(used, limit) {
|
|
484
|
+
const policy = this.memory.onExceeded;
|
|
485
|
+
if (policy !== "reject") {
|
|
486
|
+
throw new Error(`memory.onExceeded: "${policy}" is not implemented in phase 1 ` + `(state exceeded ${limit} bytes). "evict" lands in phase 2 and "spill" ` + `— DuckDB-over-Parquet reads — in phase 3, because it needs the Parquet ` + `layer. Use "reject" and raise memory.maxRoomBytes / maxTotalBytes, or ` + `shard the room. See docs/object-storage.md.`);
|
|
487
|
+
}
|
|
488
|
+
throw new MemoryLimitExceededError(used, limit);
|
|
489
|
+
}
|
|
490
|
+
indexAfter(hlc) {
|
|
491
|
+
let lo = 0;
|
|
492
|
+
let hi = this.ops.length;
|
|
493
|
+
while (lo < hi) {
|
|
494
|
+
const mid = lo + hi >>> 1;
|
|
495
|
+
if (this.ops[mid].entry.hlc > hlc)
|
|
496
|
+
hi = mid;
|
|
497
|
+
else
|
|
498
|
+
lo = mid + 1;
|
|
499
|
+
}
|
|
500
|
+
return lo;
|
|
501
|
+
}
|
|
502
|
+
indexBefore(hlc) {
|
|
503
|
+
let lo = 0;
|
|
504
|
+
let hi = this.ops.length;
|
|
505
|
+
while (lo < hi) {
|
|
506
|
+
const mid = lo + hi >>> 1;
|
|
507
|
+
if (this.ops[mid].entry.hlc >= hlc)
|
|
508
|
+
hi = mid;
|
|
509
|
+
else
|
|
510
|
+
lo = mid + 1;
|
|
511
|
+
}
|
|
512
|
+
return lo;
|
|
513
|
+
}
|
|
514
|
+
sweepReservedOps() {
|
|
515
|
+
const now = this.clock.now();
|
|
516
|
+
if (now - this.lastSweepAt < OP_ID_SWEEP_INTERVAL_MS)
|
|
517
|
+
return;
|
|
518
|
+
this.lastSweepAt = now;
|
|
519
|
+
const cutoff = now - OP_ID_RETENTION_MS;
|
|
520
|
+
let freed = 0;
|
|
521
|
+
for (const [opId, at] of this.reserved) {
|
|
522
|
+
if (at < cutoff) {
|
|
523
|
+
this.reserved.delete(opId);
|
|
524
|
+
freed += reservationBytes(opId);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
if (freed > 0)
|
|
528
|
+
this.addBytes(-freed);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// src/server/storage/object/manifest.ts
|
|
533
|
+
var MANIFEST_KEY = "_manifest";
|
|
534
|
+
var LEASE_KEY = "_lease";
|
|
535
|
+
function roomPrefix(roomId) {
|
|
536
|
+
return `rooms/${encodeURIComponent(roomId)}/`;
|
|
537
|
+
}
|
|
538
|
+
function emptyManifest() {
|
|
539
|
+
return {
|
|
540
|
+
version: 1,
|
|
541
|
+
epoch: 0,
|
|
542
|
+
commitSeq: 0,
|
|
543
|
+
lastWriter: "",
|
|
544
|
+
snapshotKey: null,
|
|
545
|
+
snapshotHlc: null,
|
|
546
|
+
walSegs: [],
|
|
547
|
+
oplogHead: null,
|
|
548
|
+
meta: {},
|
|
549
|
+
pendingGc: []
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
var encoder = new TextEncoder;
|
|
553
|
+
var decoder = new TextDecoder;
|
|
554
|
+
function encodeJson(value) {
|
|
555
|
+
return encoder.encode(JSON.stringify(value));
|
|
556
|
+
}
|
|
557
|
+
function decodeJson(body, key) {
|
|
558
|
+
try {
|
|
559
|
+
return JSON.parse(decoder.decode(body));
|
|
560
|
+
} catch (error) {
|
|
561
|
+
throw new Error(`Object storage: "${key}" is not valid JSON and cannot be recovered automatically. ` + `Refusing to continue rather than overwrite live room state. (${String(error)})`);
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
var OPTIMISTIC_MAX_ATTEMPTS = 6;
|
|
565
|
+
|
|
566
|
+
class ManifestStore {
|
|
567
|
+
driver;
|
|
568
|
+
roomId;
|
|
569
|
+
writerId;
|
|
570
|
+
leaseConfig;
|
|
571
|
+
concurrency;
|
|
572
|
+
clock;
|
|
573
|
+
manifestKey;
|
|
574
|
+
leaseKey;
|
|
575
|
+
cached = emptyManifest();
|
|
576
|
+
cachedEtag = null;
|
|
577
|
+
loaded = false;
|
|
578
|
+
leaseEtag = null;
|
|
579
|
+
leaseEpoch = 0;
|
|
580
|
+
leaseExpiresAt = 0;
|
|
581
|
+
held = false;
|
|
582
|
+
cancelRenew = null;
|
|
583
|
+
wroteSinceRenew = false;
|
|
584
|
+
fenceReason = null;
|
|
585
|
+
stopped = false;
|
|
586
|
+
chain = Promise.resolve();
|
|
587
|
+
constructor(options) {
|
|
588
|
+
this.driver = options.driver;
|
|
589
|
+
this.roomId = options.roomId;
|
|
590
|
+
this.writerId = options.writerId;
|
|
591
|
+
this.leaseConfig = options.lease;
|
|
592
|
+
this.concurrency = options.concurrency ?? "single-writer";
|
|
593
|
+
this.clock = options.clock ?? systemClock;
|
|
594
|
+
const prefix = roomPrefix(options.roomId);
|
|
595
|
+
this.manifestKey = `${prefix}${MANIFEST_KEY}`;
|
|
596
|
+
this.leaseKey = `${prefix}${LEASE_KEY}`;
|
|
597
|
+
}
|
|
598
|
+
get manifest() {
|
|
599
|
+
return this.cached;
|
|
600
|
+
}
|
|
601
|
+
get epoch() {
|
|
602
|
+
return this.leaseEpoch;
|
|
603
|
+
}
|
|
604
|
+
get isWriter() {
|
|
605
|
+
return this.held && !this.fenceReason;
|
|
606
|
+
}
|
|
607
|
+
get fenced() {
|
|
608
|
+
return this.fenceReason !== null;
|
|
609
|
+
}
|
|
610
|
+
async init() {
|
|
611
|
+
if (this.driver.caps.casWildcard)
|
|
612
|
+
return;
|
|
613
|
+
if (!await this.driver.get(this.manifestKey)) {
|
|
614
|
+
await this.driver.put(this.manifestKey, encodeJson(emptyManifest()));
|
|
615
|
+
}
|
|
616
|
+
if (!await this.driver.get(this.leaseKey)) {
|
|
617
|
+
const unowned = { owner: "", epoch: 0, expiresAt: 0 };
|
|
618
|
+
await this.driver.put(this.leaseKey, encodeJson(unowned));
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
async load() {
|
|
622
|
+
const record = await this.driver.get(this.manifestKey);
|
|
623
|
+
if (!record) {
|
|
624
|
+
this.cached = emptyManifest();
|
|
625
|
+
this.cachedEtag = null;
|
|
626
|
+
} else {
|
|
627
|
+
this.cached = decodeJson(record.body, this.manifestKey);
|
|
628
|
+
this.cachedEtag = record.etag;
|
|
629
|
+
if (typeof this.cached.commitSeq !== "number")
|
|
630
|
+
this.cached.commitSeq = 0;
|
|
631
|
+
if (typeof this.cached.lastWriter !== "string")
|
|
632
|
+
this.cached.lastWriter = "";
|
|
633
|
+
if (this.cached.epoch > this.leaseEpoch)
|
|
634
|
+
this.leaseEpoch = this.cached.epoch;
|
|
635
|
+
}
|
|
636
|
+
this.loaded = true;
|
|
637
|
+
return this.cached;
|
|
638
|
+
}
|
|
639
|
+
async start() {
|
|
640
|
+
if (this.concurrency === "optimistic")
|
|
641
|
+
return;
|
|
642
|
+
if (this.leaseConfig.mode === "always")
|
|
643
|
+
await this.acquire();
|
|
644
|
+
}
|
|
645
|
+
async acquire() {
|
|
646
|
+
this.assertNotFenced();
|
|
647
|
+
const current = await this.driver.get(this.leaseKey);
|
|
648
|
+
const record = current ? decodeJson(current.body, this.leaseKey) : null;
|
|
649
|
+
const now = this.clock.now();
|
|
650
|
+
if (record && record.owner !== this.writerId && record.expiresAt > now) {
|
|
651
|
+
throw new NotWriterError(this.roomId, `lease held by "${record.owner}" for another ${record.expiresAt - now}ms`);
|
|
652
|
+
}
|
|
653
|
+
const ours = record?.owner === this.writerId && record.expiresAt > now && this.leaseEtag !== null;
|
|
654
|
+
const epoch = ours ? record.epoch : Math.max(record?.epoch ?? 0, this.leaseEpoch) + 1;
|
|
655
|
+
const next = { owner: this.writerId, epoch, expiresAt: now + this.leaseConfig.ttlMs };
|
|
656
|
+
try {
|
|
657
|
+
this.leaseEtag = await this.driver.put(this.leaseKey, encodeJson(next), current ? { ifMatch: current.etag } : this.createOptions());
|
|
658
|
+
} catch (error) {
|
|
659
|
+
if (error instanceof PreconditionFailedError) {
|
|
660
|
+
throw new NotWriterError(this.roomId, "lost the lease CAS to a concurrent writer");
|
|
661
|
+
}
|
|
662
|
+
throw error;
|
|
663
|
+
}
|
|
664
|
+
this.leaseEpoch = epoch;
|
|
665
|
+
this.leaseExpiresAt = next.expiresAt;
|
|
666
|
+
this.held = true;
|
|
667
|
+
this.armRenew();
|
|
668
|
+
}
|
|
669
|
+
async ensureWriter() {
|
|
670
|
+
if (this.concurrency === "optimistic")
|
|
671
|
+
return;
|
|
672
|
+
this.assertNotFenced();
|
|
673
|
+
if (this.held)
|
|
674
|
+
return;
|
|
675
|
+
await this.acquire();
|
|
676
|
+
}
|
|
677
|
+
async release() {
|
|
678
|
+
this.disarmRenew();
|
|
679
|
+
if (!this.held || !this.leaseEtag)
|
|
680
|
+
return;
|
|
681
|
+
this.held = false;
|
|
682
|
+
const unowned = { owner: "", epoch: this.leaseEpoch, expiresAt: 0 };
|
|
683
|
+
try {
|
|
684
|
+
await this.driver.put(this.leaseKey, encodeJson(unowned), { ifMatch: this.leaseEtag });
|
|
685
|
+
} catch (error) {
|
|
686
|
+
if (!(error instanceof PreconditionFailedError))
|
|
687
|
+
throw error;
|
|
688
|
+
}
|
|
689
|
+
this.leaseEtag = null;
|
|
690
|
+
}
|
|
691
|
+
stop() {
|
|
692
|
+
this.stopped = true;
|
|
693
|
+
this.disarmRenew();
|
|
694
|
+
}
|
|
695
|
+
commit(mutate) {
|
|
696
|
+
const apply = this.concurrency === "optimistic" ? () => this.doCommitOptimistic(mutate) : () => this.doCommit(mutate);
|
|
697
|
+
const run = this.chain.then(apply, apply);
|
|
698
|
+
this.chain = run.catch(() => {
|
|
699
|
+
return;
|
|
700
|
+
});
|
|
701
|
+
return run;
|
|
702
|
+
}
|
|
703
|
+
async doCommitOptimistic(mutate) {
|
|
704
|
+
for (let attempt = 1;; attempt++) {
|
|
705
|
+
if (!this.loaded)
|
|
706
|
+
await this.load();
|
|
707
|
+
const next = this.buildNext();
|
|
708
|
+
mutate(next);
|
|
709
|
+
try {
|
|
710
|
+
const etag = await this.driver.put(this.manifestKey, encodeJson(next), this.cachedEtag ? { ifMatch: this.cachedEtag } : this.createOptions());
|
|
711
|
+
this.cached = next;
|
|
712
|
+
this.cachedEtag = etag;
|
|
713
|
+
return next;
|
|
714
|
+
} catch (error) {
|
|
715
|
+
if (!(error instanceof PreconditionFailedError))
|
|
716
|
+
throw error;
|
|
717
|
+
if (await this.adoptOwnCommit(next))
|
|
718
|
+
return next;
|
|
719
|
+
if (attempt >= OPTIMISTIC_MAX_ATTEMPTS) {
|
|
720
|
+
throw new Error(`Object storage: room "${this.roomId}" lost the manifest CAS ` + `${OPTIMISTIC_MAX_ATTEMPTS} times in a row. That is sustained write ` + `contention on one room, which more retries will not fix — route the ` + `room to fewer writers, or use concurrency: "single-writer".`);
|
|
721
|
+
}
|
|
722
|
+
await this.load();
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
buildNext() {
|
|
727
|
+
return {
|
|
728
|
+
...this.cached,
|
|
729
|
+
epoch: this.concurrency === "optimistic" ? this.cached.epoch : this.leaseEpoch,
|
|
730
|
+
commitSeq: this.cached.commitSeq + 1,
|
|
731
|
+
lastWriter: this.writerId,
|
|
732
|
+
walSegs: [...this.cached.walSegs],
|
|
733
|
+
meta: { ...this.cached.meta },
|
|
734
|
+
pendingGc: [...this.cached.pendingGc]
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
async refresh() {
|
|
738
|
+
const record = await this.driver.get(this.manifestKey);
|
|
739
|
+
if (!record)
|
|
740
|
+
return this.cached;
|
|
741
|
+
const stored = decodeJson(record.body, this.manifestKey);
|
|
742
|
+
if (typeof stored.commitSeq !== "number")
|
|
743
|
+
stored.commitSeq = 0;
|
|
744
|
+
if (typeof stored.lastWriter !== "string")
|
|
745
|
+
stored.lastWriter = "";
|
|
746
|
+
this.cached = stored;
|
|
747
|
+
this.cachedEtag = record.etag;
|
|
748
|
+
this.loaded = true;
|
|
749
|
+
return stored;
|
|
750
|
+
}
|
|
751
|
+
async doCommit(mutate) {
|
|
752
|
+
this.assertNotFenced();
|
|
753
|
+
if (this.stopped) {
|
|
754
|
+
throw new Error(`Object storage: room "${this.roomId}" is closed; refusing to commit its manifest.`);
|
|
755
|
+
}
|
|
756
|
+
if (!this.loaded)
|
|
757
|
+
await this.load();
|
|
758
|
+
if (!this.held)
|
|
759
|
+
await this.acquire();
|
|
760
|
+
const next = this.buildNext();
|
|
761
|
+
mutate(next);
|
|
762
|
+
let etag;
|
|
763
|
+
try {
|
|
764
|
+
etag = await this.driver.put(this.manifestKey, encodeJson(next), this.cachedEtag ? { ifMatch: this.cachedEtag } : this.createOptions());
|
|
765
|
+
} catch (error) {
|
|
766
|
+
if (error instanceof PreconditionFailedError) {
|
|
767
|
+
const adopted = await this.adoptOwnCommit(next);
|
|
768
|
+
if (adopted)
|
|
769
|
+
return next;
|
|
770
|
+
this.fence("the manifest CAS was lost — another writer owns this room");
|
|
771
|
+
throw new NotWriterError(this.roomId, "manifest CAS failed; another writer committed against this etag");
|
|
772
|
+
}
|
|
773
|
+
throw error;
|
|
774
|
+
}
|
|
775
|
+
this.cached = next;
|
|
776
|
+
this.cachedEtag = etag;
|
|
777
|
+
this.wroteSinceRenew = true;
|
|
778
|
+
return next;
|
|
779
|
+
}
|
|
780
|
+
createOptions() {
|
|
781
|
+
if (this.driver.caps.casWildcard)
|
|
782
|
+
return { ifNoneMatch: "*" };
|
|
783
|
+
throw new Error(`Object storage: this driver does not support create-if-absent ` + `(caps.casWildcard === false, e.g. MinIO), and room "${this.roomId}" has not ` + `been seeded. Run the adapter's init() once as a deploy step before starting ` + `writers. See docs/object-storage.md#the-minio-gotcha.`);
|
|
784
|
+
}
|
|
785
|
+
async adoptOwnCommit(attempted) {
|
|
786
|
+
let current;
|
|
787
|
+
try {
|
|
788
|
+
current = await this.driver.get(this.manifestKey);
|
|
789
|
+
} catch {
|
|
790
|
+
return false;
|
|
791
|
+
}
|
|
792
|
+
if (!current)
|
|
793
|
+
return false;
|
|
794
|
+
let stored;
|
|
795
|
+
try {
|
|
796
|
+
stored = decodeJson(current.body, this.manifestKey);
|
|
797
|
+
} catch {
|
|
798
|
+
return false;
|
|
799
|
+
}
|
|
800
|
+
if (stored.lastWriter !== this.writerId || stored.commitSeq !== attempted.commitSeq || stored.epoch !== attempted.epoch) {
|
|
801
|
+
return false;
|
|
802
|
+
}
|
|
803
|
+
this.cached = stored;
|
|
804
|
+
this.cachedEtag = current.etag;
|
|
805
|
+
this.wroteSinceRenew = true;
|
|
806
|
+
return true;
|
|
807
|
+
}
|
|
808
|
+
assertNotFenced() {
|
|
809
|
+
if (this.fenceReason)
|
|
810
|
+
throw new NotWriterError(this.roomId, this.fenceReason);
|
|
811
|
+
}
|
|
812
|
+
fence(reason) {
|
|
813
|
+
if (this.fenceReason)
|
|
814
|
+
return;
|
|
815
|
+
this.fenceReason = reason;
|
|
816
|
+
this.held = false;
|
|
817
|
+
this.disarmRenew();
|
|
818
|
+
}
|
|
819
|
+
armRenew() {
|
|
820
|
+
this.disarmRenew();
|
|
821
|
+
if (this.stopped)
|
|
822
|
+
return;
|
|
823
|
+
this.wroteSinceRenew = false;
|
|
824
|
+
this.cancelRenew = this.clock.setTimer(() => {
|
|
825
|
+
this.onRenewTick();
|
|
826
|
+
}, this.leaseConfig.renewMs);
|
|
827
|
+
}
|
|
828
|
+
disarmRenew() {
|
|
829
|
+
this.cancelRenew?.();
|
|
830
|
+
this.cancelRenew = null;
|
|
831
|
+
}
|
|
832
|
+
async onRenewTick() {
|
|
833
|
+
this.cancelRenew = null;
|
|
834
|
+
if (this.stopped || !this.held || this.fenceReason)
|
|
835
|
+
return;
|
|
836
|
+
if (this.leaseConfig.mode === "on-write" && !this.wroteSinceRenew) {
|
|
837
|
+
this.held = false;
|
|
838
|
+
return;
|
|
839
|
+
}
|
|
840
|
+
try {
|
|
841
|
+
await this.renew();
|
|
842
|
+
this.armRenew();
|
|
843
|
+
} catch (error) {
|
|
844
|
+
if (error instanceof PreconditionFailedError || error instanceof NotWriterError) {
|
|
845
|
+
this.fence("lease renewal lost the CAS — another writer took this room");
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
848
|
+
if (this.clock.now() >= this.leaseExpiresAt) {
|
|
849
|
+
this.fence("lease expired while renewals were failing");
|
|
850
|
+
return;
|
|
851
|
+
}
|
|
852
|
+
this.cancelRenew = this.clock.setTimer(() => {
|
|
853
|
+
this.onRenewTick();
|
|
854
|
+
}, Math.max(1, Math.floor(this.leaseConfig.renewMs / 4)));
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
async renew() {
|
|
858
|
+
const now = this.clock.now();
|
|
859
|
+
const next = {
|
|
860
|
+
owner: this.writerId,
|
|
861
|
+
epoch: this.leaseEpoch,
|
|
862
|
+
expiresAt: now + this.leaseConfig.ttlMs
|
|
863
|
+
};
|
|
864
|
+
const etag = await this.driver.put(this.leaseKey, encodeJson(next), this.leaseEtag ? { ifMatch: this.leaseEtag } : this.createOptions());
|
|
865
|
+
this.leaseEtag = etag;
|
|
866
|
+
this.leaseExpiresAt = next.expiresAt;
|
|
867
|
+
this.wroteSinceRenew = false;
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
// src/server/storage/object/wal.ts
|
|
872
|
+
var UNAVAILABLE_AFTER_FAILURES = 3;
|
|
873
|
+
var RETRY_BASE_MS = 100;
|
|
874
|
+
var RETRY_MAX_MS = 1e4;
|
|
875
|
+
var encoder2 = new TextEncoder;
|
|
876
|
+
var decoder2 = new TextDecoder;
|
|
877
|
+
function encodeSegment(lines) {
|
|
878
|
+
return encoder2.encode(lines.join(`
|
|
879
|
+
`));
|
|
880
|
+
}
|
|
881
|
+
function decodeSegment(body, key) {
|
|
882
|
+
const text = decoder2.decode(body);
|
|
883
|
+
const records = [];
|
|
884
|
+
for (const line of text.split(`
|
|
885
|
+
`)) {
|
|
886
|
+
if (line.length === 0)
|
|
887
|
+
continue;
|
|
888
|
+
try {
|
|
889
|
+
records.push(JSON.parse(line));
|
|
890
|
+
} catch (error) {
|
|
891
|
+
throw new Error(`Object storage: WAL segment "${key}" contains an unparsable record. ` + `Refusing to boot with a partial log. (${String(error)})`);
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
return records;
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
class WalWriter {
|
|
898
|
+
driver;
|
|
899
|
+
manifest;
|
|
900
|
+
batch;
|
|
901
|
+
durability;
|
|
902
|
+
concurrency;
|
|
903
|
+
writerId;
|
|
904
|
+
clock;
|
|
905
|
+
prefix;
|
|
906
|
+
onCommitted;
|
|
907
|
+
buffer = [];
|
|
908
|
+
bufferedBytes = 0;
|
|
909
|
+
seq = 0;
|
|
910
|
+
seqEpoch = -1;
|
|
911
|
+
running = false;
|
|
912
|
+
stopped = false;
|
|
913
|
+
loop = null;
|
|
914
|
+
wakeWaiter = null;
|
|
915
|
+
stopSignal;
|
|
916
|
+
releaseStopSignal = () => {};
|
|
917
|
+
forced = false;
|
|
918
|
+
flushWaiters = [];
|
|
919
|
+
failures = 0;
|
|
920
|
+
backpressured = false;
|
|
921
|
+
currentHealth = "healthy";
|
|
922
|
+
terminal = null;
|
|
923
|
+
durableHead = null;
|
|
924
|
+
durableListeners = [];
|
|
925
|
+
healthListeners = [];
|
|
926
|
+
constructor(options) {
|
|
927
|
+
this.driver = options.driver;
|
|
928
|
+
this.manifest = options.manifest;
|
|
929
|
+
this.batch = options.batch;
|
|
930
|
+
this.durability = options.durability;
|
|
931
|
+
this.concurrency = options.concurrency ?? "single-writer";
|
|
932
|
+
this.writerId = options.writerId ?? "writer";
|
|
933
|
+
this.clock = options.clock ?? systemClock;
|
|
934
|
+
this.prefix = roomPrefix(options.roomId);
|
|
935
|
+
this.onCommitted = options.onCommitted;
|
|
936
|
+
this.stopSignal = new Promise((resolve) => {
|
|
937
|
+
this.releaseStopSignal = resolve;
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
get health() {
|
|
941
|
+
return this.currentHealth;
|
|
942
|
+
}
|
|
943
|
+
get durableHlc() {
|
|
944
|
+
return this.durableHead;
|
|
945
|
+
}
|
|
946
|
+
get bufferBytes() {
|
|
947
|
+
return this.bufferedBytes;
|
|
948
|
+
}
|
|
949
|
+
onDurable(callback) {
|
|
950
|
+
this.durableListeners.push(callback);
|
|
951
|
+
}
|
|
952
|
+
onHealthChange(callback) {
|
|
953
|
+
this.healthListeners.push(callback);
|
|
954
|
+
}
|
|
955
|
+
start() {
|
|
956
|
+
if (this.running || this.stopped)
|
|
957
|
+
return;
|
|
958
|
+
this.running = true;
|
|
959
|
+
this.loop = this.run().catch((error) => {
|
|
960
|
+
console.warn("[reflectdb] object storage: the WAL flush loop exited abnormally:", error);
|
|
961
|
+
});
|
|
962
|
+
}
|
|
963
|
+
prepare(record) {
|
|
964
|
+
this.assertWritable();
|
|
965
|
+
const line = JSON.stringify(record);
|
|
966
|
+
const bytes = line.length + 1;
|
|
967
|
+
let durable = this.durability === "durable";
|
|
968
|
+
if (this.bufferedBytes + bytes > this.batch.maxBufferBytes) {
|
|
969
|
+
if (this.batch.onBackpressure === "reject") {
|
|
970
|
+
throw new BackpressureError(this.bufferedBytes + bytes, this.batch.maxBufferBytes);
|
|
971
|
+
}
|
|
972
|
+
this.setBackpressured(true);
|
|
973
|
+
durable = false;
|
|
974
|
+
}
|
|
975
|
+
return { record, line, bytes, hlc: recordHlc(record), durable };
|
|
976
|
+
}
|
|
977
|
+
append(prepared) {
|
|
978
|
+
this.assertWritable();
|
|
979
|
+
this.bufferedBytes += prepared.bytes;
|
|
980
|
+
let resolve = () => {};
|
|
981
|
+
let reject = () => {};
|
|
982
|
+
const settled = !prepared.durable;
|
|
983
|
+
const promise = settled ? Promise.resolve() : new Promise((res, rej) => {
|
|
984
|
+
resolve = res;
|
|
985
|
+
reject = rej;
|
|
986
|
+
});
|
|
987
|
+
this.buffer.push({ ...prepared, resolve, reject, settled });
|
|
988
|
+
this.start();
|
|
989
|
+
this.wake();
|
|
990
|
+
return promise;
|
|
991
|
+
}
|
|
992
|
+
enqueue(record) {
|
|
993
|
+
return this.append(this.prepare(record));
|
|
994
|
+
}
|
|
995
|
+
flush() {
|
|
996
|
+
if (this.terminal)
|
|
997
|
+
return Promise.reject(this.terminal);
|
|
998
|
+
if (this.buffer.length === 0)
|
|
999
|
+
return Promise.resolve();
|
|
1000
|
+
if (this.stopped) {
|
|
1001
|
+
return Promise.reject(new Error(`Object storage: flush() called after the adapter was closed, with ` + `${this.buffer.length} record(s) still buffered. They were not made durable.`));
|
|
1002
|
+
}
|
|
1003
|
+
this.forced = true;
|
|
1004
|
+
this.start();
|
|
1005
|
+
this.wake();
|
|
1006
|
+
return new Promise((resolve, reject) => {
|
|
1007
|
+
this.flushWaiters.push({ resolve, reject });
|
|
1008
|
+
});
|
|
1009
|
+
}
|
|
1010
|
+
async stop(timeoutMs) {
|
|
1011
|
+
this.stopped = true;
|
|
1012
|
+
if (!this.running && !this.loop)
|
|
1013
|
+
return;
|
|
1014
|
+
this.running = false;
|
|
1015
|
+
this.releaseStopSignal();
|
|
1016
|
+
this.wake();
|
|
1017
|
+
const settled = this.loop?.catch(() => {
|
|
1018
|
+
return;
|
|
1019
|
+
});
|
|
1020
|
+
if (settled) {
|
|
1021
|
+
await (timeoutMs === undefined ? settled : Promise.race([settled, this.clock.delay(timeoutMs)]));
|
|
1022
|
+
}
|
|
1023
|
+
this.loop = null;
|
|
1024
|
+
if (this.buffer.length > 0 && this.flushWaiters.length > 0) {
|
|
1025
|
+
const error = this.terminal ?? new Error(`Object storage: the adapter was closed with ${this.buffer.length} record(s) still ` + `buffered; they were not made durable. Increase shutdownFlushMs, or check ` + `storage health — the store was not accepting writes.`);
|
|
1026
|
+
const waiters = this.flushWaiters;
|
|
1027
|
+
this.flushWaiters = [];
|
|
1028
|
+
for (const waiter of waiters)
|
|
1029
|
+
waiter.reject(error);
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
async run() {
|
|
1033
|
+
while (this.running) {
|
|
1034
|
+
if (this.buffer.length === 0) {
|
|
1035
|
+
this.settleFlushWaiters();
|
|
1036
|
+
await this.waitNonEmpty();
|
|
1037
|
+
if (!this.running)
|
|
1038
|
+
break;
|
|
1039
|
+
if (this.buffer.length === 0)
|
|
1040
|
+
continue;
|
|
1041
|
+
}
|
|
1042
|
+
if (this.batch.minLingerMs > 0 && !this.forced) {
|
|
1043
|
+
await this.interruptibleDelay(this.batch.minLingerMs);
|
|
1044
|
+
}
|
|
1045
|
+
await this.flushOnce();
|
|
1046
|
+
}
|
|
1047
|
+
this.settleFlushWaiters();
|
|
1048
|
+
}
|
|
1049
|
+
async flushOnce() {
|
|
1050
|
+
if (this.buffer.length === 0)
|
|
1051
|
+
return;
|
|
1052
|
+
let epoch;
|
|
1053
|
+
try {
|
|
1054
|
+
await this.manifest.ensureWriter();
|
|
1055
|
+
epoch = this.manifest.epoch;
|
|
1056
|
+
} catch (error) {
|
|
1057
|
+
await this.onFlushError(error);
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
if (epoch !== this.seqEpoch) {
|
|
1061
|
+
this.seqEpoch = epoch;
|
|
1062
|
+
this.seq = 0;
|
|
1063
|
+
}
|
|
1064
|
+
let count = 0;
|
|
1065
|
+
let bytes = 0;
|
|
1066
|
+
let maxHlc = "";
|
|
1067
|
+
const lines = [];
|
|
1068
|
+
for (const item of this.buffer) {
|
|
1069
|
+
if (count > 0 && bytes + item.bytes > this.batch.maxBytes)
|
|
1070
|
+
break;
|
|
1071
|
+
bytes += item.bytes;
|
|
1072
|
+
lines.push(item.line);
|
|
1073
|
+
if (item.hlc > maxHlc)
|
|
1074
|
+
maxHlc = item.hlc;
|
|
1075
|
+
count++;
|
|
1076
|
+
}
|
|
1077
|
+
const seq = this.seq;
|
|
1078
|
+
const token = this.concurrency === "optimistic" ? this.writerId : String(epoch);
|
|
1079
|
+
const key = `${this.prefix}wal/${token}-${seq}.jsonl`;
|
|
1080
|
+
const body = encodeSegment(lines);
|
|
1081
|
+
const ref = { key, epoch, seq, bytes: body.length, maxHlc };
|
|
1082
|
+
this.seq++;
|
|
1083
|
+
try {
|
|
1084
|
+
await this.driver.put(key, body);
|
|
1085
|
+
await this.manifest.commit((manifest) => {
|
|
1086
|
+
manifest.walSegs.push(ref);
|
|
1087
|
+
if (maxHlc && (!manifest.oplogHead || maxHlc > manifest.oplogHead)) {
|
|
1088
|
+
manifest.oplogHead = maxHlc;
|
|
1089
|
+
}
|
|
1090
|
+
});
|
|
1091
|
+
} catch (error) {
|
|
1092
|
+
await this.onFlushError(error);
|
|
1093
|
+
return;
|
|
1094
|
+
}
|
|
1095
|
+
const committed = this.buffer.splice(0, count);
|
|
1096
|
+
this.bufferedBytes -= bytes;
|
|
1097
|
+
if (this.bufferedBytes <= this.batch.maxBufferBytes)
|
|
1098
|
+
this.setBackpressured(false);
|
|
1099
|
+
for (const item of committed) {
|
|
1100
|
+
if (!item.settled) {
|
|
1101
|
+
item.settled = true;
|
|
1102
|
+
item.resolve();
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
this.failures = 0;
|
|
1106
|
+
this.setHealth(this.backpressured ? "degraded" : "healthy");
|
|
1107
|
+
if (maxHlc && (!this.durableHead || maxHlc > this.durableHead)) {
|
|
1108
|
+
this.durableHead = maxHlc;
|
|
1109
|
+
for (const listener of this.durableListeners)
|
|
1110
|
+
listener(maxHlc);
|
|
1111
|
+
}
|
|
1112
|
+
if (this.buffer.length === 0) {
|
|
1113
|
+
this.forced = false;
|
|
1114
|
+
this.settleFlushWaiters();
|
|
1115
|
+
}
|
|
1116
|
+
if (this.onCommitted) {
|
|
1117
|
+
try {
|
|
1118
|
+
await this.onCommitted();
|
|
1119
|
+
} catch (error) {
|
|
1120
|
+
console.warn("[reflectdb] object storage: post-commit maintenance failed:", error);
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
async onFlushError(error) {
|
|
1125
|
+
if (error instanceof NotWriterError) {
|
|
1126
|
+
this.fail(error);
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
this.failures++;
|
|
1130
|
+
this.setHealth(this.failures >= UNAVAILABLE_AFTER_FAILURES ? "unavailable" : "degraded");
|
|
1131
|
+
const backoff = Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** (this.failures - 1));
|
|
1132
|
+
await this.interruptibleDelay(backoff);
|
|
1133
|
+
}
|
|
1134
|
+
fail(error) {
|
|
1135
|
+
this.terminal = error;
|
|
1136
|
+
this.running = false;
|
|
1137
|
+
this.setHealth("unavailable");
|
|
1138
|
+
const pending = this.buffer;
|
|
1139
|
+
this.buffer = [];
|
|
1140
|
+
this.bufferedBytes = 0;
|
|
1141
|
+
for (const item of pending) {
|
|
1142
|
+
if (!item.settled) {
|
|
1143
|
+
item.settled = true;
|
|
1144
|
+
item.reject(error);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
const waiters = this.flushWaiters;
|
|
1148
|
+
this.flushWaiters = [];
|
|
1149
|
+
for (const waiter of waiters)
|
|
1150
|
+
waiter.reject(error);
|
|
1151
|
+
this.releaseStopSignal();
|
|
1152
|
+
this.wake();
|
|
1153
|
+
}
|
|
1154
|
+
assertWritable() {
|
|
1155
|
+
if (this.terminal)
|
|
1156
|
+
throw this.terminal;
|
|
1157
|
+
if (this.stopped) {
|
|
1158
|
+
throw new Error("Object storage: the WAL writer is stopped; the adapter was closed.");
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
wake() {
|
|
1162
|
+
const waiter = this.wakeWaiter;
|
|
1163
|
+
this.wakeWaiter = null;
|
|
1164
|
+
waiter?.();
|
|
1165
|
+
}
|
|
1166
|
+
waitNonEmpty() {
|
|
1167
|
+
if (this.buffer.length > 0 || !this.running)
|
|
1168
|
+
return Promise.resolve();
|
|
1169
|
+
return new Promise((resolve) => {
|
|
1170
|
+
this.wakeWaiter = resolve;
|
|
1171
|
+
});
|
|
1172
|
+
}
|
|
1173
|
+
async interruptibleDelay(ms) {
|
|
1174
|
+
await Promise.race([this.clock.delay(ms), this.stopSignal]);
|
|
1175
|
+
}
|
|
1176
|
+
settleFlushWaiters() {
|
|
1177
|
+
if (this.buffer.length > 0)
|
|
1178
|
+
return;
|
|
1179
|
+
this.forced = false;
|
|
1180
|
+
const waiters = this.flushWaiters;
|
|
1181
|
+
this.flushWaiters = [];
|
|
1182
|
+
for (const waiter of waiters)
|
|
1183
|
+
waiter.resolve();
|
|
1184
|
+
}
|
|
1185
|
+
setBackpressured(value) {
|
|
1186
|
+
if (this.backpressured === value)
|
|
1187
|
+
return;
|
|
1188
|
+
this.backpressured = value;
|
|
1189
|
+
if (this.failures === 0)
|
|
1190
|
+
this.setHealth(value ? "degraded" : "healthy");
|
|
1191
|
+
}
|
|
1192
|
+
setHealth(health) {
|
|
1193
|
+
if (this.currentHealth === health)
|
|
1194
|
+
return;
|
|
1195
|
+
this.currentHealth = health;
|
|
1196
|
+
for (const listener of this.healthListeners)
|
|
1197
|
+
listener(health);
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
// src/server/storage/object/sigv4.ts
|
|
1202
|
+
var ALGORITHM = "AWS4-HMAC-SHA256";
|
|
1203
|
+
var EMPTY_PAYLOAD_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
|
1204
|
+
var UNRESERVED = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~";
|
|
1205
|
+
var textEncoder = new TextEncoder;
|
|
1206
|
+
function subtle() {
|
|
1207
|
+
const webCrypto = globalThis.crypto;
|
|
1208
|
+
if (!webCrypto?.subtle) {
|
|
1209
|
+
throw new Error("The S3 driver needs WebCrypto (globalThis.crypto.subtle) to sign requests, and this runtime does not expose it. It is available on Bun, Node >= 18, Deno, Cloudflare Workers and browsers; on Node 18 outside a secure context it may need --experimental-global-webcrypto.");
|
|
1210
|
+
}
|
|
1211
|
+
return webCrypto.subtle;
|
|
1212
|
+
}
|
|
1213
|
+
function toHex(bytes) {
|
|
1214
|
+
let out = "";
|
|
1215
|
+
for (const byte of bytes)
|
|
1216
|
+
out += byte.toString(16).padStart(2, "0");
|
|
1217
|
+
return out;
|
|
1218
|
+
}
|
|
1219
|
+
async function hexSha256(data) {
|
|
1220
|
+
const bytes = typeof data === "string" ? textEncoder.encode(data) : data;
|
|
1221
|
+
return toHex(new Uint8Array(await subtle().digest("SHA-256", bytes)));
|
|
1222
|
+
}
|
|
1223
|
+
async function hmacSha256(key, data) {
|
|
1224
|
+
const bytes = typeof data === "string" ? textEncoder.encode(data) : data;
|
|
1225
|
+
const cryptoKey = await subtle().importKey("raw", key, { name: "HMAC", hash: "SHA-256" }, false, ["sign"]);
|
|
1226
|
+
return new Uint8Array(await subtle().sign("HMAC", cryptoKey, bytes));
|
|
1227
|
+
}
|
|
1228
|
+
function percentEncode(value) {
|
|
1229
|
+
let out = "";
|
|
1230
|
+
for (const byte of textEncoder.encode(value)) {
|
|
1231
|
+
const char = String.fromCharCode(byte);
|
|
1232
|
+
out += byte < 128 && UNRESERVED.includes(char) ? char : `%${byte.toString(16).toUpperCase().padStart(2, "0")}`;
|
|
1233
|
+
}
|
|
1234
|
+
return out;
|
|
1235
|
+
}
|
|
1236
|
+
function encodeUriPath(path) {
|
|
1237
|
+
return path.split("/").map(percentEncode).join("/");
|
|
1238
|
+
}
|
|
1239
|
+
function canonicalQueryString(params) {
|
|
1240
|
+
const encoded = [...params].map(([key, value]) => [percentEncode(key), percentEncode(value)]);
|
|
1241
|
+
encoded.sort((a, b) => {
|
|
1242
|
+
if (a[0] !== b[0])
|
|
1243
|
+
return a[0] < b[0] ? -1 : 1;
|
|
1244
|
+
if (a[1] !== b[1])
|
|
1245
|
+
return a[1] < b[1] ? -1 : 1;
|
|
1246
|
+
return 0;
|
|
1247
|
+
});
|
|
1248
|
+
return encoded.map(([key, value]) => `${key}=${value}`).join("&");
|
|
1249
|
+
}
|
|
1250
|
+
function canonicalHeaders(headers) {
|
|
1251
|
+
const entries = Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value.trim().replace(/\s+/g, " ")]).sort((a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0);
|
|
1252
|
+
return {
|
|
1253
|
+
canonical: entries.map(([name, value]) => `${name}:${value}
|
|
1254
|
+
`).join(""),
|
|
1255
|
+
signedHeaders: entries.map(([name]) => name).join(";")
|
|
1256
|
+
};
|
|
1257
|
+
}
|
|
1258
|
+
function canonicalRequest(input) {
|
|
1259
|
+
const { canonical: headerBlock, signedHeaders } = canonicalHeaders(input.headers);
|
|
1260
|
+
const canonicalUri = input.url.pathname || "/";
|
|
1261
|
+
const canonical = [
|
|
1262
|
+
input.method.toUpperCase(),
|
|
1263
|
+
canonicalUri,
|
|
1264
|
+
canonicalQueryString(input.url.searchParams),
|
|
1265
|
+
headerBlock,
|
|
1266
|
+
signedHeaders,
|
|
1267
|
+
input.payloadHash
|
|
1268
|
+
].join(`
|
|
1269
|
+
`);
|
|
1270
|
+
return { canonical, signedHeaders };
|
|
1271
|
+
}
|
|
1272
|
+
function formatAmzDate(now) {
|
|
1273
|
+
return now.toISOString().replace(/[:-]/g, "").replace(/\.\d+/, "");
|
|
1274
|
+
}
|
|
1275
|
+
async function deriveSigningKey(secret, dateStamp, region, service) {
|
|
1276
|
+
const dateKey = await hmacSha256(textEncoder.encode(`AWS4${secret}`), dateStamp);
|
|
1277
|
+
const regionKey = await hmacSha256(dateKey, region);
|
|
1278
|
+
const serviceKey = await hmacSha256(regionKey, service);
|
|
1279
|
+
return hmacSha256(serviceKey, "aws4_request");
|
|
1280
|
+
}
|
|
1281
|
+
async function signRequest(input) {
|
|
1282
|
+
const { method, url, credentials, region, service, now } = input;
|
|
1283
|
+
const headers = {};
|
|
1284
|
+
for (const [name, value] of Object.entries(input.headers))
|
|
1285
|
+
headers[name.toLowerCase()] = value;
|
|
1286
|
+
headers.host = url.host;
|
|
1287
|
+
const payloadHash = input.body ? await hexSha256(input.body) : EMPTY_PAYLOAD_SHA256;
|
|
1288
|
+
headers["x-amz-content-sha256"] = payloadHash;
|
|
1289
|
+
const amzDate = formatAmzDate(now);
|
|
1290
|
+
headers["x-amz-date"] = amzDate;
|
|
1291
|
+
if (credentials.sessionToken)
|
|
1292
|
+
headers["x-amz-security-token"] = credentials.sessionToken;
|
|
1293
|
+
const { canonical, signedHeaders } = canonicalRequest({ method, url, headers, payloadHash });
|
|
1294
|
+
const dateStamp = amzDate.slice(0, 8);
|
|
1295
|
+
const scope = `${dateStamp}/${region}/${service}/aws4_request`;
|
|
1296
|
+
const stringToSign = [ALGORITHM, amzDate, scope, await hexSha256(canonical)].join(`
|
|
1297
|
+
`);
|
|
1298
|
+
const signingKey = await deriveSigningKey(credentials.secret, dateStamp, region, service);
|
|
1299
|
+
const signature = toHex(await hmacSha256(signingKey, stringToSign));
|
|
1300
|
+
headers.authorization = `${ALGORITHM} Credential=${credentials.keyId}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
|
|
1301
|
+
return headers;
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
// src/server/storage/object/drivers/s3.ts
|
|
1305
|
+
var textEncoder2 = new TextEncoder;
|
|
1306
|
+
var MAX_KEYS_PER_DELETE = 1000;
|
|
1307
|
+
var MAX_RETRIES = 3;
|
|
1308
|
+
var RETRY_BASE_MS2 = 100;
|
|
1309
|
+
var MAX_RETRY_AFTER_MS = 20000;
|
|
1310
|
+
var DELETE_CONCURRENCY = 8;
|
|
1311
|
+
function missingEndpoint(provider) {
|
|
1312
|
+
throw new Error(`store.endpoint is required for provider "${provider}": there is no well-known endpoint to default to. Pass the URL of your ${provider} server, e.g. { provider: "${provider}", endpoint: "http://localhost:9000", bucket: "..." }.`);
|
|
1313
|
+
}
|
|
1314
|
+
var PROVIDER_PRESETS = {
|
|
1315
|
+
aws: {
|
|
1316
|
+
endpoint: (config) => `https://s3.${config.region ?? "us-east-1"}.amazonaws.com`,
|
|
1317
|
+
region: "us-east-1",
|
|
1318
|
+
urlStyle: "vhost",
|
|
1319
|
+
casWildcard: true,
|
|
1320
|
+
batchDelete: true
|
|
1321
|
+
},
|
|
1322
|
+
r2: {
|
|
1323
|
+
endpoint: (config) => {
|
|
1324
|
+
if (!config.accountId) {
|
|
1325
|
+
throw new Error(`store.accountId is required for provider "r2": the R2 endpoint is https://<accountId>.r2.cloudflarestorage.com and cannot be derived from the bucket name. Find it in the Cloudflare dashboard under R2 > Overview, or pass store.endpoint explicitly.`);
|
|
1326
|
+
}
|
|
1327
|
+
return `https://${config.accountId}.r2.cloudflarestorage.com`;
|
|
1328
|
+
},
|
|
1329
|
+
region: "auto",
|
|
1330
|
+
urlStyle: "vhost",
|
|
1331
|
+
casWildcard: true,
|
|
1332
|
+
batchDelete: true
|
|
1333
|
+
},
|
|
1334
|
+
tigris: {
|
|
1335
|
+
endpoint: () => "https://t3.storage.dev",
|
|
1336
|
+
region: "auto",
|
|
1337
|
+
urlStyle: "vhost",
|
|
1338
|
+
casWildcard: true,
|
|
1339
|
+
batchDelete: true
|
|
1340
|
+
},
|
|
1341
|
+
minio: {
|
|
1342
|
+
endpoint: () => missingEndpoint("minio"),
|
|
1343
|
+
region: "us-east-1",
|
|
1344
|
+
urlStyle: "path",
|
|
1345
|
+
casWildcard: false,
|
|
1346
|
+
batchDelete: true
|
|
1347
|
+
},
|
|
1348
|
+
gcs: {
|
|
1349
|
+
endpoint: () => "https://storage.googleapis.com",
|
|
1350
|
+
region: "auto",
|
|
1351
|
+
urlStyle: "path",
|
|
1352
|
+
casWildcard: true,
|
|
1353
|
+
batchDelete: false
|
|
1354
|
+
}
|
|
1355
|
+
};
|
|
1356
|
+
var GENERIC_PRESET = {
|
|
1357
|
+
endpoint: () => {
|
|
1358
|
+
throw new Error(`store.endpoint is required when store.provider is not set. Either name a provider ("aws" | "r2" | "tigris" | "minio" | "gcs") so the endpoint can be derived, or pass the endpoint URL explicitly.`);
|
|
1359
|
+
},
|
|
1360
|
+
region: "auto",
|
|
1361
|
+
urlStyle: "path",
|
|
1362
|
+
casWildcard: true,
|
|
1363
|
+
batchDelete: true
|
|
1364
|
+
};
|
|
1365
|
+
function isRetryable(status) {
|
|
1366
|
+
if (status === null)
|
|
1367
|
+
return true;
|
|
1368
|
+
if (status === 412 || status === 409)
|
|
1369
|
+
return false;
|
|
1370
|
+
if (status === 429)
|
|
1371
|
+
return true;
|
|
1372
|
+
return status >= 500;
|
|
1373
|
+
}
|
|
1374
|
+
function retryDelayMs(attempt, response) {
|
|
1375
|
+
const retryAfter = response?.headers.get("retry-after");
|
|
1376
|
+
if (retryAfter) {
|
|
1377
|
+
const seconds = Number(retryAfter);
|
|
1378
|
+
if (Number.isFinite(seconds) && seconds >= 0) {
|
|
1379
|
+
return Math.min(seconds * 1000, MAX_RETRY_AFTER_MS);
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
return Math.random() * RETRY_BASE_MS2 * 2 ** attempt;
|
|
1383
|
+
}
|
|
1384
|
+
var CONTENTS_RE = /<Contents>([\s\S]*?)<\/Contents>/g;
|
|
1385
|
+
var KEY_RE = /<Key>([\s\S]*?)<\/Key>/;
|
|
1386
|
+
var SIZE_RE = /<Size>\s*(\d+)\s*<\/Size>/;
|
|
1387
|
+
var TRUNCATED_RE = /<IsTruncated>\s*true\s*<\/IsTruncated>/i;
|
|
1388
|
+
var NEXT_TOKEN_RE = /<NextContinuationToken>([\s\S]*?)<\/NextContinuationToken>/;
|
|
1389
|
+
var DELETE_ERROR_RE = /<Error>([\s\S]*?)<\/Error>/g;
|
|
1390
|
+
var CODE_RE = /<Code>([\s\S]*?)<\/Code>/;
|
|
1391
|
+
var MESSAGE_RE = /<Message>([\s\S]*?)<\/Message>/;
|
|
1392
|
+
function xmlUnescape(value) {
|
|
1393
|
+
return value.replace(/&(#x?[0-9a-fA-F]+|amp|lt|gt|quot|apos);/g, (entity, name) => {
|
|
1394
|
+
switch (name) {
|
|
1395
|
+
case "amp":
|
|
1396
|
+
return "&";
|
|
1397
|
+
case "lt":
|
|
1398
|
+
return "<";
|
|
1399
|
+
case "gt":
|
|
1400
|
+
return ">";
|
|
1401
|
+
case "quot":
|
|
1402
|
+
return '"';
|
|
1403
|
+
case "apos":
|
|
1404
|
+
return "'";
|
|
1405
|
+
default: {
|
|
1406
|
+
const code = name.startsWith("#x") ? Number.parseInt(name.slice(2), 16) : Number.parseInt(name.slice(1), 10);
|
|
1407
|
+
return Number.isFinite(code) ? String.fromCodePoint(code) : entity;
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
});
|
|
1411
|
+
}
|
|
1412
|
+
function xmlEscape(value) {
|
|
1413
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
1414
|
+
}
|
|
1415
|
+
var crcTable;
|
|
1416
|
+
function crc32(bytes) {
|
|
1417
|
+
if (!crcTable) {
|
|
1418
|
+
crcTable = new Uint32Array(256);
|
|
1419
|
+
for (let i = 0;i < 256; i++) {
|
|
1420
|
+
let value = i;
|
|
1421
|
+
for (let bit = 0;bit < 8; bit++) {
|
|
1422
|
+
value = value & 1 ? 3988292384 ^ value >>> 1 : value >>> 1;
|
|
1423
|
+
}
|
|
1424
|
+
crcTable[i] = value >>> 0;
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
let crc = 4294967295;
|
|
1428
|
+
for (const byte of bytes)
|
|
1429
|
+
crc = crcTable[(crc ^ byte) & 255] ^ crc >>> 8;
|
|
1430
|
+
return (crc ^ 4294967295) >>> 0;
|
|
1431
|
+
}
|
|
1432
|
+
function base64Crc32(bytes) {
|
|
1433
|
+
const crc = crc32(bytes);
|
|
1434
|
+
return btoa(String.fromCharCode(crc >>> 24 & 255, crc >>> 16 & 255, crc >>> 8 & 255, crc & 255));
|
|
1435
|
+
}
|
|
1436
|
+
function searchString(query) {
|
|
1437
|
+
return [...query].map(([key, value]) => [percentEncode(key), percentEncode(value)]).sort((a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0).map(([key, value]) => value === "" ? key : `${key}=${value}`).join("&");
|
|
1438
|
+
}
|
|
1439
|
+
function createS3Driver(config) {
|
|
1440
|
+
const preset = config.provider ? PROVIDER_PRESETS[config.provider] : GENERIC_PRESET;
|
|
1441
|
+
const providerName = config.provider ?? "custom";
|
|
1442
|
+
const region = config.region ?? preset.region;
|
|
1443
|
+
const urlStyle = config.urlStyle ?? preset.urlStyle;
|
|
1444
|
+
const endpoint = parseEndpoint(config.endpoint ?? preset.endpoint(config));
|
|
1445
|
+
const bucket = config.bucket;
|
|
1446
|
+
const credentials = config.credentials;
|
|
1447
|
+
const doFetch = config.fetch ?? globalThis.fetch;
|
|
1448
|
+
const now = config.now ?? (() => new Date);
|
|
1449
|
+
const sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1450
|
+
const caps = { casWildcard: preset.casWildcard };
|
|
1451
|
+
const prefixRoot = (config.prefix ?? "").replace(/^\/+|\/+$/g, "");
|
|
1452
|
+
function toStoreKey(key) {
|
|
1453
|
+
const trimmed = key.replace(/^\/+/, "");
|
|
1454
|
+
return prefixRoot ? `${prefixRoot}/${trimmed}` : trimmed;
|
|
1455
|
+
}
|
|
1456
|
+
function fromStoreKey(key) {
|
|
1457
|
+
if (!prefixRoot)
|
|
1458
|
+
return key;
|
|
1459
|
+
return key.startsWith(`${prefixRoot}/`) ? key.slice(prefixRoot.length + 1) : key;
|
|
1460
|
+
}
|
|
1461
|
+
function buildUrl(key, query = []) {
|
|
1462
|
+
const host = urlStyle === "vhost" ? `${bucket}.${endpoint.host}` : endpoint.host;
|
|
1463
|
+
let path = endpoint.basePath;
|
|
1464
|
+
if (urlStyle === "path")
|
|
1465
|
+
path += `/${encodeUriPath(bucket)}`;
|
|
1466
|
+
if (key !== null)
|
|
1467
|
+
path += `/${encodeUriPath(toStoreKey(key))}`;
|
|
1468
|
+
const url = new URL(`${endpoint.protocol}//${host}${path || "/"}`);
|
|
1469
|
+
if (query.length > 0)
|
|
1470
|
+
url.search = searchString(query);
|
|
1471
|
+
return url;
|
|
1472
|
+
}
|
|
1473
|
+
async function send(spec) {
|
|
1474
|
+
let lastError;
|
|
1475
|
+
for (let attempt = 0;; attempt++) {
|
|
1476
|
+
let response;
|
|
1477
|
+
try {
|
|
1478
|
+
const headers = await signRequest({
|
|
1479
|
+
method: spec.method,
|
|
1480
|
+
url: spec.url,
|
|
1481
|
+
headers: spec.headers ?? {},
|
|
1482
|
+
body: spec.body,
|
|
1483
|
+
region,
|
|
1484
|
+
service: "s3",
|
|
1485
|
+
credentials,
|
|
1486
|
+
now: now()
|
|
1487
|
+
});
|
|
1488
|
+
response = await doFetch(spec.url.toString(), {
|
|
1489
|
+
method: spec.method,
|
|
1490
|
+
headers,
|
|
1491
|
+
body: spec.body
|
|
1492
|
+
});
|
|
1493
|
+
} catch (error) {
|
|
1494
|
+
lastError = error;
|
|
1495
|
+
if (attempt >= MAX_RETRIES) {
|
|
1496
|
+
throw new Error(`S3 ${spec.method} ${spec.url.pathname} failed after ${attempt + 1} attempts: ${String(error)}`, { cause: lastError });
|
|
1497
|
+
}
|
|
1498
|
+
await sleep(retryDelayMs(attempt, null));
|
|
1499
|
+
continue;
|
|
1500
|
+
}
|
|
1501
|
+
if (response.ok || !isRetryable(response.status) || attempt >= MAX_RETRIES)
|
|
1502
|
+
return response;
|
|
1503
|
+
await response.text().catch(() => "");
|
|
1504
|
+
await sleep(retryDelayMs(attempt, response));
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
async function failure(operation, subject, response) {
|
|
1508
|
+
const body = await response.text().catch(() => "");
|
|
1509
|
+
const detail = body.trim().slice(0, 512);
|
|
1510
|
+
return new Error(`S3 ${operation} failed for "${subject}": ${response.status} ${response.statusText}${detail ? ` — ${detail}` : ""}`);
|
|
1511
|
+
}
|
|
1512
|
+
async function deleteBatch(keys) {
|
|
1513
|
+
const xml = `<?xml version="1.0" encoding="UTF-8"?><Delete><Quiet>true</Quiet>${keys.map((key) => `<Object><Key>${xmlEscape(toStoreKey(key))}</Key></Object>`).join("")}</Delete>`;
|
|
1514
|
+
const body = textEncoder2.encode(xml);
|
|
1515
|
+
const response = await send({
|
|
1516
|
+
method: "POST",
|
|
1517
|
+
url: buildUrl(null, [["delete", ""]]),
|
|
1518
|
+
headers: {
|
|
1519
|
+
"content-type": "application/xml",
|
|
1520
|
+
"x-amz-sdk-checksum-algorithm": "CRC32",
|
|
1521
|
+
"x-amz-checksum-crc32": base64Crc32(body)
|
|
1522
|
+
},
|
|
1523
|
+
body
|
|
1524
|
+
});
|
|
1525
|
+
if (!response.ok)
|
|
1526
|
+
throw await failure("DELETE (batch)", keys[0] ?? "", response);
|
|
1527
|
+
const text = await response.text();
|
|
1528
|
+
const failures = [...text.matchAll(DELETE_ERROR_RE)].map((match) => {
|
|
1529
|
+
const block = match[1] ?? "";
|
|
1530
|
+
const key = xmlUnescape(KEY_RE.exec(block)?.[1] ?? "?");
|
|
1531
|
+
const code = CODE_RE.exec(block)?.[1] ?? "Unknown";
|
|
1532
|
+
const message = MESSAGE_RE.exec(block)?.[1] ?? "";
|
|
1533
|
+
return `${key}: ${code}${message ? ` (${message})` : ""}`;
|
|
1534
|
+
});
|
|
1535
|
+
if (failures.length > 0) {
|
|
1536
|
+
throw new Error(`S3 batch delete reported ${failures.length} failure(s) — ${failures.join("; ")}`);
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
async function deleteIndividually(keys) {
|
|
1540
|
+
for (let i = 0;i < keys.length; i += DELETE_CONCURRENCY) {
|
|
1541
|
+
const window = keys.slice(i, i + DELETE_CONCURRENCY);
|
|
1542
|
+
await Promise.all(window.map(async (key) => {
|
|
1543
|
+
const response = await send({ method: "DELETE", url: buildUrl(key) });
|
|
1544
|
+
if (response.ok || response.status === 404) {
|
|
1545
|
+
await response.text().catch(() => "");
|
|
1546
|
+
return;
|
|
1547
|
+
}
|
|
1548
|
+
throw await failure("DELETE", key, response);
|
|
1549
|
+
}));
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
return {
|
|
1553
|
+
caps,
|
|
1554
|
+
async get(key) {
|
|
1555
|
+
const response = await send({ method: "GET", url: buildUrl(key) });
|
|
1556
|
+
if (response.status === 404) {
|
|
1557
|
+
await response.text().catch(() => "");
|
|
1558
|
+
return null;
|
|
1559
|
+
}
|
|
1560
|
+
if (!response.ok)
|
|
1561
|
+
throw await failure("GET", key, response);
|
|
1562
|
+
const etag = response.headers.get("etag");
|
|
1563
|
+
if (!etag) {
|
|
1564
|
+
throw new Error(`S3 GET of "${key}" returned no ETag header. The etag is the CAS token for every later conditional write, so continuing without one would turn a lost race into a silent overwrite.`);
|
|
1565
|
+
}
|
|
1566
|
+
return { body: new Uint8Array(await response.arrayBuffer()), etag };
|
|
1567
|
+
},
|
|
1568
|
+
async put(key, body, opts) {
|
|
1569
|
+
if (opts?.ifNoneMatch === "*" && !caps.casWildcard) {
|
|
1570
|
+
throw new Error(`Provider "${providerName}" does not support create-if-absent (If-None-Match: *); it requires an exact etag. Seed "_lease" and "_manifest" once with init() as a deploy step — not from N racing servers — after which every write is a plain ifMatch, which this store handles fine. See docs/object-storage.md, "The MinIO gotcha".`);
|
|
1571
|
+
}
|
|
1572
|
+
const headers = {};
|
|
1573
|
+
if (opts?.ifMatch)
|
|
1574
|
+
headers["if-match"] = opts.ifMatch;
|
|
1575
|
+
if (opts?.ifNoneMatch)
|
|
1576
|
+
headers["if-none-match"] = opts.ifNoneMatch;
|
|
1577
|
+
const response = await send({ method: "PUT", url: buildUrl(key), headers, body });
|
|
1578
|
+
if (response.status === 412 || response.status === 409) {
|
|
1579
|
+
await response.text().catch(() => "");
|
|
1580
|
+
throw new PreconditionFailedError(key);
|
|
1581
|
+
}
|
|
1582
|
+
if (!response.ok)
|
|
1583
|
+
throw await failure("PUT", key, response);
|
|
1584
|
+
const etag = response.headers.get("etag");
|
|
1585
|
+
if (!etag) {
|
|
1586
|
+
throw new Error(`S3 PUT of "${key}" returned no ETag header, so the next conditional write has nothing to CAS against.`);
|
|
1587
|
+
}
|
|
1588
|
+
await response.text().catch(() => "");
|
|
1589
|
+
return etag;
|
|
1590
|
+
},
|
|
1591
|
+
async list(prefix) {
|
|
1592
|
+
const fullPrefix = prefixRoot ? prefix ? `${prefixRoot}/${prefix.replace(/^\/+/, "")}` : `${prefixRoot}/` : prefix.replace(/^\/+/, "");
|
|
1593
|
+
const entries = [];
|
|
1594
|
+
let continuationToken;
|
|
1595
|
+
do {
|
|
1596
|
+
const query = [["list-type", "2"]];
|
|
1597
|
+
if (fullPrefix)
|
|
1598
|
+
query.push(["prefix", fullPrefix]);
|
|
1599
|
+
if (continuationToken)
|
|
1600
|
+
query.push(["continuation-token", continuationToken]);
|
|
1601
|
+
const response = await send({ method: "GET", url: buildUrl(null, query) });
|
|
1602
|
+
if (!response.ok)
|
|
1603
|
+
throw await failure("LIST", fullPrefix, response);
|
|
1604
|
+
const xml = await response.text();
|
|
1605
|
+
for (const match of xml.matchAll(CONTENTS_RE)) {
|
|
1606
|
+
const block = match[1] ?? "";
|
|
1607
|
+
const rawKey = KEY_RE.exec(block)?.[1];
|
|
1608
|
+
if (rawKey === undefined)
|
|
1609
|
+
continue;
|
|
1610
|
+
entries.push({
|
|
1611
|
+
key: fromStoreKey(xmlUnescape(rawKey)),
|
|
1612
|
+
size: Number(SIZE_RE.exec(block)?.[1] ?? 0)
|
|
1613
|
+
});
|
|
1614
|
+
}
|
|
1615
|
+
if (TRUNCATED_RE.test(xml)) {
|
|
1616
|
+
continuationToken = xmlUnescape(NEXT_TOKEN_RE.exec(xml)?.[1] ?? "");
|
|
1617
|
+
if (!continuationToken) {
|
|
1618
|
+
throw new Error(`S3 LIST of "${fullPrefix}" reported IsTruncated with no NextContinuationToken; refusing to loop on a malformed response.`);
|
|
1619
|
+
}
|
|
1620
|
+
} else {
|
|
1621
|
+
continuationToken = undefined;
|
|
1622
|
+
}
|
|
1623
|
+
} while (continuationToken);
|
|
1624
|
+
entries.sort((a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0);
|
|
1625
|
+
return entries;
|
|
1626
|
+
},
|
|
1627
|
+
async delete(keys) {
|
|
1628
|
+
if (keys.length === 0)
|
|
1629
|
+
return;
|
|
1630
|
+
if (!preset.batchDelete)
|
|
1631
|
+
return deleteIndividually(keys);
|
|
1632
|
+
for (let i = 0;i < keys.length; i += MAX_KEYS_PER_DELETE) {
|
|
1633
|
+
await deleteBatch(keys.slice(i, i + MAX_KEYS_PER_DELETE));
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
};
|
|
1637
|
+
}
|
|
1638
|
+
function parseEndpoint(raw) {
|
|
1639
|
+
const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(raw) ? raw : `https://${raw}`;
|
|
1640
|
+
let url;
|
|
1641
|
+
try {
|
|
1642
|
+
url = new URL(withScheme);
|
|
1643
|
+
} catch {
|
|
1644
|
+
throw new Error(`store.endpoint is not a valid URL: ${JSON.stringify(raw)}`);
|
|
1645
|
+
}
|
|
1646
|
+
return {
|
|
1647
|
+
protocol: url.protocol,
|
|
1648
|
+
host: url.host,
|
|
1649
|
+
basePath: url.pathname.replace(/\/+$/, "")
|
|
1650
|
+
};
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
// src/server/storage/object/drivers/memory.ts
|
|
1654
|
+
function compareKeys(a, b) {
|
|
1655
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
1656
|
+
}
|
|
1657
|
+
function delay(ms) {
|
|
1658
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1659
|
+
}
|
|
1660
|
+
function createMemoryDriver(options = {}) {
|
|
1661
|
+
const casWildcard = options.casWildcard ?? true;
|
|
1662
|
+
const store = new Map;
|
|
1663
|
+
let etagCounter = 0;
|
|
1664
|
+
let calls = 0;
|
|
1665
|
+
const stats = { get: 0, put: 0, list: 0, delete: 0 };
|
|
1666
|
+
async function enter(op, key) {
|
|
1667
|
+
calls += 1;
|
|
1668
|
+
stats[op] += 1;
|
|
1669
|
+
const fault = options.faults?.before?.({ op, key, call: calls });
|
|
1670
|
+
if (!fault)
|
|
1671
|
+
return;
|
|
1672
|
+
if ("delayMs" in fault && fault.delayMs > 0)
|
|
1673
|
+
await delay(fault.delayMs);
|
|
1674
|
+
if ("throw" in fault)
|
|
1675
|
+
throw fault.throw;
|
|
1676
|
+
}
|
|
1677
|
+
return {
|
|
1678
|
+
caps: { casWildcard },
|
|
1679
|
+
stats,
|
|
1680
|
+
dump() {
|
|
1681
|
+
const out = new Map;
|
|
1682
|
+
const sorted = [...store].sort((a, b) => compareKeys(a[0], b[0]));
|
|
1683
|
+
for (const [key, record] of sorted)
|
|
1684
|
+
out.set(key, new Uint8Array(record.body));
|
|
1685
|
+
return out;
|
|
1686
|
+
},
|
|
1687
|
+
async get(key) {
|
|
1688
|
+
await enter("get", key);
|
|
1689
|
+
const record = store.get(key);
|
|
1690
|
+
if (!record)
|
|
1691
|
+
return null;
|
|
1692
|
+
return { body: new Uint8Array(record.body), etag: record.etag };
|
|
1693
|
+
},
|
|
1694
|
+
async put(key, body, opts) {
|
|
1695
|
+
await enter("put", key);
|
|
1696
|
+
const wantsIfMatch = opts?.ifMatch !== undefined;
|
|
1697
|
+
const wantsIfNoneMatch = opts?.ifNoneMatch !== undefined;
|
|
1698
|
+
if (wantsIfMatch && wantsIfNoneMatch) {
|
|
1699
|
+
throw new Error(`put("${key}") passed both ifMatch and ifNoneMatch. A conditional write is either create-if-absent or overwrite-if-unchanged, never both.`);
|
|
1700
|
+
}
|
|
1701
|
+
if (wantsIfNoneMatch) {
|
|
1702
|
+
if (!casWildcard) {
|
|
1703
|
+
throw new Error(`put("${key}") used ifNoneMatch: "*", but this driver reports caps.casWildcard === false. Stores without wildcard CAS (MinIO) need a one-time init() that seeds _lease and _manifest unconditionally; every later write is a plain ifMatch.`);
|
|
1704
|
+
}
|
|
1705
|
+
if (store.has(key)) {
|
|
1706
|
+
throw new PreconditionFailedError(key, `Conditional write failed for "${key}": ifNoneMatch: "*" but the key already exists`);
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
if (wantsIfMatch) {
|
|
1710
|
+
const current = store.get(key);
|
|
1711
|
+
if (!current || current.etag !== opts?.ifMatch) {
|
|
1712
|
+
throw new PreconditionFailedError(key, `Conditional write failed for "${key}": ifMatch ${opts?.ifMatch} but stored etag is ${current ? current.etag : "(absent)"}`);
|
|
1713
|
+
}
|
|
1714
|
+
}
|
|
1715
|
+
etagCounter += 1;
|
|
1716
|
+
const etag = `"${etagCounter}"`;
|
|
1717
|
+
store.set(key, { body: new Uint8Array(body), etag });
|
|
1718
|
+
return etag;
|
|
1719
|
+
},
|
|
1720
|
+
async list(prefix) {
|
|
1721
|
+
await enter("list", prefix);
|
|
1722
|
+
const entries = [];
|
|
1723
|
+
for (const [key, record] of store) {
|
|
1724
|
+
if (key.startsWith(prefix))
|
|
1725
|
+
entries.push({ key, size: record.body.byteLength });
|
|
1726
|
+
}
|
|
1727
|
+
entries.sort((a, b) => compareKeys(a.key, b.key));
|
|
1728
|
+
return entries;
|
|
1729
|
+
},
|
|
1730
|
+
async delete(keys) {
|
|
1731
|
+
await enter("delete", keys.join(","));
|
|
1732
|
+
for (const key of keys)
|
|
1733
|
+
store.delete(key);
|
|
1734
|
+
}
|
|
1735
|
+
};
|
|
1736
|
+
}
|
|
1737
|
+
// src/server/node-require.ts
|
|
1738
|
+
var cached;
|
|
1739
|
+
function nodeRequire(id) {
|
|
1740
|
+
if (!cached) {
|
|
1741
|
+
const proc = globalThis.process;
|
|
1742
|
+
const mod = proc?.getBuiltinModule?.("node:module");
|
|
1743
|
+
if (!mod?.createRequire) {
|
|
1744
|
+
throw new Error(`Cannot resolve "${id}": this runtime has no CommonJS resolver (process.getBuiltinModule is unavailable). reflectdb requires Node >= 22.3 or Bun >= 1.1.30 on the server.`);
|
|
1745
|
+
}
|
|
1746
|
+
cached = mod.createRequire(module.filename);
|
|
1747
|
+
}
|
|
1748
|
+
return cached(id);
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
// src/server/storage/object/drivers/filesystem.ts
|
|
1752
|
+
function loadFs() {
|
|
1753
|
+
try {
|
|
1754
|
+
return nodeRequire("node:fs");
|
|
1755
|
+
} catch {
|
|
1756
|
+
throw new Error("createFilesystemDriver requires a Node-compatible runtime (node:fs is unavailable). Use createMemoryDriver where there is no filesystem.");
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
var TEMP_PREFIX = ".reflectdb-tmp-";
|
|
1760
|
+
function isNotFound(error) {
|
|
1761
|
+
const code = error?.code;
|
|
1762
|
+
return code === "ENOENT" || code === "ENOTDIR";
|
|
1763
|
+
}
|
|
1764
|
+
function contentEtag(bytes) {
|
|
1765
|
+
let h0 = 8997;
|
|
1766
|
+
let h1 = 33826;
|
|
1767
|
+
let h2 = 40164;
|
|
1768
|
+
let h3 = 52210;
|
|
1769
|
+
for (const byte of bytes) {
|
|
1770
|
+
h0 ^= byte;
|
|
1771
|
+
const t0 = h0 * 435;
|
|
1772
|
+
const t1 = h1 * 435 + (t0 >>> 16);
|
|
1773
|
+
const t2 = h2 * 435 + h0 * 256 + (t1 >>> 16);
|
|
1774
|
+
const t3 = h3 * 435 + h1 * 256 + (t2 >>> 16);
|
|
1775
|
+
h0 = t0 & 65535;
|
|
1776
|
+
h1 = t1 & 65535;
|
|
1777
|
+
h2 = t2 & 65535;
|
|
1778
|
+
h3 = t3 & 65535;
|
|
1779
|
+
}
|
|
1780
|
+
const limb = (value) => value.toString(16).padStart(4, "0");
|
|
1781
|
+
return `"${limb(h3)}${limb(h2)}${limb(h1)}${limb(h0)}"`;
|
|
1782
|
+
}
|
|
1783
|
+
function compareKeys2(a, b) {
|
|
1784
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
1785
|
+
}
|
|
1786
|
+
function rejectKey(key, reason) {
|
|
1787
|
+
throw new Error(`Invalid object key ${JSON.stringify(key)}: ${reason}. Keys are store-relative paths; the filesystem driver maps them under its root and refuses anything that could escape it.`);
|
|
1788
|
+
}
|
|
1789
|
+
function checkSegments(key, segments, allowEmpty) {
|
|
1790
|
+
for (const segment of segments) {
|
|
1791
|
+
if (segment === "" && !allowEmpty)
|
|
1792
|
+
rejectKey(key, "it contains an empty path segment");
|
|
1793
|
+
if (segment === "." || segment === "..") {
|
|
1794
|
+
rejectKey(key, `it contains a "${segment}" segment, which escapes or re-roots the path`);
|
|
1795
|
+
}
|
|
1796
|
+
if (segment.includes("\\"))
|
|
1797
|
+
rejectKey(key, "it contains a backslash");
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
function resolveKeyPath(root, key) {
|
|
1801
|
+
if (key === "")
|
|
1802
|
+
rejectKey(key, "it is empty");
|
|
1803
|
+
if (key.startsWith("/"))
|
|
1804
|
+
rejectKey(key, "it is an absolute path");
|
|
1805
|
+
if (/^[a-zA-Z]:/.test(key))
|
|
1806
|
+
rejectKey(key, "it starts with a drive letter");
|
|
1807
|
+
checkSegments(key, key.split("/"), false);
|
|
1808
|
+
return `${root}/${key}`;
|
|
1809
|
+
}
|
|
1810
|
+
function dirnameOf(path) {
|
|
1811
|
+
return path.slice(0, path.lastIndexOf("/"));
|
|
1812
|
+
}
|
|
1813
|
+
function walk(fs, dir, base, out) {
|
|
1814
|
+
let entries;
|
|
1815
|
+
try {
|
|
1816
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
1817
|
+
} catch (error) {
|
|
1818
|
+
if (isNotFound(error))
|
|
1819
|
+
return;
|
|
1820
|
+
throw error;
|
|
1821
|
+
}
|
|
1822
|
+
for (const entry of entries) {
|
|
1823
|
+
if (entry.name.startsWith(TEMP_PREFIX))
|
|
1824
|
+
continue;
|
|
1825
|
+
const path = `${dir}/${entry.name}`;
|
|
1826
|
+
const key = base === "" ? entry.name : `${base}/${entry.name}`;
|
|
1827
|
+
if (entry.isDirectory()) {
|
|
1828
|
+
walk(fs, path, key, out);
|
|
1829
|
+
} else if (entry.isFile()) {
|
|
1830
|
+
out.push({ key, size: fs.statSync(path).size });
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
}
|
|
1834
|
+
function createFilesystemDriver(rootDir) {
|
|
1835
|
+
const fs = loadFs();
|
|
1836
|
+
const root = rootDir.replace(/[/\\]+$/, "");
|
|
1837
|
+
if (root === "") {
|
|
1838
|
+
throw new Error("createFilesystemDriver requires a non-empty rootDir. Refusing to treat the filesystem root as an object store.");
|
|
1839
|
+
}
|
|
1840
|
+
fs.mkdirSync(root, { recursive: true });
|
|
1841
|
+
let tempCounter = 0;
|
|
1842
|
+
function readEtag(path) {
|
|
1843
|
+
try {
|
|
1844
|
+
return contentEtag(fs.readFileSync(path));
|
|
1845
|
+
} catch (error) {
|
|
1846
|
+
if (isNotFound(error))
|
|
1847
|
+
return null;
|
|
1848
|
+
throw error;
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
return {
|
|
1852
|
+
caps: { casWildcard: true },
|
|
1853
|
+
async get(key) {
|
|
1854
|
+
const path = resolveKeyPath(root, key);
|
|
1855
|
+
let body;
|
|
1856
|
+
try {
|
|
1857
|
+
body = fs.readFileSync(path);
|
|
1858
|
+
} catch (error) {
|
|
1859
|
+
if (isNotFound(error) || error?.code === "EISDIR")
|
|
1860
|
+
return null;
|
|
1861
|
+
throw error;
|
|
1862
|
+
}
|
|
1863
|
+
return {
|
|
1864
|
+
body: new Uint8Array(body.buffer, body.byteOffset, body.byteLength),
|
|
1865
|
+
etag: contentEtag(body)
|
|
1866
|
+
};
|
|
1867
|
+
},
|
|
1868
|
+
async put(key, body, opts) {
|
|
1869
|
+
const path = resolveKeyPath(root, key);
|
|
1870
|
+
const wantsIfMatch = opts?.ifMatch !== undefined;
|
|
1871
|
+
const wantsIfNoneMatch = opts?.ifNoneMatch !== undefined;
|
|
1872
|
+
if (wantsIfMatch && wantsIfNoneMatch) {
|
|
1873
|
+
throw new Error(`put("${key}") passed both ifMatch and ifNoneMatch. A conditional write is either create-if-absent or overwrite-if-unchanged, never both.`);
|
|
1874
|
+
}
|
|
1875
|
+
if (wantsIfNoneMatch && fs.existsSync(path)) {
|
|
1876
|
+
throw new PreconditionFailedError(key, `Conditional write failed for "${key}": ifNoneMatch: "*" but the key already exists`);
|
|
1877
|
+
}
|
|
1878
|
+
if (wantsIfMatch) {
|
|
1879
|
+
const current = readEtag(path);
|
|
1880
|
+
if (current === null || current !== opts?.ifMatch) {
|
|
1881
|
+
throw new PreconditionFailedError(key, `Conditional write failed for "${key}": ifMatch ${opts?.ifMatch} but stored etag is ${current ?? "(absent)"}`);
|
|
1882
|
+
}
|
|
1883
|
+
}
|
|
1884
|
+
const dir = dirnameOf(path);
|
|
1885
|
+
try {
|
|
1886
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
1887
|
+
} catch (error) {
|
|
1888
|
+
if (error?.code === "EEXIST") {
|
|
1889
|
+
throw new Error(`Cannot write object "${key}": a prefix of that key already exists as an object. Object stores allow "a" and "a/b" to coexist; a filesystem cannot, so the filesystem driver rejects the collision instead of losing one of them. Use createMemoryDriver or a real bucket if you need a key that is also a prefix.`, { cause: error });
|
|
1890
|
+
}
|
|
1891
|
+
throw error;
|
|
1892
|
+
}
|
|
1893
|
+
tempCounter += 1;
|
|
1894
|
+
const temp = `${dir}/${TEMP_PREFIX}${tempCounter}-${Math.random().toString(36).slice(2)}`;
|
|
1895
|
+
try {
|
|
1896
|
+
fs.writeFileSync(temp, body);
|
|
1897
|
+
fs.renameSync(temp, path);
|
|
1898
|
+
} catch (error) {
|
|
1899
|
+
try {
|
|
1900
|
+
fs.unlinkSync(temp);
|
|
1901
|
+
} catch {}
|
|
1902
|
+
throw error;
|
|
1903
|
+
}
|
|
1904
|
+
return contentEtag(body);
|
|
1905
|
+
},
|
|
1906
|
+
async list(prefix) {
|
|
1907
|
+
if (prefix.startsWith("/"))
|
|
1908
|
+
rejectKey(prefix, "it is an absolute path");
|
|
1909
|
+
if (/^[a-zA-Z]:/.test(prefix))
|
|
1910
|
+
rejectKey(prefix, "it starts with a drive letter");
|
|
1911
|
+
checkSegments(prefix, prefix.split("/"), true);
|
|
1912
|
+
const cut = prefix.lastIndexOf("/");
|
|
1913
|
+
const base = cut === -1 ? "" : prefix.slice(0, cut);
|
|
1914
|
+
const entries = [];
|
|
1915
|
+
walk(fs, base === "" ? root : `${root}/${base}`, base, entries);
|
|
1916
|
+
const matched = entries.filter((entry) => entry.key.startsWith(prefix));
|
|
1917
|
+
matched.sort((a, b) => compareKeys2(a.key, b.key));
|
|
1918
|
+
return matched;
|
|
1919
|
+
},
|
|
1920
|
+
async delete(keys) {
|
|
1921
|
+
for (const key of keys) {
|
|
1922
|
+
const path = resolveKeyPath(root, key);
|
|
1923
|
+
try {
|
|
1924
|
+
fs.unlinkSync(path);
|
|
1925
|
+
} catch (error) {
|
|
1926
|
+
if (isNotFound(error))
|
|
1927
|
+
continue;
|
|
1928
|
+
throw error;
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
}
|
|
1932
|
+
};
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
// src/server/storage/object/index.ts
|
|
1936
|
+
function applyOverrides(target, overrides) {
|
|
1937
|
+
if (!overrides)
|
|
1938
|
+
return;
|
|
1939
|
+
for (const [key, value] of Object.entries(overrides)) {
|
|
1940
|
+
if (value !== undefined)
|
|
1941
|
+
target[key] = value;
|
|
1942
|
+
}
|
|
1943
|
+
}
|
|
1944
|
+
function resolveObjectStorageConfig(config) {
|
|
1945
|
+
const batch = { ...OBJECT_STORAGE_DEFAULTS.batch };
|
|
1946
|
+
applyOverrides(batch, config.batch);
|
|
1947
|
+
const compaction = { ...OBJECT_STORAGE_DEFAULTS.compaction };
|
|
1948
|
+
applyOverrides(compaction, config.compaction);
|
|
1949
|
+
const lease = { ...OBJECT_STORAGE_DEFAULTS.lease };
|
|
1950
|
+
applyOverrides(lease, config.lease);
|
|
1951
|
+
const memory = { ...OBJECT_STORAGE_DEFAULTS.memory };
|
|
1952
|
+
applyOverrides(memory, config.memory);
|
|
1953
|
+
return {
|
|
1954
|
+
roomId: config.roomId,
|
|
1955
|
+
writerId: config.writerId ?? `writer-${crypto.randomUUID()}`,
|
|
1956
|
+
durability: config.durability ?? OBJECT_STORAGE_DEFAULTS.durability,
|
|
1957
|
+
concurrency: config.concurrency ?? OBJECT_STORAGE_DEFAULTS.concurrency,
|
|
1958
|
+
retentionMs: config.retentionMs ?? OBJECT_STORAGE_DEFAULTS.retentionMs,
|
|
1959
|
+
batch,
|
|
1960
|
+
compaction,
|
|
1961
|
+
lease,
|
|
1962
|
+
memory,
|
|
1963
|
+
shutdownFlushMs: config.shutdownFlushMs ?? OBJECT_STORAGE_DEFAULTS.shutdownFlushMs
|
|
1964
|
+
};
|
|
1965
|
+
}
|
|
1966
|
+
function resolveDriver(config) {
|
|
1967
|
+
if (config.driver && config.store) {
|
|
1968
|
+
throw new Error("createObjectStorage: pass either `driver` or `store`, not both. `store` builds " + "the S3 driver for you; `driver` is for a pre-built or custom one.");
|
|
1969
|
+
}
|
|
1970
|
+
if (config.driver)
|
|
1971
|
+
return config.driver;
|
|
1972
|
+
if (config.store)
|
|
1973
|
+
return createS3Driver(config.store);
|
|
1974
|
+
throw new Error("createObjectStorage: one of `driver` or `store` is required — there is no " + "default object store to fall back to.");
|
|
1975
|
+
}
|
|
1976
|
+
function createObjectStorage(config, options = {}) {
|
|
1977
|
+
const resolved = resolveObjectStorageConfig(config);
|
|
1978
|
+
const driver = resolveDriver(config);
|
|
1979
|
+
const clock = options.clock ?? systemClock;
|
|
1980
|
+
const prefix = roomPrefix(resolved.roomId);
|
|
1981
|
+
const state = new RoomState({ memory: resolved.memory, clock, budget: options.budget });
|
|
1982
|
+
const manifest = new ManifestStore({
|
|
1983
|
+
driver,
|
|
1984
|
+
roomId: resolved.roomId,
|
|
1985
|
+
writerId: resolved.writerId,
|
|
1986
|
+
lease: resolved.lease,
|
|
1987
|
+
concurrency: resolved.concurrency,
|
|
1988
|
+
clock
|
|
1989
|
+
});
|
|
1990
|
+
const wal = new WalWriter({
|
|
1991
|
+
driver,
|
|
1992
|
+
manifest,
|
|
1993
|
+
roomId: resolved.roomId,
|
|
1994
|
+
batch: resolved.batch,
|
|
1995
|
+
durability: resolved.durability,
|
|
1996
|
+
concurrency: resolved.concurrency,
|
|
1997
|
+
writerId: resolved.writerId,
|
|
1998
|
+
clock,
|
|
1999
|
+
onCommitted: () => maintain()
|
|
2000
|
+
});
|
|
2001
|
+
if (config.onDurable)
|
|
2002
|
+
wal.onDurable(config.onDurable);
|
|
2003
|
+
if (config.onHealthChange)
|
|
2004
|
+
wal.onHealthChange(config.onHealthChange);
|
|
2005
|
+
let snapshotSeq = 0;
|
|
2006
|
+
let appliedSegments = new Set;
|
|
2007
|
+
let appliedSnapshotKey = null;
|
|
2008
|
+
let closed = false;
|
|
2009
|
+
let closing = null;
|
|
2010
|
+
let bootPromise = null;
|
|
2011
|
+
function isOwnSegment(key) {
|
|
2012
|
+
return resolved.concurrency === "optimistic" && key.startsWith(`${prefix}wal/${resolved.writerId}-`);
|
|
2013
|
+
}
|
|
2014
|
+
async function applySegment(key) {
|
|
2015
|
+
const object = await driver.get(key);
|
|
2016
|
+
if (!object) {
|
|
2017
|
+
throw new Error(`Object storage: WAL segment "${key}" named by the manifest for room ` + `"${resolved.roomId}" is missing. Refusing to serve a truncated log.`);
|
|
2018
|
+
}
|
|
2019
|
+
state.applyRecords(decodeSegment(object.body, key));
|
|
2020
|
+
}
|
|
2021
|
+
async function boot() {
|
|
2022
|
+
await manifest.init();
|
|
2023
|
+
const record = await manifest.load();
|
|
2024
|
+
state.loadMeta(record.meta);
|
|
2025
|
+
if (record.snapshotKey) {
|
|
2026
|
+
const object = await driver.get(record.snapshotKey);
|
|
2027
|
+
if (!object) {
|
|
2028
|
+
throw new IncompleteStateError(resolved.roomId, record.snapshotKey, "Refusing to boot with incomplete state.");
|
|
2029
|
+
}
|
|
2030
|
+
state.loadSnapshot(decodeJson(object.body, record.snapshotKey));
|
|
2031
|
+
appliedSnapshotKey = record.snapshotKey;
|
|
2032
|
+
}
|
|
2033
|
+
for (const segment of record.walSegs)
|
|
2034
|
+
await applySegment(segment.key);
|
|
2035
|
+
appliedSegments = new Set(record.walSegs.map((segment) => segment.key));
|
|
2036
|
+
await manifest.start();
|
|
2037
|
+
}
|
|
2038
|
+
function ready() {
|
|
2039
|
+
if (!bootPromise) {
|
|
2040
|
+
bootPromise = boot().catch((error) => {
|
|
2041
|
+
bootPromise = null;
|
|
2042
|
+
throw error;
|
|
2043
|
+
});
|
|
2044
|
+
}
|
|
2045
|
+
return bootPromise;
|
|
2046
|
+
}
|
|
2047
|
+
function assertOpen() {
|
|
2048
|
+
if (closed) {
|
|
2049
|
+
throw new Error(`Object storage: room "${resolved.roomId}" is closed and no longer accepts writes.`);
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
function submit(record, mutate) {
|
|
2053
|
+
const prepared = wal.prepare(record);
|
|
2054
|
+
mutate();
|
|
2055
|
+
return wal.append(prepared);
|
|
2056
|
+
}
|
|
2057
|
+
function persistReservations(opIds) {
|
|
2058
|
+
try {
|
|
2059
|
+
wal.append(wal.prepare({ k: "reserve", opIds, at: clock.now() })).catch(() => {
|
|
2060
|
+
return;
|
|
2061
|
+
});
|
|
2062
|
+
} catch {}
|
|
2063
|
+
}
|
|
2064
|
+
async function maintain() {
|
|
2065
|
+
await compactIfDue();
|
|
2066
|
+
await collectGarbage();
|
|
2067
|
+
}
|
|
2068
|
+
async function compactIfDue() {
|
|
2069
|
+
const current = manifest.manifest;
|
|
2070
|
+
let bytes = 0;
|
|
2071
|
+
for (const segment of current.walSegs)
|
|
2072
|
+
bytes += segment.bytes;
|
|
2073
|
+
if (current.walSegs.length < resolved.compaction.afterSegments && bytes < resolved.compaction.afterBytes) {
|
|
2074
|
+
return;
|
|
2075
|
+
}
|
|
2076
|
+
const snapshot = state.toSnapshot();
|
|
2077
|
+
const token = resolved.concurrency === "optimistic" ? resolved.writerId : String(manifest.epoch);
|
|
2078
|
+
const key = `${prefix}snap/${token}-${snapshotSeq++}.json`;
|
|
2079
|
+
await driver.put(key, encodeJson(snapshot));
|
|
2080
|
+
const deletableAt = clock.now() + resolved.compaction.gcGraceMs;
|
|
2081
|
+
const committed = await manifest.commit((next) => {
|
|
2082
|
+
for (const segment of next.walSegs)
|
|
2083
|
+
next.pendingGc.push({ key: segment.key, deletableAt });
|
|
2084
|
+
if (next.snapshotKey)
|
|
2085
|
+
next.pendingGc.push({ key: next.snapshotKey, deletableAt });
|
|
2086
|
+
next.walSegs = [];
|
|
2087
|
+
next.snapshotKey = key;
|
|
2088
|
+
next.snapshotHlc = snapshot.hlc;
|
|
2089
|
+
const cutoff = snapshot.hlc;
|
|
2090
|
+
if (cutoff && (!next.meta.compactionCutoff || cutoff > next.meta.compactionCutoff)) {
|
|
2091
|
+
next.meta.compactionCutoff = cutoff;
|
|
2092
|
+
}
|
|
2093
|
+
});
|
|
2094
|
+
const committedCutoff = committed.meta.compactionCutoff;
|
|
2095
|
+
if (committedCutoff)
|
|
2096
|
+
state.setMeta("compactionCutoff", committedCutoff);
|
|
2097
|
+
appliedSegments = new Set;
|
|
2098
|
+
appliedSnapshotKey = key;
|
|
2099
|
+
}
|
|
2100
|
+
async function collectGarbage() {
|
|
2101
|
+
const now = clock.now();
|
|
2102
|
+
const due = manifest.manifest.pendingGc.filter((entry) => entry.deletableAt <= now);
|
|
2103
|
+
if (due.length === 0)
|
|
2104
|
+
return;
|
|
2105
|
+
const live = new Set;
|
|
2106
|
+
if (manifest.manifest.snapshotKey)
|
|
2107
|
+
live.add(manifest.manifest.snapshotKey);
|
|
2108
|
+
for (const segment of manifest.manifest.walSegs)
|
|
2109
|
+
live.add(segment.key);
|
|
2110
|
+
const deletable = due.filter((entry) => !live.has(entry.key));
|
|
2111
|
+
if (deletable.length > 0)
|
|
2112
|
+
await driver.delete(deletable.map((entry) => entry.key));
|
|
2113
|
+
await manifest.commit((next) => {
|
|
2114
|
+
next.pendingGc = next.pendingGc.filter((entry) => entry.deletableAt > now);
|
|
2115
|
+
});
|
|
2116
|
+
}
|
|
2117
|
+
const adapter = {
|
|
2118
|
+
async init() {
|
|
2119
|
+
await ready();
|
|
2120
|
+
},
|
|
2121
|
+
async getRow(table, rowId) {
|
|
2122
|
+
await ready();
|
|
2123
|
+
return state.getRow(table, rowId);
|
|
2124
|
+
},
|
|
2125
|
+
async getRowsByIds(table, rowIds) {
|
|
2126
|
+
await ready();
|
|
2127
|
+
if (rowIds.length === 0)
|
|
2128
|
+
return {};
|
|
2129
|
+
return state.getRowsByIds(table, rowIds);
|
|
2130
|
+
},
|
|
2131
|
+
async putRow(table, rowId, row, colClocks, hlc) {
|
|
2132
|
+
await ready();
|
|
2133
|
+
assertOpen();
|
|
2134
|
+
const mutation = { table, rowId, row, colClocks, hlc };
|
|
2135
|
+
await submit({ k: "row", row: mutation }, () => state.putRow(table, rowId, row, colClocks, hlc));
|
|
2136
|
+
},
|
|
2137
|
+
async getRows(table, filter) {
|
|
2138
|
+
await ready();
|
|
2139
|
+
return state.getRows(table, filter);
|
|
2140
|
+
},
|
|
2141
|
+
async appendOp(entry) {
|
|
2142
|
+
await ready();
|
|
2143
|
+
assertOpen();
|
|
2144
|
+
await submit({ k: "op", op: entry }, () => state.appendOp(entry));
|
|
2145
|
+
},
|
|
2146
|
+
async applyOp(table, rowId, row, colClocks, hlc, opType, payload) {
|
|
2147
|
+
await ready();
|
|
2148
|
+
assertOpen();
|
|
2149
|
+
const mutation = { table, rowId, row, colClocks, hlc };
|
|
2150
|
+
const op = { table, op: opType, rowId, payload, hlc, colClocks };
|
|
2151
|
+
state.assertCanAdmitApply(mutation, op);
|
|
2152
|
+
await submit({ k: "apply", row: mutation, op }, () => {
|
|
2153
|
+
state.putRow(table, rowId, row, colClocks, hlc);
|
|
2154
|
+
state.appendOp(op);
|
|
2155
|
+
});
|
|
2156
|
+
},
|
|
2157
|
+
async getOpsSince(since, tables) {
|
|
2158
|
+
await ready();
|
|
2159
|
+
return state.getOpsSince(since, tables);
|
|
2160
|
+
},
|
|
2161
|
+
async getChangedTablesSince(since, tables) {
|
|
2162
|
+
await ready();
|
|
2163
|
+
return state.getChangedTablesSince(since, tables);
|
|
2164
|
+
},
|
|
2165
|
+
async getOplogHead(tables) {
|
|
2166
|
+
await ready();
|
|
2167
|
+
return state.getOplogHead(tables);
|
|
2168
|
+
},
|
|
2169
|
+
async deleteOpsBefore(hlc) {
|
|
2170
|
+
await ready();
|
|
2171
|
+
return state.deleteOpsBefore(hlc);
|
|
2172
|
+
},
|
|
2173
|
+
async reserveOp(opId) {
|
|
2174
|
+
await ready();
|
|
2175
|
+
const fresh = state.reserveOp(opId);
|
|
2176
|
+
if (fresh)
|
|
2177
|
+
persistReservations([opId]);
|
|
2178
|
+
return fresh;
|
|
2179
|
+
},
|
|
2180
|
+
async reserveOps(opIds) {
|
|
2181
|
+
await ready();
|
|
2182
|
+
const fresh = state.reserveOps(opIds);
|
|
2183
|
+
if (fresh.length > 0)
|
|
2184
|
+
persistReservations(fresh);
|
|
2185
|
+
return fresh;
|
|
2186
|
+
},
|
|
2187
|
+
async getMeta(key) {
|
|
2188
|
+
await ready();
|
|
2189
|
+
return state.getMeta(key);
|
|
2190
|
+
},
|
|
2191
|
+
async refresh() {
|
|
2192
|
+
if (resolved.concurrency !== "optimistic")
|
|
2193
|
+
return false;
|
|
2194
|
+
await ready();
|
|
2195
|
+
const next = await manifest.refresh();
|
|
2196
|
+
if (next.snapshotKey && next.snapshotKey !== appliedSnapshotKey) {
|
|
2197
|
+
const object = await driver.get(next.snapshotKey);
|
|
2198
|
+
if (!object) {
|
|
2199
|
+
throw new IncompleteStateError(resolved.roomId, next.snapshotKey, "Refusing to serve incomplete state.");
|
|
2200
|
+
}
|
|
2201
|
+
state.loadSnapshot(decodeJson(object.body, next.snapshotKey));
|
|
2202
|
+
state.loadMeta(next.meta);
|
|
2203
|
+
appliedSnapshotKey = next.snapshotKey;
|
|
2204
|
+
appliedSegments = new Set;
|
|
2205
|
+
for (const segment of next.walSegs) {
|
|
2206
|
+
await applySegment(segment.key);
|
|
2207
|
+
appliedSegments.add(segment.key);
|
|
2208
|
+
}
|
|
2209
|
+
return true;
|
|
2210
|
+
}
|
|
2211
|
+
let applied = false;
|
|
2212
|
+
for (const segment of next.walSegs) {
|
|
2213
|
+
if (appliedSegments.has(segment.key) || isOwnSegment(segment.key))
|
|
2214
|
+
continue;
|
|
2215
|
+
await applySegment(segment.key);
|
|
2216
|
+
appliedSegments.add(segment.key);
|
|
2217
|
+
applied = true;
|
|
2218
|
+
}
|
|
2219
|
+
if (applied)
|
|
2220
|
+
state.loadMeta(next.meta);
|
|
2221
|
+
return applied;
|
|
2222
|
+
},
|
|
2223
|
+
async setMeta(key, value) {
|
|
2224
|
+
await ready();
|
|
2225
|
+
assertOpen();
|
|
2226
|
+
state.setMeta(key, value);
|
|
2227
|
+
await manifest.commit((next) => {
|
|
2228
|
+
next.meta[key] = value;
|
|
2229
|
+
});
|
|
2230
|
+
},
|
|
2231
|
+
get health() {
|
|
2232
|
+
return wal.health;
|
|
2233
|
+
},
|
|
2234
|
+
get durableHlc() {
|
|
2235
|
+
return wal.durableHlc;
|
|
2236
|
+
},
|
|
2237
|
+
onDurable(callback) {
|
|
2238
|
+
wal.onDurable(callback);
|
|
2239
|
+
},
|
|
2240
|
+
onHealthChange(callback) {
|
|
2241
|
+
wal.onHealthChange(callback);
|
|
2242
|
+
},
|
|
2243
|
+
async flush() {
|
|
2244
|
+
await ready();
|
|
2245
|
+
await wal.flush();
|
|
2246
|
+
},
|
|
2247
|
+
close() {
|
|
2248
|
+
if (closing)
|
|
2249
|
+
return closing;
|
|
2250
|
+
closed = true;
|
|
2251
|
+
closing = (async () => {
|
|
2252
|
+
const outcome = await Promise.race([
|
|
2253
|
+
wal.flush().then(() => null, (error) => ({ error })),
|
|
2254
|
+
clock.delay(resolved.shutdownFlushMs).then(() => "timeout")
|
|
2255
|
+
]);
|
|
2256
|
+
if (outcome === "timeout") {
|
|
2257
|
+
console.warn(`[reflectdb] object storage: room "${resolved.roomId}" did not finish ` + `flushing within shutdownFlushMs (${resolved.shutdownFlushMs}ms); ` + `buffered writes were not made durable.`);
|
|
2258
|
+
} else if (outcome) {
|
|
2259
|
+
console.warn(`[reflectdb] object storage: the final drain for room "${resolved.roomId}" ` + `failed; buffered writes were not made durable:`, outcome.error);
|
|
2260
|
+
}
|
|
2261
|
+
await wal.stop(resolved.shutdownFlushMs);
|
|
2262
|
+
manifest.stop();
|
|
2263
|
+
try {
|
|
2264
|
+
await Promise.race([
|
|
2265
|
+
manifest.release(),
|
|
2266
|
+
clock.delay(resolved.shutdownFlushMs).then(() => {
|
|
2267
|
+
console.warn(`[reflectdb] object storage: releasing the lease for room ` + `"${resolved.roomId}" timed out; the next writer will wait out ` + `lease.ttlMs (${resolved.lease.ttlMs}ms) instead.`);
|
|
2268
|
+
})
|
|
2269
|
+
]);
|
|
2270
|
+
} catch (error) {
|
|
2271
|
+
console.warn("[reflectdb] object storage: releasing the lease failed:", error);
|
|
2272
|
+
}
|
|
2273
|
+
state.dispose();
|
|
2274
|
+
try {
|
|
2275
|
+
await driver.close?.();
|
|
2276
|
+
} catch (error) {
|
|
2277
|
+
console.warn("[reflectdb] object storage: closing the driver failed:", error);
|
|
2278
|
+
}
|
|
2279
|
+
})();
|
|
2280
|
+
return closing;
|
|
2281
|
+
}
|
|
2282
|
+
};
|
|
2283
|
+
return adapter;
|
|
2284
|
+
}
|