sliftutils 1.7.32 → 1.7.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/builders/extensionBuild.ts +1 -1
- package/bundler/bundleEntry.ts +1 -3
- package/bundler/bundler.ts +2 -4
- package/bundler/sourceMaps.ts +1 -2
- package/examplestorage/browser.tsx +1 -2
- package/examplestorage/exampleserver.ts +1 -2
- package/index.d.ts +105 -100
- package/misc/dist/environment.ts.cache +2 -2
- package/misc/dist/getSecret.ts.cache +2 -2
- package/misc/dist/strings.ts.cache +2 -2
- package/misc/dist/zip.ts.cache +2 -2
- package/misc/https/dist/certs.ts.cache +2 -2
- package/misc/https/dist/dns.ts.cache +7 -16
- package/misc/https/dist/persistentLocalStorage.ts.cache +2 -2
- package/misc/https/dns.d.ts +1 -4
- package/misc/https/dns.ts +5 -14
- package/misc/https/hostServer.d.ts +2 -0
- package/misc/https/hostServer.ts +6 -7
- package/misc/https/httpsCerts.ts +4 -10
- package/misc/openrouter.ts +1 -2
- package/package.json +1 -1
- package/render-utils/FullscreenModal.tsx +1 -4
- package/render-utils/GenericFormat.tsx +2 -8
- package/render-utils/Input.tsx +5 -13
- package/render-utils/InputLabel.tsx +4 -6
- package/render-utils/InputPicker.tsx +1 -2
- package/render-utils/URLParam.ts +2 -3
- package/render-utils/dist/observer.tsx.cache +2 -2
- package/render-utils/mobxTyped.ts +2 -3
- package/render-utils/niceStringify.ts +2 -4
- package/storage/ArchivesDisk.ts +4 -14
- package/storage/BulkDatabase2/BulkDatabase2.d.ts +1 -2
- package/storage/BulkDatabase2/BulkDatabase2.ts +4 -11
- package/storage/BulkDatabase2/BulkDatabaseBase.ts +42 -114
- package/storage/BulkDatabase2/BulkDatabaseFormat.ts +31 -91
- package/storage/BulkDatabase2/BulkDatabaseMerge.ts +19 -57
- package/storage/BulkDatabase2/BulkDatabaseReader.ts +4 -12
- package/storage/BulkDatabase2/LoadedIndex.ts +6 -18
- package/storage/BulkDatabase2/WriteOverlay.ts +3 -8
- package/storage/BulkDatabase2/blockCache.ts +12 -28
- package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +40 -103
- package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +19 -50
- package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +18 -50
- package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +7 -15
- package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +9 -21
- package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +6 -11
- package/storage/BulkDatabase2/dist/blockCache.ts.cache +15 -31
- package/storage/BulkDatabase2/dist/mergeLock.ts.cache +10 -29
- package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +10 -18
- package/storage/BulkDatabase2/dist/streamLog.ts.cache +17 -43
- package/storage/BulkDatabase2/dist/syncClient.ts.cache +11 -27
- package/storage/BulkDatabase2/mergeLock.ts +7 -26
- package/storage/BulkDatabase2/mergeMarkers.ts +7 -15
- package/storage/BulkDatabase2/streamLog.ts +14 -40
- package/storage/BulkDatabase2/syncClient.ts +8 -24
- package/storage/FileFolderAPI.tsx +28 -75
- package/storage/IArchives.d.ts +3 -3
- package/storage/IArchives.ts +21 -50
- package/storage/IStorage.ts +2 -4
- package/storage/PendingManager.tsx +2 -3
- package/storage/TransactionStorage.ts +11 -24
- package/storage/backblaze.ts +12 -43
- package/storage/dist/ArchivesDisk.ts.cache +7 -17
- package/storage/dist/FileFolderAPI.tsx.cache +25 -63
- package/storage/dist/IArchives.ts.cache +10 -20
- package/storage/dist/IndexedDBFileFolderAPI.ts.cache +2 -2
- package/storage/dist/JSONStorage.ts.cache +2 -2
- package/storage/dist/PendingManager.tsx.cache +5 -6
- package/storage/dist/TransactionStorage.ts.cache +14 -27
- package/storage/dist/backblaze.ts.cache +15 -46
- package/storage/dist/embeddingFormats.ts.cache +11 -24
- package/storage/dist/fileSystemPointer.ts.cache +4 -7
- package/storage/dist/remoteFileStorage.ts.cache +17 -36
- package/storage/embeddingFormats.ts +11 -27
- package/storage/fileSystemPointer.ts +2 -7
- package/storage/proxydatabase/dist/ivfEmbeddingDatabase.ts.cache +8 -18
- package/storage/proxydatabase/dist/transactionSet.ts.cache +4 -7
- package/storage/proxydatabase/inMemoryDatabase.ts +1 -3
- package/storage/proxydatabase/ivfEmbeddingDatabase.ts +8 -23
- package/storage/proxydatabase/transactionSet.ts +2 -5
- package/storage/remoteFileServer.ts +20 -48
- package/storage/remoteFileStorage.ts +14 -33
- package/storage/remoteStorage/ArchivesRemote.d.ts +1 -4
- package/storage/remoteStorage/ArchivesRemote.ts +11 -26
- package/storage/remoteStorage/ArchivesUrl.ts +2 -4
- package/storage/remoteStorage/accessPage.tsx +1 -5
- package/storage/remoteStorage/blobStore.d.ts +5 -13
- package/storage/remoteStorage/blobStore.ts +52 -130
- package/storage/remoteStorage/createArchives.d.ts +9 -13
- package/storage/remoteStorage/createArchives.ts +43 -187
- package/storage/remoteStorage/deployTakeover.d.ts +18 -32
- package/storage/remoteStorage/deployTakeover.ts +70 -384
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +12 -24
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +5 -7
- package/storage/remoteStorage/dist/blobStore.ts.cache +47 -117
- package/storage/remoteStorage/dist/cliArgs.ts.cache +2 -2
- package/storage/remoteStorage/dist/createArchives.ts.cache +48 -189
- package/storage/remoteStorage/dist/deployTakeover.ts.cache +81 -387
- package/storage/remoteStorage/dist/intermediateSources.ts.cache +119 -0
- package/storage/remoteStorage/dist/remoteConfig.ts.cache +36 -24
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +20 -40
- package/storage/remoteStorage/dist/storageClientController.ts.cache +4 -6
- package/storage/remoteStorage/dist/storageController.ts.cache +11 -73
- package/storage/remoteStorage/dist/storageServerState.ts.cache +265 -178
- package/storage/remoteStorage/grantAccessCli.ts +1 -3
- package/storage/remoteStorage/intermediateSources.d.ts +21 -0
- package/storage/remoteStorage/intermediateSources.ts +117 -0
- package/storage/remoteStorage/remoteConfig.ts +33 -20
- package/storage/remoteStorage/sourceWrapper.d.ts +8 -19
- package/storage/remoteStorage/sourceWrapper.ts +19 -41
- package/storage/remoteStorage/storageClientController.ts +1 -3
- package/storage/remoteStorage/storageController.d.ts +4 -5
- package/storage/remoteStorage/storageController.ts +10 -81
- package/storage/remoteStorage/storageServer.ts +20 -34
- package/storage/remoteStorage/storageServerState.d.ts +28 -5
- package/storage/remoteStorage/storageServerState.ts +271 -207
|
@@ -1,19 +1,12 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// Browser-side cross-tab write sync for BulkDatabase2, over BroadcastChannel (one channel per
|
|
3
|
-
// collection, same-origin tabs and workers). It does NOT persist anything — each side writes to disk
|
|
4
|
-
// itself; this just relays live writes to other open tabs/workers and, when one starts up, asks peers
|
|
5
|
-
// for writes they've made recently that may not be on disk yet. No-op in Node / where
|
|
6
|
-
// BroadcastChannel is unavailable, so BulkDatabase2 can call these unconditionally. This is an
|
|
7
|
-
// optional feature, so there is no fallback.
|
|
1
|
+
"use strict"; var console = (function shimConsole() { let baseConsole = globalThis.console; if (typeof window !== "undefined") return baseConsole; let name = __filename.replaceAll("\\", "/").split("/").pop(); let dirName = __dirname.replaceAll("\\", "/").split("/").pop(); let context = { __FILE__: __filename, __DIR__: __dirname, __NAME__: name, __DIRNAME__: dirName }; let newConsole = { ...baseConsole }; for (let shim of ["log", "info", "warn", "error"]) { newConsole[shim] = (...args) => { args.push(context); baseConsole[shim](...args); }; } newConsole.__context = context; return newConsole; })(); "use strict";
|
|
2
|
+
// Browser-side cross-tab write sync for BulkDatabase2, over BroadcastChannel (one channel per collection, same-origin tabs and workers). It does NOT persist anything — each side writes to disk itself; this just relays live writes to other open tabs/workers and, when one starts up, asks peers for writes they've made recently that may not be on disk yet. No-op in Node / where BroadcastChannel is unavailable, so BulkDatabase2 can call these unconditionally. This is an optional feature, so there is no fallback.
|
|
8
3
|
Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
|
|
9
4
|
//exports.broadcastSeal = exports.broadcast = exports.connect = exports.isSyncSupported = void 0;
|
|
10
|
-
// Writes older than this are assumed already flushed to disk (a freshly-opened tab gets those by
|
|
11
|
-
// reading disk), so a tab only replays writes newer than this when a peer says hello.
|
|
5
|
+
// Writes older than this are assumed already flushed to disk (a freshly-opened tab gets those by reading disk), so a tab only replays writes newer than this when a peer says hello.
|
|
12
6
|
const RECENT_WINDOW_MS = 60000;
|
|
13
7
|
const channels = new Map();
|
|
14
8
|
function isSyncSupported() {
|
|
15
|
-
// window (main-thread browser) or WorkerGlobalScope (Web/Service/Shared worker) — but not Node
|
|
16
|
-
// (typesafecss's isNode is `typeof window === "undefined"`, which misclassifies workers)
|
|
9
|
+
// window (main-thread browser) or WorkerGlobalScope (Web/Service/Shared worker) — but not Node (typesafecss's isNode is `typeof window === "undefined"`, which misclassifies workers)
|
|
17
10
|
let isBrowserOrWorker = typeof window !== "undefined"
|
|
18
11
|
|| "WorkerGlobalScope" in globalThis;
|
|
19
12
|
return isBrowserOrWorker && typeof BroadcastChannel !== "undefined";
|
|
@@ -33,8 +26,7 @@ function ensure(collection) {
|
|
|
33
26
|
let channel = channels.get(collection);
|
|
34
27
|
if (channel)
|
|
35
28
|
return channel;
|
|
36
|
-
// BroadcastChannel never delivers a message back to the instance that sent it, so a tab never
|
|
37
|
-
// hears its own writes — only the other open tabs do.
|
|
29
|
+
// BroadcastChannel never delivers a message back to the instance that sent it, so a tab never hears its own writes — only the other open tabs do.
|
|
38
30
|
const bc = new BroadcastChannel(`bulkDatabase2:${collection}`);
|
|
39
31
|
const created = { bc, subscribers: [], sealSubscribers: [], recent: [] };
|
|
40
32
|
bc.onmessage = (event) => {
|
|
@@ -49,15 +41,12 @@ function ensure(collection) {
|
|
|
49
41
|
deliver(created, write);
|
|
50
42
|
}
|
|
51
43
|
else if (msg.type === "seal") {
|
|
52
|
-
// A peer is about to fold recent data; stop appending to our current stream file so it
|
|
53
|
-
// becomes complete and the merge can fold it whole. Our next write starts a fresh file.
|
|
44
|
+
// A peer is about to fold recent data; stop appending to our current stream file so it becomes complete and the merge can fold it whole. Our next write starts a fresh file.
|
|
54
45
|
for (const sub of created.sealSubscribers)
|
|
55
46
|
sub();
|
|
56
47
|
}
|
|
57
48
|
else if (msg.type === "hello") {
|
|
58
|
-
// Another tab just started; replay our recent writes so it doesn't miss any. The reply goes
|
|
59
|
-
// to every tab, but peers that already have a write ignore it (its timestamp isn't newer
|
|
60
|
-
// than what they hold), so the redundant broadcast is harmless.
|
|
49
|
+
// Another tab just started; replay our recent writes so it doesn't miss any. The reply goes to every tab, but peers that already have a write ignore it (its timestamp isn't newer than what they hold), so the redundant broadcast is harmless.
|
|
61
50
|
pruneRecent(created);
|
|
62
51
|
if (created.recent.length)
|
|
63
52
|
created.bc.postMessage({ type: "recent", writes: created.recent });
|
|
@@ -66,9 +55,7 @@ function ensure(collection) {
|
|
|
66
55
|
channels.set(collection, created);
|
|
67
56
|
return created;
|
|
68
57
|
}
|
|
69
|
-
// Subscribe to remote writes for a collection. Recent writes from already-open tabs arrive through the
|
|
70
|
-
// same onWrite callback (as the reply to our hello), so the returned array is always empty — it's kept
|
|
71
|
-
// only for API compatibility with callers that await it.
|
|
58
|
+
// Subscribe to remote writes for a collection. Recent writes from already-open tabs arrive through the same onWrite callback (as the reply to our hello), so the returned array is always empty — it's kept only for API compatibility with callers that await it.
|
|
72
59
|
function connect(collection, onWrite, onSeal) {
|
|
73
60
|
const channel = ensure(collection);
|
|
74
61
|
if (!channel)
|
|
@@ -89,10 +76,7 @@ function broadcast(collection, write) {
|
|
|
89
76
|
channel.bc.postMessage({ type: "write", write });
|
|
90
77
|
}
|
|
91
78
|
exports.broadcast = broadcast;
|
|
92
|
-
// Ask every other open tab of this collection to seal (stop appending to) its current stream file, so
|
|
93
|
-
// a merge can fold recent data up to the present without racing an append. Best-effort: no-op in Node,
|
|
94
|
-
// and a peer that misses it just keeps appending — worst case the merge folds a prefix and the rest
|
|
95
|
-
// folds later (duplication, resolved by write-time), never data loss.
|
|
79
|
+
// Ask every other open tab of this collection to seal (stop appending to) its current stream file, so a merge can fold recent data up to the present without racing an append. Best-effort: no-op in Node, and a peer that misses it just keeps appending — worst case the merge folds a prefix and the rest folds later (duplication, resolved by write-time), never data loss.
|
|
96
80
|
function broadcastSeal(collection) {
|
|
97
81
|
const channel = ensure(collection);
|
|
98
82
|
if (!channel)
|
|
@@ -100,5 +84,5 @@ function broadcastSeal(collection) {
|
|
|
100
84
|
channel.bc.postMessage({ type: "seal" });
|
|
101
85
|
}
|
|
102
86
|
exports.broadcastSeal = broadcastSeal;
|
|
103
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3luY0NsaWVudC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInN5bmNDbGllbnQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBLDhGQUE4RjtBQUM5RixxR0FBcUc7QUFDckcsc0dBQXNHO0FBQ3RHLHNGQUFzRjtBQUN0RiwrRkFBK0Y7QUFDL0YsNkNBQTZDOzs7QUFJN0MsaUdBQWlHO0FBQ2pHLHNGQUFzRjtBQUN0RixNQUFNLGdCQUFnQixHQUFHLEtBQU0sQ0FBQztBQVdoQyxNQUFNLFFBQVEsR0FBRyxJQUFJLEdBQUcsRUFBbUIsQ0FBQztBQUU1QyxTQUFnQixlQUFlO0lBQzNCLCtGQUErRjtJQUMvRix5RkFBeUY7SUFDekYsSUFBSSxpQkFBaUIsR0FBRyxPQUFPLE1BQU0sS0FBSyxXQUFXO1dBQzlDLG1CQUFtQixJQUFJLFVBQVUsQ0FBQztJQUN6QyxPQUFPLGlCQUFpQixJQUFJLE9BQU8sZ0JBQWdCLEtBQUssV0FBVyxDQUFDO0FBQ3hFLENBQUM7QUFORCwwQ0FNQztBQUVELFNBQVMsV0FBVyxDQUFDLE9BQWdCO0lBQ2pDLE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxHQUFHLEVBQUUsR0FBRyxnQkFBZ0IsQ0FBQztJQUM3QyxPQUFPLENBQUMsTUFBTSxHQUFHLE9BQU8sQ0FBQyxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLElBQUksSUFBSSxNQUFNLENBQUMsQ0FBQztBQUNsRSxDQUFDO0FBRUQsU0FBUyxPQUFPLENBQUMsT0FBZ0IsRUFBRSxLQUFrQjtJQUNqRCxLQUFLLE1BQU0sR0FBRyxJQUFJLE9BQU8sQ0FBQyxXQUFXO1FBQUUsR0FBRyxDQUFDLEtBQUssQ0FBQyxDQUFDO0FBQ3RELENBQUM7QUFFRCxTQUFTLE1BQU0sQ0FBQyxVQUFrQjtJQUM5QixJQUFJLENBQUMsZUFBZSxFQUFFO1FBQUUsT0FBTyxTQUFTLENBQUM7SUFDekMsSUFBSSxPQUFPLEdBQUcsUUFBUSxDQUFDLEdBQUcsQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUN2QyxJQUFJLE9BQU87UUFBRSxPQUFPLE9BQU8sQ0FBQztJQUM1Qiw4RkFBOEY7SUFDOUYsc0RBQXNEO0lBQ3RELE1BQU0sRUFBRSxHQUFHLElBQUksZ0JBQWdCLENBQUMsaUJBQWlCLFVBQVUsRUFBRSxDQUFDLENBQUM7SUFDL0QsTUFBTSxPQUFPLEdBQVksRUFBRSxFQUFFLEVBQUUsV0FBVyxFQUFFLEVBQUUsRUFBRSxlQUFlLEVBQUUsRUFBRSxFQUFFLE1BQU0sRUFBRSxFQUFFLEVBQUUsQ0FBQztJQUNsRixFQUFFLENBQUMsU0FBUyxHQUFHLENBQUMsS0FBbUIsRUFBRSxFQUFFO1FBQ25DLE1BQU0sR0FBRyxHQUFHLEtBQUssQ0FBQyxJQUFxRSxDQUFDO1FBQ3hGLElBQUksQ0FBQyxHQUFHO1lBQUUsT0FBTztRQUNqQixJQUFJLEdBQUcsQ0FBQyxJQUFJLEtBQUssT0FBTyxJQUFJLEdBQUcsQ0FBQyxLQUFLLEVBQUUsQ0FBQztZQUNwQyxPQUFPLENBQUMsT0FBTyxFQUFFLEdBQUcsQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUNoQyxDQUFDO2FBQU0sSUFBSSxHQUFHLENBQUMsSUFBSSxLQUFLLFFBQVEsSUFBSSxHQUFHLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDN0MsS0FBSyxNQUFNLEtBQUssSUFBSSxHQUFHLENBQUMsTUFBTTtnQkFBRSxPQUFPLENBQUMsT0FBTyxFQUFFLEtBQUssQ0FBQyxDQUFDO1FBQzVELENBQUM7YUFBTSxJQUFJLEdBQUcsQ0FBQyxJQUFJLEtBQUssTUFBTSxFQUFFLENBQUM7WUFDN0IsdUZBQXVGO1lBQ3ZGLHdGQUF3RjtZQUN4RixLQUFLLE1BQU0sR0FBRyxJQUFJLE9BQU8sQ0FBQyxlQUFlO2dCQUFFLEdBQUcsRUFBRSxDQUFDO1FBQ3JELENBQUM7YUFBTSxJQUFJLEdBQUcsQ0FBQyxJQUFJLEtBQUssT0FBTyxFQUFFLENBQUM7WUFDOUIsNEZBQTRGO1lBQzVGLHlGQUF5RjtZQUN6RixnRUFBZ0U7WUFDaEUsV0FBVyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1lBQ3JCLElBQUksT0FBTyxDQUFDLE1BQU0sQ0FBQyxNQUFNO2dCQUFFLE9BQU8sQ0FBQyxFQUFFLENBQUMsV0FBVyxDQUFDLEVBQUUsSUFBSSxFQUFFLFFBQVEsRUFBRSxNQUFNLEVBQUUsT0FBTyxDQUFDLE1BQU0sRUFBRSxDQUFDLENBQUM7UUFDbEcsQ0FBQztJQUNMLENBQUMsQ0FBQztJQUNGLFFBQVEsQ0FBQyxHQUFHLENBQUMsVUFBVSxFQUFFLE9BQU8sQ0FBQyxDQUFDO0lBQ2xDLE9BQU8sT0FBTyxDQUFDO0FBQ25CLENBQUM7QUFFRCx1R0FBdUc7QUFDdkcsdUdBQXVHO0FBQ3ZHLHlEQUF5RDtBQUN6RCxTQUFnQixPQUFPLENBQUMsVUFBa0IsRUFBRSxPQUFxQyxFQUFFLE1BQW1CO0lBQ2xHLE1BQU0sT0FBTyxHQUFHLE1BQU0sQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUNuQyxJQUFJLENBQUMsT0FBTztRQUFFLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUN6QyxPQUFPLENBQUMsV0FBVyxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQztJQUNsQyxJQUFJLE1BQU07UUFBRSxPQUFPLENBQUMsZUFBZSxDQUFDLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUNqRCxPQUFPLENBQUMsRUFBRSxDQUFDLFdBQVcsQ0FBQyxFQUFFLElBQUksRUFBRSxPQUFPLEVBQUUsQ0FBQyxDQUFDO0lBQzFDLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyxFQUFFLENBQUMsQ0FBQztBQUMvQixDQUFDO0FBUEQsMEJBT0M7QUFFRCxTQUFnQixTQUFTLENBQUMsVUFBa0IsRUFBRSxLQUFrQjtJQUM1RCxNQUFNLE9BQU8sR0FBRyxNQUFNLENBQUMsVUFBVSxDQUFDLENBQUM7SUFDbkMsSUFBSSxDQUFDLE9BQU87UUFBRSxPQUFPO0lBQ3JCLE9BQU8sQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQzNCLFdBQVcsQ0FBQyxPQUFPLENBQUMsQ0FBQztJQUNyQixPQUFPLENBQUMsRUFBRSxDQUFDLFdBQVcsQ0FBQyxFQUFFLElBQUksRUFBRSxPQUFPLEVBQUUsS0FBSyxFQUFFLENBQUMsQ0FBQztBQUNyRCxDQUFDO0FBTkQsOEJBTUM7QUFFRCxzR0FBc0c7QUFDdEcsdUdBQXVHO0FBQ3ZHLG9HQUFvRztBQUNwRyxzRUFBc0U7QUFDdEUsU0FBZ0IsYUFBYSxDQUFDLFVBQWtCO0lBQzVDLE1BQU0sT0FBTyxHQUFHLE1BQU0sQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUNuQyxJQUFJLENBQUMsT0FBTztRQUFFLE9BQU87SUFDckIsT0FBTyxDQUFDLEVBQUUsQ0FBQyxXQUFXLENBQUMsRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLENBQUMsQ0FBQztBQUM3QyxDQUFDO0FBSkQsc0NBSUMiLCJzb3VyY2VzQ29udGVudCI6WyIvLyBCcm93c2VyLXNpZGUgY3Jvc3MtdGFiIHdyaXRlIHN5bmMgZm9yIEJ1bGtEYXRhYmFzZTIsIG92ZXIgQnJvYWRjYXN0Q2hhbm5lbCAob25lIGNoYW5uZWwgcGVyXG4vLyBjb2xsZWN0aW9uLCBzYW1lLW9yaWdpbiB0YWJzIGFuZCB3b3JrZXJzKS4gSXQgZG9lcyBOT1QgcGVyc2lzdCBhbnl0aGluZyDigJQgZWFjaCBzaWRlIHdyaXRlcyB0byBkaXNrXG4vLyBpdHNlbGY7IHRoaXMganVzdCByZWxheXMgbGl2ZSB3cml0ZXMgdG8gb3RoZXIgb3BlbiB0YWJzL3dvcmtlcnMgYW5kLCB3aGVuIG9uZSBzdGFydHMgdXAsIGFza3MgcGVlcnNcbi8vIGZvciB3cml0ZXMgdGhleSd2ZSBtYWRlIHJlY2VudGx5IHRoYXQgbWF5IG5vdCBiZSBvbiBkaXNrIHlldC4gTm8tb3AgaW4gTm9kZSAvIHdoZXJlXG4vLyBCcm9hZGNhc3RDaGFubmVsIGlzIHVuYXZhaWxhYmxlLCBzbyBCdWxrRGF0YWJhc2UyIGNhbiBjYWxsIHRoZXNlIHVuY29uZGl0aW9uYWxseS4gVGhpcyBpcyBhblxuLy8gb3B0aW9uYWwgZmVhdHVyZSwgc28gdGhlcmUgaXMgbm8gZmFsbGJhY2suXG5cbmV4cG9ydCB0eXBlIFJlbW90ZVdyaXRlID0geyBrZXk6IHN0cmluZzsgdGltZTogbnVtYmVyOyBkZWxldGVkPzogYm9vbGVhbjsgdmFsdWU/OiB1bmtub3duIH07XG5cbi8vIFdyaXRlcyBvbGRlciB0aGFuIHRoaXMgYXJlIGFzc3VtZWQgYWxyZWFkeSBmbHVzaGVkIHRvIGRpc2sgKGEgZnJlc2hseS1vcGVuZWQgdGFiIGdldHMgdGhvc2UgYnlcbi8vIHJlYWRpbmcgZGlzayksIHNvIGEgdGFiIG9ubHkgcmVwbGF5cyB3cml0ZXMgbmV3ZXIgdGhhbiB0aGlzIHdoZW4gYSBwZWVyIHNheXMgaGVsbG8uXG5jb25zdCBSRUNFTlRfV0lORE9XX01TID0gNjBfMDAwO1xuXG50eXBlIENoYW5uZWwgPSB7XG4gICAgYmM6IEJyb2FkY2FzdENoYW5uZWw7XG4gICAgc3Vic2NyaWJlcnM6ICgod3JpdGU6IFJlbW90ZVdyaXRlKSA9PiB2b2lkKVtdO1xuICAgIC8vIENhbGxlZCB3aGVuIGEgcGVlciBhc2tzIGV2ZXJ5b25lIHRvIHNlYWwgKHN0b3AgYXBwZW5kaW5nIHRvKSB0aGVpciBjdXJyZW50IHN0cmVhbSBmaWxlLlxuICAgIHNlYWxTdWJzY3JpYmVyczogKCgpID0+IHZvaWQpW107XG4gICAgLy8gVGhpcyB0YWIncyBvd24gcmVjZW50IHdyaXRlcywga2VwdCBzbyBpdCBjYW4gYW5zd2VyIGFub3RoZXIgdGFiJ3MgXCJoZWxsb1wiLlxuICAgIHJlY2VudDogUmVtb3RlV3JpdGVbXTtcbn07XG5cbmNvbnN0IGNoYW5uZWxzID0gbmV3IE1hcDxzdHJpbmcsIENoYW5uZWw+KCk7XG5cbmV4cG9ydCBmdW5jdGlvbiBpc1N5bmNTdXBwb3J0ZWQoKTogYm9vbGVhbiB7XG4gICAgLy8gd2luZG93IChtYWluLXRocmVhZCBicm93c2VyKSBvciBXb3JrZXJHbG9iYWxTY29wZSAoV2ViL1NlcnZpY2UvU2hhcmVkIHdvcmtlcikg4oCUIGJ1dCBub3QgTm9kZVxuICAgIC8vICh0eXBlc2FmZWNzcydzIGlzTm9kZSBpcyBgdHlwZW9mIHdpbmRvdyA9PT0gXCJ1bmRlZmluZWRcImAsIHdoaWNoIG1pc2NsYXNzaWZpZXMgd29ya2VycylcbiAgICBsZXQgaXNCcm93c2VyT3JXb3JrZXIgPSB0eXBlb2Ygd2luZG93ICE9PSBcInVuZGVmaW5lZFwiXG4gICAgICAgIHx8IFwiV29ya2VyR2xvYmFsU2NvcGVcIiBpbiBnbG9iYWxUaGlzO1xuICAgIHJldHVybiBpc0Jyb3dzZXJPcldvcmtlciAmJiB0eXBlb2YgQnJvYWRjYXN0Q2hhbm5lbCAhPT0gXCJ1bmRlZmluZWRcIjtcbn1cblxuZnVuY3Rpb24gcHJ1bmVSZWNlbnQoY2hhbm5lbDogQ2hhbm5lbCkge1xuICAgIGNvbnN0IGN1dG9mZiA9IERhdGUubm93KCkgLSBSRUNFTlRfV0lORE9XX01TO1xuICAgIGNoYW5uZWwucmVjZW50ID0gY2hhbm5lbC5yZWNlbnQuZmlsdGVyKHcgPT4gdy50aW1lID49IGN1dG9mZik7XG59XG5cbmZ1bmN0aW9uIGRlbGl2ZXIoY2hhbm5lbDogQ2hhbm5lbCwgd3JpdGU6IFJlbW90ZVdyaXRlKSB7XG4gICAgZm9yIChjb25zdCBzdWIgb2YgY2hhbm5lbC5zdWJzY3JpYmVycykgc3ViKHdyaXRlKTtcbn1cblxuZnVuY3Rpb24gZW5zdXJlKGNvbGxlY3Rpb246IHN0cmluZyk6IENoYW5uZWwgfCB1bmRlZmluZWQge1xuICAgIGlmICghaXNTeW5jU3VwcG9ydGVkKCkpIHJldHVybiB1bmRlZmluZWQ7XG4gICAgbGV0IGNoYW5uZWwgPSBjaGFubmVscy5nZXQoY29sbGVjdGlvbik7XG4gICAgaWYgKGNoYW5uZWwpIHJldHVybiBjaGFubmVsO1xuICAgIC8vIEJyb2FkY2FzdENoYW5uZWwgbmV2ZXIgZGVsaXZlcnMgYSBtZXNzYWdlIGJhY2sgdG8gdGhlIGluc3RhbmNlIHRoYXQgc2VudCBpdCwgc28gYSB0YWIgbmV2ZXJcbiAgICAvLyBoZWFycyBpdHMgb3duIHdyaXRlcyDigJQgb25seSB0aGUgb3RoZXIgb3BlbiB0YWJzIGRvLlxuICAgIGNvbnN0IGJjID0gbmV3IEJyb2FkY2FzdENoYW5uZWwoYGJ1bGtEYXRhYmFzZTI6JHtjb2xsZWN0aW9ufWApO1xuICAgIGNvbnN0IGNyZWF0ZWQ6IENoYW5uZWwgPSB7IGJjLCBzdWJzY3JpYmVyczogW10sIHNlYWxTdWJzY3JpYmVyczogW10sIHJlY2VudDogW10gfTtcbiAgICBiYy5vbm1lc3NhZ2UgPSAoZXZlbnQ6IE1lc3NhZ2VFdmVudCkgPT4ge1xuICAgICAgICBjb25zdCBtc2cgPSBldmVudC5kYXRhIGFzIHsgdHlwZTogc3RyaW5nOyB3cml0ZT86IFJlbW90ZVdyaXRlOyB3cml0ZXM/OiBSZW1vdGVXcml0ZVtdIH07XG4gICAgICAgIGlmICghbXNnKSByZXR1cm47XG4gICAgICAgIGlmIChtc2cudHlwZSA9PT0gXCJ3cml0ZVwiICYmIG1zZy53cml0ZSkge1xuICAgICAgICAgICAgZGVsaXZlcihjcmVhdGVkLCBtc2cud3JpdGUpO1xuICAgICAgICB9IGVsc2UgaWYgKG1zZy50eXBlID09PSBcInJlY2VudFwiICYmIG1zZy53cml0ZXMpIHtcbiAgICAgICAgICAgIGZvciAoY29uc3Qgd3JpdGUgb2YgbXNnLndyaXRlcykgZGVsaXZlcihjcmVhdGVkLCB3cml0ZSk7XG4gICAgICAgIH0gZWxzZSBpZiAobXNnLnR5cGUgPT09IFwic2VhbFwiKSB7XG4gICAgICAgICAgICAvLyBBIHBlZXIgaXMgYWJvdXQgdG8gZm9sZCByZWNlbnQgZGF0YTsgc3RvcCBhcHBlbmRpbmcgdG8gb3VyIGN1cnJlbnQgc3RyZWFtIGZpbGUgc28gaXRcbiAgICAgICAgICAgIC8vIGJlY29tZXMgY29tcGxldGUgYW5kIHRoZSBtZXJnZSBjYW4gZm9sZCBpdCB3aG9sZS4gT3VyIG5leHQgd3JpdGUgc3RhcnRzIGEgZnJlc2ggZmlsZS5cbiAgICAgICAgICAgIGZvciAoY29uc3Qgc3ViIG9mIGNyZWF0ZWQuc2VhbFN1YnNjcmliZXJzKSBzdWIoKTtcbiAgICAgICAgfSBlbHNlIGlmIChtc2cudHlwZSA9PT0gXCJoZWxsb1wiKSB7XG4gICAgICAgICAgICAvLyBBbm90aGVyIHRhYiBqdXN0IHN0YXJ0ZWQ7IHJlcGxheSBvdXIgcmVjZW50IHdyaXRlcyBzbyBpdCBkb2Vzbid0IG1pc3MgYW55LiBUaGUgcmVwbHkgZ29lc1xuICAgICAgICAgICAgLy8gdG8gZXZlcnkgdGFiLCBidXQgcGVlcnMgdGhhdCBhbHJlYWR5IGhhdmUgYSB3cml0ZSBpZ25vcmUgaXQgKGl0cyB0aW1lc3RhbXAgaXNuJ3QgbmV3ZXJcbiAgICAgICAgICAgIC8vIHRoYW4gd2hhdCB0aGV5IGhvbGQpLCBzbyB0aGUgcmVkdW5kYW50IGJyb2FkY2FzdCBpcyBoYXJtbGVzcy5cbiAgICAgICAgICAgIHBydW5lUmVjZW50KGNyZWF0ZWQpO1xuICAgICAgICAgICAgaWYgKGNyZWF0ZWQucmVjZW50Lmxlbmd0aCkgY3JlYXRlZC5iYy5wb3N0TWVzc2FnZSh7IHR5cGU6IFwicmVjZW50XCIsIHdyaXRlczogY3JlYXRlZC5yZWNlbnQgfSk7XG4gICAgICAgIH1cbiAgICB9O1xuICAgIGNoYW5uZWxzLnNldChjb2xsZWN0aW9uLCBjcmVhdGVkKTtcbiAgICByZXR1cm4gY3JlYXRlZDtcbn1cblxuLy8gU3Vic2NyaWJlIHRvIHJlbW90ZSB3cml0ZXMgZm9yIGEgY29sbGVjdGlvbi4gUmVjZW50IHdyaXRlcyBmcm9tIGFscmVhZHktb3BlbiB0YWJzIGFycml2ZSB0aHJvdWdoIHRoZVxuLy8gc2FtZSBvbldyaXRlIGNhbGxiYWNrIChhcyB0aGUgcmVwbHkgdG8gb3VyIGhlbGxvKSwgc28gdGhlIHJldHVybmVkIGFycmF5IGlzIGFsd2F5cyBlbXB0eSDigJQgaXQncyBrZXB0XG4vLyBvbmx5IGZvciBBUEkgY29tcGF0aWJpbGl0eSB3aXRoIGNhbGxlcnMgdGhhdCBhd2FpdCBpdC5cbmV4cG9ydCBmdW5jdGlvbiBjb25uZWN0KGNvbGxlY3Rpb246IHN0cmluZywgb25Xcml0ZTogKHdyaXRlOiBSZW1vdGVXcml0ZSkgPT4gdm9pZCwgb25TZWFsPzogKCkgPT4gdm9pZCk6IFByb21pc2U8UmVtb3RlV3JpdGVbXT4ge1xuICAgIGNvbnN0IGNoYW5uZWwgPSBlbnN1cmUoY29sbGVjdGlvbik7XG4gICAgaWYgKCFjaGFubmVsKSByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKFtdKTtcbiAgICBjaGFubmVsLnN1YnNjcmliZXJzLnB1c2gob25Xcml0ZSk7XG4gICAgaWYgKG9uU2VhbCkgY2hhbm5lbC5zZWFsU3Vic2NyaWJlcnMucHVzaChvblNlYWwpO1xuICAgIGNoYW5uZWwuYmMucG9zdE1lc3NhZ2UoeyB0eXBlOiBcImhlbGxvXCIgfSk7XG4gICAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShbXSk7XG59XG5cbmV4cG9ydCBmdW5jdGlvbiBicm9hZGNhc3QoY29sbGVjdGlvbjogc3RyaW5nLCB3cml0ZTogUmVtb3RlV3JpdGUpOiB2b2lkIHtcbiAgICBjb25zdCBjaGFubmVsID0gZW5zdXJlKGNvbGxlY3Rpb24pO1xuICAgIGlmICghY2hhbm5lbCkgcmV0dXJuO1xuICAgIGNoYW5uZWwucmVjZW50LnB1c2god3JpdGUpO1xuICAgIHBydW5lUmVjZW50KGNoYW5uZWwpO1xuICAgIGNoYW5uZWwuYmMucG9zdE1lc3NhZ2UoeyB0eXBlOiBcIndyaXRlXCIsIHdyaXRlIH0pO1xufVxuXG4vLyBBc2sgZXZlcnkgb3RoZXIgb3BlbiB0YWIgb2YgdGhpcyBjb2xsZWN0aW9uIHRvIHNlYWwgKHN0b3AgYXBwZW5kaW5nIHRvKSBpdHMgY3VycmVudCBzdHJlYW0gZmlsZSwgc29cbi8vIGEgbWVyZ2UgY2FuIGZvbGQgcmVjZW50IGRhdGEgdXAgdG8gdGhlIHByZXNlbnQgd2l0aG91dCByYWNpbmcgYW4gYXBwZW5kLiBCZXN0LWVmZm9ydDogbm8tb3AgaW4gTm9kZSxcbi8vIGFuZCBhIHBlZXIgdGhhdCBtaXNzZXMgaXQganVzdCBrZWVwcyBhcHBlbmRpbmcg4oCUIHdvcnN0IGNhc2UgdGhlIG1lcmdlIGZvbGRzIGEgcHJlZml4IGFuZCB0aGUgcmVzdFxuLy8gZm9sZHMgbGF0ZXIgKGR1cGxpY2F0aW9uLCByZXNvbHZlZCBieSB3cml0ZS10aW1lKSwgbmV2ZXIgZGF0YSBsb3NzLlxuZXhwb3J0IGZ1bmN0aW9uIGJyb2FkY2FzdFNlYWwoY29sbGVjdGlvbjogc3RyaW5nKTogdm9pZCB7XG4gICAgY29uc3QgY2hhbm5lbCA9IGVuc3VyZShjb2xsZWN0aW9uKTtcbiAgICBpZiAoIWNoYW5uZWwpIHJldHVybjtcbiAgICBjaGFubmVsLmJjLnBvc3RNZXNzYWdlKHsgdHlwZTogXCJzZWFsXCIgfSk7XG59XG4iXX0=
|
|
104
|
-
/* _JS_SOURCE_HASH = "
|
|
87
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3luY0NsaWVudC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInN5bmNDbGllbnQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBLGlnQkFBaWdCOzs7QUFJamdCLHFMQUFxTDtBQUNyTCxNQUFNLGdCQUFnQixHQUFHLEtBQU0sQ0FBQztBQVdoQyxNQUFNLFFBQVEsR0FBRyxJQUFJLEdBQUcsRUFBbUIsQ0FBQztBQUU1QyxTQUFnQixlQUFlO0lBQzNCLHNMQUFzTDtJQUN0TCxJQUFJLGlCQUFpQixHQUFHLE9BQU8sTUFBTSxLQUFLLFdBQVc7V0FDOUMsbUJBQW1CLElBQUksVUFBVSxDQUFDO0lBQ3pDLE9BQU8saUJBQWlCLElBQUksT0FBTyxnQkFBZ0IsS0FBSyxXQUFXLENBQUM7QUFDeEUsQ0FBQztBQUxELDBDQUtDO0FBRUQsU0FBUyxXQUFXLENBQUMsT0FBZ0I7SUFDakMsTUFBTSxNQUFNLEdBQUcsSUFBSSxDQUFDLEdBQUcsRUFBRSxHQUFHLGdCQUFnQixDQUFDO0lBQzdDLE9BQU8sQ0FBQyxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsSUFBSSxJQUFJLE1BQU0sQ0FBQyxDQUFDO0FBQ2xFLENBQUM7QUFFRCxTQUFTLE9BQU8sQ0FBQyxPQUFnQixFQUFFLEtBQWtCO0lBQ2pELEtBQUssTUFBTSxHQUFHLElBQUksT0FBTyxDQUFDLFdBQVc7UUFBRSxHQUFHLENBQUMsS0FBSyxDQUFDLENBQUM7QUFDdEQsQ0FBQztBQUVELFNBQVMsTUFBTSxDQUFDLFVBQWtCO0lBQzlCLElBQUksQ0FBQyxlQUFlLEVBQUU7UUFBRSxPQUFPLFNBQVMsQ0FBQztJQUN6QyxJQUFJLE9BQU8sR0FBRyxRQUFRLENBQUMsR0FBRyxDQUFDLFVBQVUsQ0FBQyxDQUFDO0lBQ3ZDLElBQUksT0FBTztRQUFFLE9BQU8sT0FBTyxDQUFDO0lBQzVCLGtKQUFrSjtJQUNsSixNQUFNLEVBQUUsR0FBRyxJQUFJLGdCQUFnQixDQUFDLGlCQUFpQixVQUFVLEVBQUUsQ0FBQyxDQUFDO0lBQy9ELE1BQU0sT0FBTyxHQUFZLEVBQUUsRUFBRSxFQUFFLFdBQVcsRUFBRSxFQUFFLEVBQUUsZUFBZSxFQUFFLEVBQUUsRUFBRSxNQUFNLEVBQUUsRUFBRSxFQUFFLENBQUM7SUFDbEYsRUFBRSxDQUFDLFNBQVMsR0FBRyxDQUFDLEtBQW1CLEVBQUUsRUFBRTtRQUNuQyxNQUFNLEdBQUcsR0FBRyxLQUFLLENBQUMsSUFBcUUsQ0FBQztRQUN4RixJQUFJLENBQUMsR0FBRztZQUFFLE9BQU87UUFDakIsSUFBSSxHQUFHLENBQUMsSUFBSSxLQUFLLE9BQU8sSUFBSSxHQUFHLENBQUMsS0FBSyxFQUFFLENBQUM7WUFDcEMsT0FBTyxDQUFDLE9BQU8sRUFBRSxHQUFHLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDaEMsQ0FBQzthQUFNLElBQUksR0FBRyxDQUFDLElBQUksS0FBSyxRQUFRLElBQUksR0FBRyxDQUFDLE1BQU0sRUFBRSxDQUFDO1lBQzdDLEtBQUssTUFBTSxLQUFLLElBQUksR0FBRyxDQUFDLE1BQU07Z0JBQUUsT0FBTyxDQUFDLE9BQU8sRUFBRSxLQUFLLENBQUMsQ0FBQztRQUM1RCxDQUFDO2FBQU0sSUFBSSxHQUFHLENBQUMsSUFBSSxLQUFLLE1BQU0sRUFBRSxDQUFDO1lBQzdCLDZLQUE2SztZQUM3SyxLQUFLLE1BQU0sR0FBRyxJQUFJLE9BQU8sQ0FBQyxlQUFlO2dCQUFFLEdBQUcsRUFBRSxDQUFDO1FBQ3JELENBQUM7YUFBTSxJQUFJLEdBQUcsQ0FBQyxJQUFJLEtBQUssT0FBTyxFQUFFLENBQUM7WUFDOUIsaVBBQWlQO1lBQ2pQLFdBQVcsQ0FBQyxPQUFPLENBQUMsQ0FBQztZQUNyQixJQUFJLE9BQU8sQ0FBQyxNQUFNLENBQUMsTUFBTTtnQkFBRSxPQUFPLENBQUMsRUFBRSxDQUFDLFdBQVcsQ0FBQyxFQUFFLElBQUksRUFBRSxRQUFRLEVBQUUsTUFBTSxFQUFFLE9BQU8sQ0FBQyxNQUFNLEVBQUUsQ0FBQyxDQUFDO1FBQ2xHLENBQUM7SUFDTCxDQUFDLENBQUM7SUFDRixRQUFRLENBQUMsR0FBRyxDQUFDLFVBQVUsRUFBRSxPQUFPLENBQUMsQ0FBQztJQUNsQyxPQUFPLE9BQU8sQ0FBQztBQUNuQixDQUFDO0FBRUQsbVFBQW1RO0FBQ25RLFNBQWdCLE9BQU8sQ0FBQyxVQUFrQixFQUFFLE9BQXFDLEVBQUUsTUFBbUI7SUFDbEcsTUFBTSxPQUFPLEdBQUcsTUFBTSxDQUFDLFVBQVUsQ0FBQyxDQUFDO0lBQ25DLElBQUksQ0FBQyxPQUFPO1FBQUUsT0FBTyxPQUFPLENBQUMsT0FBTyxDQUFDLEVBQUUsQ0FBQyxDQUFDO0lBQ3pDLE9BQU8sQ0FBQyxXQUFXLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFDO0lBQ2xDLElBQUksTUFBTTtRQUFFLE9BQU8sQ0FBQyxlQUFlLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFDO0lBQ2pELE9BQU8sQ0FBQyxFQUFFLENBQUMsV0FBVyxDQUFDLEVBQUUsSUFBSSxFQUFFLE9BQU8sRUFBRSxDQUFDLENBQUM7SUFDMUMsT0FBTyxPQUFPLENBQUMsT0FBTyxDQUFDLEVBQUUsQ0FBQyxDQUFDO0FBQy9CLENBQUM7QUFQRCwwQkFPQztBQUVELFNBQWdCLFNBQVMsQ0FBQyxVQUFrQixFQUFFLEtBQWtCO0lBQzVELE1BQU0sT0FBTyxHQUFHLE1BQU0sQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUNuQyxJQUFJLENBQUMsT0FBTztRQUFFLE9BQU87SUFDckIsT0FBTyxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUM7SUFDM0IsV0FBVyxDQUFDLE9BQU8sQ0FBQyxDQUFDO0lBQ3JCLE9BQU8sQ0FBQyxFQUFFLENBQUMsV0FBVyxDQUFDLEVBQUUsSUFBSSxFQUFFLE9BQU8sRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFDO0FBQ3JELENBQUM7QUFORCw4QkFNQztBQUVELGlYQUFpWDtBQUNqWCxTQUFnQixhQUFhLENBQUMsVUFBa0I7SUFDNUMsTUFBTSxPQUFPLEdBQUcsTUFBTSxDQUFDLFVBQVUsQ0FBQyxDQUFDO0lBQ25DLElBQUksQ0FBQyxPQUFPO1FBQUUsT0FBTztJQUNyQixPQUFPLENBQUMsRUFBRSxDQUFDLFdBQVcsQ0FBQyxFQUFFLElBQUksRUFBRSxNQUFNLEVBQUUsQ0FBQyxDQUFDO0FBQzdDLENBQUM7QUFKRCxzQ0FJQyIsInNvdXJjZXNDb250ZW50IjpbIi8vIEJyb3dzZXItc2lkZSBjcm9zcy10YWIgd3JpdGUgc3luYyBmb3IgQnVsa0RhdGFiYXNlMiwgb3ZlciBCcm9hZGNhc3RDaGFubmVsIChvbmUgY2hhbm5lbCBwZXIgY29sbGVjdGlvbiwgc2FtZS1vcmlnaW4gdGFicyBhbmQgd29ya2VycykuIEl0IGRvZXMgTk9UIHBlcnNpc3QgYW55dGhpbmcg4oCUIGVhY2ggc2lkZSB3cml0ZXMgdG8gZGlzayBpdHNlbGY7IHRoaXMganVzdCByZWxheXMgbGl2ZSB3cml0ZXMgdG8gb3RoZXIgb3BlbiB0YWJzL3dvcmtlcnMgYW5kLCB3aGVuIG9uZSBzdGFydHMgdXAsIGFza3MgcGVlcnMgZm9yIHdyaXRlcyB0aGV5J3ZlIG1hZGUgcmVjZW50bHkgdGhhdCBtYXkgbm90IGJlIG9uIGRpc2sgeWV0LiBOby1vcCBpbiBOb2RlIC8gd2hlcmUgQnJvYWRjYXN0Q2hhbm5lbCBpcyB1bmF2YWlsYWJsZSwgc28gQnVsa0RhdGFiYXNlMiBjYW4gY2FsbCB0aGVzZSB1bmNvbmRpdGlvbmFsbHkuIFRoaXMgaXMgYW4gb3B0aW9uYWwgZmVhdHVyZSwgc28gdGhlcmUgaXMgbm8gZmFsbGJhY2suXG5cbmV4cG9ydCB0eXBlIFJlbW90ZVdyaXRlID0geyBrZXk6IHN0cmluZzsgdGltZTogbnVtYmVyOyBkZWxldGVkPzogYm9vbGVhbjsgdmFsdWU/OiB1bmtub3duIH07XG5cbi8vIFdyaXRlcyBvbGRlciB0aGFuIHRoaXMgYXJlIGFzc3VtZWQgYWxyZWFkeSBmbHVzaGVkIHRvIGRpc2sgKGEgZnJlc2hseS1vcGVuZWQgdGFiIGdldHMgdGhvc2UgYnkgcmVhZGluZyBkaXNrKSwgc28gYSB0YWIgb25seSByZXBsYXlzIHdyaXRlcyBuZXdlciB0aGFuIHRoaXMgd2hlbiBhIHBlZXIgc2F5cyBoZWxsby5cbmNvbnN0IFJFQ0VOVF9XSU5ET1dfTVMgPSA2MF8wMDA7XG5cbnR5cGUgQ2hhbm5lbCA9IHtcbiAgICBiYzogQnJvYWRjYXN0Q2hhbm5lbDtcbiAgICBzdWJzY3JpYmVyczogKCh3cml0ZTogUmVtb3RlV3JpdGUpID0+IHZvaWQpW107XG4gICAgLy8gQ2FsbGVkIHdoZW4gYSBwZWVyIGFza3MgZXZlcnlvbmUgdG8gc2VhbCAoc3RvcCBhcHBlbmRpbmcgdG8pIHRoZWlyIGN1cnJlbnQgc3RyZWFtIGZpbGUuXG4gICAgc2VhbFN1YnNjcmliZXJzOiAoKCkgPT4gdm9pZClbXTtcbiAgICAvLyBUaGlzIHRhYidzIG93biByZWNlbnQgd3JpdGVzLCBrZXB0IHNvIGl0IGNhbiBhbnN3ZXIgYW5vdGhlciB0YWIncyBcImhlbGxvXCIuXG4gICAgcmVjZW50OiBSZW1vdGVXcml0ZVtdO1xufTtcblxuY29uc3QgY2hhbm5lbHMgPSBuZXcgTWFwPHN0cmluZywgQ2hhbm5lbD4oKTtcblxuZXhwb3J0IGZ1bmN0aW9uIGlzU3luY1N1cHBvcnRlZCgpOiBib29sZWFuIHtcbiAgICAvLyB3aW5kb3cgKG1haW4tdGhyZWFkIGJyb3dzZXIpIG9yIFdvcmtlckdsb2JhbFNjb3BlIChXZWIvU2VydmljZS9TaGFyZWQgd29ya2VyKSDigJQgYnV0IG5vdCBOb2RlICh0eXBlc2FmZWNzcydzIGlzTm9kZSBpcyBgdHlwZW9mIHdpbmRvdyA9PT0gXCJ1bmRlZmluZWRcImAsIHdoaWNoIG1pc2NsYXNzaWZpZXMgd29ya2VycylcbiAgICBsZXQgaXNCcm93c2VyT3JXb3JrZXIgPSB0eXBlb2Ygd2luZG93ICE9PSBcInVuZGVmaW5lZFwiXG4gICAgICAgIHx8IFwiV29ya2VyR2xvYmFsU2NvcGVcIiBpbiBnbG9iYWxUaGlzO1xuICAgIHJldHVybiBpc0Jyb3dzZXJPcldvcmtlciAmJiB0eXBlb2YgQnJvYWRjYXN0Q2hhbm5lbCAhPT0gXCJ1bmRlZmluZWRcIjtcbn1cblxuZnVuY3Rpb24gcHJ1bmVSZWNlbnQoY2hhbm5lbDogQ2hhbm5lbCkge1xuICAgIGNvbnN0IGN1dG9mZiA9IERhdGUubm93KCkgLSBSRUNFTlRfV0lORE9XX01TO1xuICAgIGNoYW5uZWwucmVjZW50ID0gY2hhbm5lbC5yZWNlbnQuZmlsdGVyKHcgPT4gdy50aW1lID49IGN1dG9mZik7XG59XG5cbmZ1bmN0aW9uIGRlbGl2ZXIoY2hhbm5lbDogQ2hhbm5lbCwgd3JpdGU6IFJlbW90ZVdyaXRlKSB7XG4gICAgZm9yIChjb25zdCBzdWIgb2YgY2hhbm5lbC5zdWJzY3JpYmVycykgc3ViKHdyaXRlKTtcbn1cblxuZnVuY3Rpb24gZW5zdXJlKGNvbGxlY3Rpb246IHN0cmluZyk6IENoYW5uZWwgfCB1bmRlZmluZWQge1xuICAgIGlmICghaXNTeW5jU3VwcG9ydGVkKCkpIHJldHVybiB1bmRlZmluZWQ7XG4gICAgbGV0IGNoYW5uZWwgPSBjaGFubmVscy5nZXQoY29sbGVjdGlvbik7XG4gICAgaWYgKGNoYW5uZWwpIHJldHVybiBjaGFubmVsO1xuICAgIC8vIEJyb2FkY2FzdENoYW5uZWwgbmV2ZXIgZGVsaXZlcnMgYSBtZXNzYWdlIGJhY2sgdG8gdGhlIGluc3RhbmNlIHRoYXQgc2VudCBpdCwgc28gYSB0YWIgbmV2ZXIgaGVhcnMgaXRzIG93biB3cml0ZXMg4oCUIG9ubHkgdGhlIG90aGVyIG9wZW4gdGFicyBkby5cbiAgICBjb25zdCBiYyA9IG5ldyBCcm9hZGNhc3RDaGFubmVsKGBidWxrRGF0YWJhc2UyOiR7Y29sbGVjdGlvbn1gKTtcbiAgICBjb25zdCBjcmVhdGVkOiBDaGFubmVsID0geyBiYywgc3Vic2NyaWJlcnM6IFtdLCBzZWFsU3Vic2NyaWJlcnM6IFtdLCByZWNlbnQ6IFtdIH07XG4gICAgYmMub25tZXNzYWdlID0gKGV2ZW50OiBNZXNzYWdlRXZlbnQpID0+IHtcbiAgICAgICAgY29uc3QgbXNnID0gZXZlbnQuZGF0YSBhcyB7IHR5cGU6IHN0cmluZzsgd3JpdGU/OiBSZW1vdGVXcml0ZTsgd3JpdGVzPzogUmVtb3RlV3JpdGVbXSB9O1xuICAgICAgICBpZiAoIW1zZykgcmV0dXJuO1xuICAgICAgICBpZiAobXNnLnR5cGUgPT09IFwid3JpdGVcIiAmJiBtc2cud3JpdGUpIHtcbiAgICAgICAgICAgIGRlbGl2ZXIoY3JlYXRlZCwgbXNnLndyaXRlKTtcbiAgICAgICAgfSBlbHNlIGlmIChtc2cudHlwZSA9PT0gXCJyZWNlbnRcIiAmJiBtc2cud3JpdGVzKSB7XG4gICAgICAgICAgICBmb3IgKGNvbnN0IHdyaXRlIG9mIG1zZy53cml0ZXMpIGRlbGl2ZXIoY3JlYXRlZCwgd3JpdGUpO1xuICAgICAgICB9IGVsc2UgaWYgKG1zZy50eXBlID09PSBcInNlYWxcIikge1xuICAgICAgICAgICAgLy8gQSBwZWVyIGlzIGFib3V0IHRvIGZvbGQgcmVjZW50IGRhdGE7IHN0b3AgYXBwZW5kaW5nIHRvIG91ciBjdXJyZW50IHN0cmVhbSBmaWxlIHNvIGl0IGJlY29tZXMgY29tcGxldGUgYW5kIHRoZSBtZXJnZSBjYW4gZm9sZCBpdCB3aG9sZS4gT3VyIG5leHQgd3JpdGUgc3RhcnRzIGEgZnJlc2ggZmlsZS5cbiAgICAgICAgICAgIGZvciAoY29uc3Qgc3ViIG9mIGNyZWF0ZWQuc2VhbFN1YnNjcmliZXJzKSBzdWIoKTtcbiAgICAgICAgfSBlbHNlIGlmIChtc2cudHlwZSA9PT0gXCJoZWxsb1wiKSB7XG4gICAgICAgICAgICAvLyBBbm90aGVyIHRhYiBqdXN0IHN0YXJ0ZWQ7IHJlcGxheSBvdXIgcmVjZW50IHdyaXRlcyBzbyBpdCBkb2Vzbid0IG1pc3MgYW55LiBUaGUgcmVwbHkgZ29lcyB0byBldmVyeSB0YWIsIGJ1dCBwZWVycyB0aGF0IGFscmVhZHkgaGF2ZSBhIHdyaXRlIGlnbm9yZSBpdCAoaXRzIHRpbWVzdGFtcCBpc24ndCBuZXdlciB0aGFuIHdoYXQgdGhleSBob2xkKSwgc28gdGhlIHJlZHVuZGFudCBicm9hZGNhc3QgaXMgaGFybWxlc3MuXG4gICAgICAgICAgICBwcnVuZVJlY2VudChjcmVhdGVkKTtcbiAgICAgICAgICAgIGlmIChjcmVhdGVkLnJlY2VudC5sZW5ndGgpIGNyZWF0ZWQuYmMucG9zdE1lc3NhZ2UoeyB0eXBlOiBcInJlY2VudFwiLCB3cml0ZXM6IGNyZWF0ZWQucmVjZW50IH0pO1xuICAgICAgICB9XG4gICAgfTtcbiAgICBjaGFubmVscy5zZXQoY29sbGVjdGlvbiwgY3JlYXRlZCk7XG4gICAgcmV0dXJuIGNyZWF0ZWQ7XG59XG5cbi8vIFN1YnNjcmliZSB0byByZW1vdGUgd3JpdGVzIGZvciBhIGNvbGxlY3Rpb24uIFJlY2VudCB3cml0ZXMgZnJvbSBhbHJlYWR5LW9wZW4gdGFicyBhcnJpdmUgdGhyb3VnaCB0aGUgc2FtZSBvbldyaXRlIGNhbGxiYWNrIChhcyB0aGUgcmVwbHkgdG8gb3VyIGhlbGxvKSwgc28gdGhlIHJldHVybmVkIGFycmF5IGlzIGFsd2F5cyBlbXB0eSDigJQgaXQncyBrZXB0IG9ubHkgZm9yIEFQSSBjb21wYXRpYmlsaXR5IHdpdGggY2FsbGVycyB0aGF0IGF3YWl0IGl0LlxuZXhwb3J0IGZ1bmN0aW9uIGNvbm5lY3QoY29sbGVjdGlvbjogc3RyaW5nLCBvbldyaXRlOiAod3JpdGU6IFJlbW90ZVdyaXRlKSA9PiB2b2lkLCBvblNlYWw/OiAoKSA9PiB2b2lkKTogUHJvbWlzZTxSZW1vdGVXcml0ZVtdPiB7XG4gICAgY29uc3QgY2hhbm5lbCA9IGVuc3VyZShjb2xsZWN0aW9uKTtcbiAgICBpZiAoIWNoYW5uZWwpIHJldHVybiBQcm9taXNlLnJlc29sdmUoW10pO1xuICAgIGNoYW5uZWwuc3Vic2NyaWJlcnMucHVzaChvbldyaXRlKTtcbiAgICBpZiAob25TZWFsKSBjaGFubmVsLnNlYWxTdWJzY3JpYmVycy5wdXNoKG9uU2VhbCk7XG4gICAgY2hhbm5lbC5iYy5wb3N0TWVzc2FnZSh7IHR5cGU6IFwiaGVsbG9cIiB9KTtcbiAgICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKFtdKTtcbn1cblxuZXhwb3J0IGZ1bmN0aW9uIGJyb2FkY2FzdChjb2xsZWN0aW9uOiBzdHJpbmcsIHdyaXRlOiBSZW1vdGVXcml0ZSk6IHZvaWQge1xuICAgIGNvbnN0IGNoYW5uZWwgPSBlbnN1cmUoY29sbGVjdGlvbik7XG4gICAgaWYgKCFjaGFubmVsKSByZXR1cm47XG4gICAgY2hhbm5lbC5yZWNlbnQucHVzaCh3cml0ZSk7XG4gICAgcHJ1bmVSZWNlbnQoY2hhbm5lbCk7XG4gICAgY2hhbm5lbC5iYy5wb3N0TWVzc2FnZSh7IHR5cGU6IFwid3JpdGVcIiwgd3JpdGUgfSk7XG59XG5cbi8vIEFzayBldmVyeSBvdGhlciBvcGVuIHRhYiBvZiB0aGlzIGNvbGxlY3Rpb24gdG8gc2VhbCAoc3RvcCBhcHBlbmRpbmcgdG8pIGl0cyBjdXJyZW50IHN0cmVhbSBmaWxlLCBzbyBhIG1lcmdlIGNhbiBmb2xkIHJlY2VudCBkYXRhIHVwIHRvIHRoZSBwcmVzZW50IHdpdGhvdXQgcmFjaW5nIGFuIGFwcGVuZC4gQmVzdC1lZmZvcnQ6IG5vLW9wIGluIE5vZGUsIGFuZCBhIHBlZXIgdGhhdCBtaXNzZXMgaXQganVzdCBrZWVwcyBhcHBlbmRpbmcg4oCUIHdvcnN0IGNhc2UgdGhlIG1lcmdlIGZvbGRzIGEgcHJlZml4IGFuZCB0aGUgcmVzdCBmb2xkcyBsYXRlciAoZHVwbGljYXRpb24sIHJlc29sdmVkIGJ5IHdyaXRlLXRpbWUpLCBuZXZlciBkYXRhIGxvc3MuXG5leHBvcnQgZnVuY3Rpb24gYnJvYWRjYXN0U2VhbChjb2xsZWN0aW9uOiBzdHJpbmcpOiB2b2lkIHtcbiAgICBjb25zdCBjaGFubmVsID0gZW5zdXJlKGNvbGxlY3Rpb24pO1xuICAgIGlmICghY2hhbm5lbCkgcmV0dXJuO1xuICAgIGNoYW5uZWwuYmMucG9zdE1lc3NhZ2UoeyB0eXBlOiBcInNlYWxcIiB9KTtcbn1cbiJdfQ==
|
|
88
|
+
/* _JS_SOURCE_HASH = "c3aed40d112d1e5210b8ee9a02d667a1c8f1331f529823f55959039b68084cfd"; */
|
|
@@ -1,17 +1,6 @@
|
|
|
1
|
-
// Best-effort "only one tab merges this collection at a time" guard. This is purely an efficiency
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// merging at once only waste work and briefly duplicate data, never corrupt or lose it. The lock just
|
|
5
|
-
// stops that wasted work. It uses localStorage (shared across same-origin tabs) and is a no-op where
|
|
6
|
-
// localStorage is unavailable (Node) — there concurrent merges are harmless, which the Node stress
|
|
7
|
-
// tests exercise. localStorage has no atomic compare-and-swap, so we write-then-reread to shrink the
|
|
8
|
-
// race window, and a TTL frees a lock left behind by a tab that crashed or closed mid-merge.
|
|
9
|
-
//
|
|
10
|
-
// File-lock layer (see acquireMergeFileLock at the bottom): for cross-PROCESS safety (different
|
|
11
|
-
// browsers, different machines pointing at the same remote storage), we ALSO write a `.merge-lock`
|
|
12
|
-
// file inside the collection's storage with our ID. The protocol is check-write-wait-recheck — last
|
|
13
|
-
// writer wins after the 15s settle window. The localStorage lock above covers same-origin tabs; this
|
|
14
|
-
// covers everything else (separate processes, remote shared storage).
|
|
1
|
+
// Best-effort "only one tab merges this collection at a time" guard. This is purely an efficiency measure: correctness does NOT depend on it. Reads resolve by per-row write-time, merges write new files before deleting consumed ones, and a duplicate-heavy region gets re-merged — so two tabs merging at once only waste work and briefly duplicate data, never corrupt or lose it. The lock just stops that wasted work. It uses localStorage (shared across same-origin tabs) and is a no-op where localStorage is unavailable (Node) — there concurrent merges are harmless, which the Node stress tests exercise. localStorage has no atomic compare-and-swap, so we write-then-reread to shrink the race window, and a TTL frees a lock left behind by a tab that crashed or closed mid-merge.
|
|
2
|
+
//
|
|
3
|
+
// File-lock layer (see acquireMergeFileLock at the bottom): for cross-PROCESS safety (different browsers, different machines pointing at the same remote storage), we ALSO write a `.merge-lock` file inside the collection's storage with our ID. The protocol is check-write-wait-recheck — last writer wins after the 15s settle window. The localStorage lock above covers same-origin tabs; this covers everything else (separate processes, remote shared storage).
|
|
15
4
|
|
|
16
5
|
import type { FileStorage } from "../FileFolderAPI";
|
|
17
6
|
|
|
@@ -39,10 +28,7 @@ function lockKey(collection: string): string {
|
|
|
39
28
|
return "bulkDatabase2-merge:" + collection;
|
|
40
29
|
}
|
|
41
30
|
|
|
42
|
-
// Returns true if we now hold the lock (or there's no localStorage to coordinate through, in which case
|
|
43
|
-
// we proceed — concurrent merges are harmless). Returns false only if a DIFFERENT holder has a fresh
|
|
44
|
-
// lock. Re-stamping our own lock always succeeds, so this doubles as a heartbeat: call it periodically
|
|
45
|
-
// during a long, spaced-out merge to keep the lock alive (and detect if another tab took it over).
|
|
31
|
+
// Returns true if we now hold the lock (or there's no localStorage to coordinate through, in which case we proceed — concurrent merges are harmless). Returns false only if a DIFFERENT holder has a fresh lock. Re-stamping our own lock always succeeds, so this doubles as a heartbeat: call it periodically during a long, spaced-out merge to keep the lock alive (and detect if another tab took it over).
|
|
46
32
|
export function tryAcquireMergeLock(collection: string, holderId: string): boolean {
|
|
47
33
|
const ls = getLocalStorage();
|
|
48
34
|
if (!ls) return true;
|
|
@@ -59,8 +45,7 @@ export function tryAcquireMergeLock(collection: string, holderId: string): boole
|
|
|
59
45
|
return ls.getItem(key) === token;
|
|
60
46
|
}
|
|
61
47
|
|
|
62
|
-
// Non-mutating look at the tab lock: who holds it and how long until it goes stale (negative = already
|
|
63
|
-
// stale). For logging why a merge was skipped.
|
|
48
|
+
// Non-mutating look at the tab lock: who holds it and how long until it goes stale (negative = already stale). For logging why a merge was skipped.
|
|
64
49
|
export function peekMergeLock(collection: string): MergeLockInfo | undefined {
|
|
65
50
|
const ls = getLocalStorage();
|
|
66
51
|
if (!ls) return undefined;
|
|
@@ -104,9 +89,7 @@ async function writeMergeFileLock(storage: FileStorage, holderId: string, time:
|
|
|
104
89
|
await storage.set(FILE_LOCK_KEY, Buffer.from(`${holderId}:${time}`, "utf8") as Buffer);
|
|
105
90
|
}
|
|
106
91
|
|
|
107
|
-
// Acquire the cross-process file lock. Resolves to true ONLY if, after writing our ID and waiting
|
|
108
|
-
// FILE_LOCK_SETTLE_MS for concurrent writers to settle, the file still names us. Caller must
|
|
109
|
-
// releaseMergeFileLock() on the same storage + holderId when done.
|
|
92
|
+
// Acquire the cross-process file lock. Resolves to true ONLY if, after writing our ID and waiting FILE_LOCK_SETTLE_MS for concurrent writers to settle, the file still names us. Caller must releaseMergeFileLock() on the same storage + holderId when done.
|
|
110
93
|
export async function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean> {
|
|
111
94
|
const now = Date.now();
|
|
112
95
|
const existing = await readMergeFileLock(storage);
|
|
@@ -117,9 +100,7 @@ export async function tryAcquireMergeFileLock(storage: FileStorage, holderId: st
|
|
|
117
100
|
return !!after && after.holderId === holderId;
|
|
118
101
|
}
|
|
119
102
|
|
|
120
|
-
// Keep the lock fresh. The merge might run longer than FILE_LOCK_TTL_MS; without periodic re-stamping,
|
|
121
|
-
// another process would see the lock as stale and acquire it. Returns a stop function the caller calls
|
|
122
|
-
// in `finally`. Heartbeat is best-effort: a failed write is logged but not propagated.
|
|
103
|
+
// Keep the lock fresh. The merge might run longer than FILE_LOCK_TTL_MS; without periodic re-stamping, another process would see the lock as stale and acquire it. Returns a stop function the caller calls in `finally`. Heartbeat is best-effort: a failed write is logged but not propagated.
|
|
123
104
|
export function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void {
|
|
124
105
|
const interval = setInterval(() => {
|
|
125
106
|
void writeMergeFileLock(storage, holderId, Date.now()).catch(() => { /* best-effort */ });
|
|
@@ -1,18 +1,11 @@
|
|
|
1
1
|
// Deletion markers: how a merge retires the input files it consumed WITHOUT deleting them inline.
|
|
2
|
-
//
|
|
3
|
-
// The old approach deleted consumed inputs on a 15-min timer. If the tab closed before the timer fired
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// EXCLUDED from reads the moment the marker exists (so they're never re-merged), and their physical
|
|
7
|
-
// deletion is driven off the marker on a later read instead of an in-process timer.
|
|
8
|
-
//
|
|
9
|
-
// A marker names the input files it wants gone (`deleteFiles`) and the output files that supersede them
|
|
10
|
-
// (`replacedBy`). On every read we:
|
|
2
|
+
//
|
|
3
|
+
// The old approach deleted consumed inputs on a 15-min timer. If the tab closed before the timer fired the inputs survived AND were still read — so the next merge re-merged the same duplicates, forever (the "1739 files, 155GB, 99% duplicate" loop). Markers fix both halves: the consumed inputs are EXCLUDED from reads the moment the marker exists (so they're never re-merged), and their physical deletion is driven off the marker on a later read instead of an in-process timer.
|
|
4
|
+
//
|
|
5
|
+
// A marker names the input files it wants gone (`deleteFiles`) and the output files that supersede them (`replacedBy`). On every read we:
|
|
11
6
|
// 1) exclude every `deleteFiles` entry from the active file set (unconditionally — that's the loop fix);
|
|
12
|
-
// 2) delete a marker's `deleteFiles` once its `replacedBy` outputs all exist OR the marker is older than
|
|
13
|
-
//
|
|
14
|
-
// 3) once a marker's `deleteFiles` are all gone, and the marker has aged past the grace window, delete
|
|
15
|
-
// the marker itself (kept around briefly so a concurrent reader still excludes the files consistently).
|
|
7
|
+
// 2) delete a marker's `deleteFiles` once its `replacedBy` outputs all exist OR the marker is older than the grace window (a crash-safety floor so a marker whose outputs never landed still gets cleaned);
|
|
8
|
+
// 3) once a marker's `deleteFiles` are all gone, and the marker has aged past the grace window, delete the marker itself (kept around briefly so a concurrent reader still excludes the files consistently).
|
|
16
9
|
|
|
17
10
|
import type { FileStorage } from "../FileFolderAPI";
|
|
18
11
|
import { blue, magenta } from "socket-function/src/formatting/logColors";
|
|
@@ -66,8 +59,7 @@ export function markerExclusions(markers: DeleteMarker[]): Set<string> {
|
|
|
66
59
|
return excluded;
|
|
67
60
|
}
|
|
68
61
|
|
|
69
|
-
// Act on each marker (see file header). Idempotent and best-effort: every removal is guarded, so two
|
|
70
|
-
// processes (or two reads) running this at once just race to the same already-correct end state.
|
|
62
|
+
// Act on each marker (see file header). Idempotent and best-effort: every removal is guarded, so two processes (or two reads) running this at once just race to the same already-correct end state.
|
|
71
63
|
export async function processDeleteMarkers(name: string, storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void> {
|
|
72
64
|
const present = new Set(allNames);
|
|
73
65
|
const now = Date.now();
|
|
@@ -1,16 +1,11 @@
|
|
|
1
1
|
import cborx from "cbor-x";
|
|
2
2
|
import { ABSENT, BaseBulkDatabaseReader, ColumnIndex, EMPTY_BUFFER, encodeValue, RawCell, TYPE_ABSENT_TAG } from "./BulkDatabaseFormat";
|
|
3
3
|
|
|
4
|
-
// Tier-0 streaming format: an append log of whole-row writes and deletes (row-format, not columnar),
|
|
5
|
-
//
|
|
6
|
-
//
|
|
4
|
+
// Tier-0 streaming format: an append log of whole-row writes and deletes (row-format, not columnar), so small mutations are a single cheap append instead of rewriting a columnar file. Each block is:
|
|
5
|
+
//
|
|
7
6
|
// [u32 len][CBOR({ t, v }) or CBOR({ t, d })][u32 len]
|
|
8
|
-
//
|
|
9
|
-
// `t` is a per-write unique timestamp (getTimeUnique); `v` is a set row; `d` is a deleted key. Because
|
|
10
|
-
// every thread streams to its own file, the only way to recover the global mutation order — needed
|
|
11
|
-
// for newest-wins — is a per-write timestamp, with ties broken by file name. The trailing length must
|
|
12
|
-
// match the leading one; a torn/incomplete append leaves a mismatched or missing suffix, so we stop
|
|
13
|
-
// there and report trailing bad bytes instead of throwing. structuredClone preserves typed arrays.
|
|
7
|
+
//
|
|
8
|
+
// `t` is a per-write unique timestamp (getTimeUnique); `v` is a set row; `d` is a deleted key. Because every thread streams to its own file, the only way to recover the global mutation order — needed for newest-wins — is a per-write timestamp, with ties broken by file name. The trailing length must match the leading one; a torn/incomplete append leaves a mismatched or missing suffix, so we stop there and report trailing bad bytes instead of throwing. structuredClone preserves typed arrays.
|
|
14
9
|
|
|
15
10
|
export const STREAM_EXTENSION = ".stream";
|
|
16
11
|
|
|
@@ -24,13 +19,9 @@ function frame(payload: Buffer): Buffer {
|
|
|
24
19
|
return Buffer.concat([len, payload, len]);
|
|
25
20
|
}
|
|
26
21
|
|
|
27
|
-
// Framing only — no batching here. BulkDatabase2 coalesces and flushes these framed bytes on a ramping
|
|
28
|
-
// per-collection schedule (see streamAppend in BulkDatabaseBase), because the browser File System Access
|
|
29
|
-
// API rewrites the whole file on every append, so one-append-per-write is quadratic. The first write
|
|
30
|
-
// after a lull still flushes immediately, so a single action then a tab close is saved at once.
|
|
22
|
+
// Framing only — no batching here. BulkDatabase2 coalesces and flushes these framed bytes on a ramping per-collection schedule (see streamAppend in BulkDatabaseBase), because the browser File System Access API rewrites the whole file on every append, so one-append-per-write is quadratic. The first write after a lull still flushes immediately, so a single action then a tab close is saved at once.
|
|
31
23
|
|
|
32
|
-
// Times are assigned by the caller (BulkDatabase2) so the exact same timestamp lands on disk, in the
|
|
33
|
-
// in-memory overlay, and in the cross-tab broadcast — keeping the global write order consistent.
|
|
24
|
+
// Times are assigned by the caller (BulkDatabase2) so the exact same timestamp lands on disk, in the in-memory overlay, and in the cross-tab broadcast — keeping the global write order consistent.
|
|
34
25
|
export function frameRows(entries: { time: number; row: Record<string, unknown> }[]): Buffer {
|
|
35
26
|
return Buffer.concat(entries.map(e => frame(Buffer.from(cborEncoder.encode({ t: e.time, v: e.row })))));
|
|
36
27
|
}
|
|
@@ -61,12 +52,7 @@ export function parseStream(buffer: Buffer): { entries: StreamEntry[]; badBytes:
|
|
|
61
52
|
return { entries, badBytes: buffer.length - pos };
|
|
62
53
|
}
|
|
63
54
|
|
|
64
|
-
// Wraps streamed entries (already ordered oldest-first) as a BaseBulkDatabaseReader. Each set MERGES
|
|
65
|
-
// its fields onto the key's current row (so a partial write/update only changes the columns it
|
|
66
|
-
// includes); a delete tombstones the key (exposed via deletedKeys so the join suppresses it in older
|
|
67
|
-
// bulk readers) and resets the merge. A column the merged row never set reads as ABSENT, so the join
|
|
68
|
-
// falls through to older readers for it. Also returns the latest timestamp seen per key (live or
|
|
69
|
-
// deleted), used for cross-tab conflict resolution.
|
|
55
|
+
// Wraps streamed entries (already ordered oldest-first) as a BaseBulkDatabaseReader. Each set MERGES its fields onto the key's current row (so a partial write/update only changes the columns it includes); a delete tombstones the key (exposed via deletedKeys so the join suppresses it in older bulk readers) and resets the merge. A column the merged row never set reads as ABSENT, so the join falls through to older readers for it. Also returns the latest timestamp seen per key (live or deleted), used for cross-tab conflict resolution.
|
|
70
56
|
export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: number): { reader: BaseBulkDatabaseReader; times: Map<string, number> } {
|
|
71
57
|
let byKey = new Map<string, Record<string, unknown>>();
|
|
72
58
|
let deletedKeys = new Set<string>();
|
|
@@ -97,8 +83,7 @@ export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: numb
|
|
|
97
83
|
// Per-key tombstone times for the join's delete resolution.
|
|
98
84
|
let deleteTimes = new Map<string, number>();
|
|
99
85
|
for (let key of deletedKeys) deleteTimes.set(key, times.get(key) || 0);
|
|
100
|
-
// Iterate to find min/max — NOT Math.min(...times)/Math.max(...times): spreading a large array as call
|
|
101
|
-
// arguments overflows the stack ("Maximum call stack size exceeded") once a stream has many keys.
|
|
86
|
+
// Iterate to find min/max — NOT Math.min(...times)/Math.max(...times): spreading a large array as call arguments overflows the stack ("Maximum call stack size exceeded") once a stream has many keys.
|
|
102
87
|
let minTime = Infinity, maxTime = -Infinity;
|
|
103
88
|
for (let t of times.values()) {
|
|
104
89
|
if (t < minTime) minTime = t;
|
|
@@ -115,8 +100,7 @@ export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: numb
|
|
|
115
100
|
keyTimes: new Map(keys.map(key => [key, times.get(key) || 0])),
|
|
116
101
|
columns,
|
|
117
102
|
deleteTimes,
|
|
118
|
-
// A key's time is its latest write across all columns (per-key, not per-column). For a column
|
|
119
|
-
// it never set we return ABSENT so the join falls through to an older reader.
|
|
103
|
+
// A key's time is its latest write across all columns (per-key, not per-column). For a column it never set we return ABSENT so the join falls through to an older reader.
|
|
120
104
|
async getColumn(column) {
|
|
121
105
|
return keys.map(key => {
|
|
122
106
|
let row = byKey.get(key);
|
|
@@ -128,9 +112,7 @@ export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: numb
|
|
|
128
112
|
if (!row || !(column in row)) return ABSENT;
|
|
129
113
|
return { value: row[column], time: times.get(key) || 0 };
|
|
130
114
|
},
|
|
131
|
-
// Stream values are decoded objects (CBOR), so unlike a bulk file we have to encode them to raw
|
|
132
|
-
// cells here. Stream data is small (tier-0, size-capped), so this is cheap. A column the row never
|
|
133
|
-
// set is omitted (ABSENT → fall through); a stored undefined is kept (a real clear).
|
|
115
|
+
// Stream values are decoded objects (CBOR), so unlike a bulk file we have to encode them to raw cells here. Stream data is small (tier-0, size-capped), so this is cheap. A column the row never set is omitted (ABSENT → fall through); a stored undefined is kept (a real clear).
|
|
134
116
|
async getRawColumn(column) {
|
|
135
117
|
let map = new Map<string, RawCell>();
|
|
136
118
|
for (let key of keys) {
|
|
@@ -140,10 +122,7 @@ export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: numb
|
|
|
140
122
|
}
|
|
141
123
|
return map;
|
|
142
124
|
},
|
|
143
|
-
// Per-column index in the same shape a bulk file exposes, materialized in memory once (stream data
|
|
144
|
-
// is tier-0, size-capped, so the encode is cheap). Lets the planned merge treat stream + bulk
|
|
145
|
-
// sources uniformly: it loads each (source, column) index, uses offsets/types to plan the output's
|
|
146
|
-
// byte layout, and reads contiguous runs from this index's in-memory data buffer during execute.
|
|
125
|
+
// Per-column index in the same shape a bulk file exposes, materialized in memory once (stream data is tier-0, size-capped, so the encode is cheap). Lets the planned merge treat stream + bulk sources uniformly: it loads each (source, column) index, uses offsets/types to plan the output's byte layout, and reads contiguous runs from this index's in-memory data buffer during execute.
|
|
147
126
|
async getColumnIndex(column) {
|
|
148
127
|
return getStreamColumnIndex(keys, byKey, column);
|
|
149
128
|
},
|
|
@@ -154,18 +133,14 @@ export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: numb
|
|
|
154
133
|
return { reader, times };
|
|
155
134
|
}
|
|
156
135
|
|
|
157
|
-
// keys[] is the column-side row order for this stream reader; rowOfKey is used by the planner to look up
|
|
158
|
-
// a winning cell's source row without going through a column read. Stays consistent with getColumnIndex,
|
|
159
|
-
// which builds its offsets/types over the same `keys` array in the same order.
|
|
136
|
+
// keys[] is the column-side row order for this stream reader; rowOfKey is used by the planner to look up a winning cell's source row without going through a column read. Stays consistent with getColumnIndex, which builds its offsets/types over the same `keys` array in the same order.
|
|
160
137
|
function buildKeyIndex(keys: string[]): Map<string, number> {
|
|
161
138
|
const m = new Map<string, number>();
|
|
162
139
|
for (let i = 0; i < keys.length; i++) m.set(keys[i], i);
|
|
163
140
|
return m;
|
|
164
141
|
}
|
|
165
142
|
|
|
166
|
-
// Build a stream-side ColumnIndex (offsets + types + a contiguous-row byte slicer) over the in-memory
|
|
167
|
-
// merged rows. Encodes each row's value for this column once into a single backing buffer; readValueBytes
|
|
168
|
-
// then slices it. Cached per (reader, column) so a merge planning + execution pass doesn't re-encode.
|
|
143
|
+
// Build a stream-side ColumnIndex (offsets + types + a contiguous-row byte slicer) over the in-memory merged rows. Encodes each row's value for this column once into a single backing buffer; readValueBytes then slices it. Cached per (reader, column) so a merge planning + execution pass doesn't re-encode.
|
|
169
144
|
function getStreamColumnIndex(keys: string[], byKey: Map<string, Record<string, unknown>>, column: string): ColumnIndex {
|
|
170
145
|
let cached = columnIndexCache.get(byKey)?.get(column);
|
|
171
146
|
if (cached) return cached;
|
|
@@ -204,6 +179,5 @@ function getStreamColumnIndex(keys: string[], byKey: Map<string, Record<string,
|
|
|
204
179
|
return idx;
|
|
205
180
|
}
|
|
206
181
|
|
|
207
|
-
// Per stream-reader cache of its column indexes. Keyed by the reader's byKey map (a fresh
|
|
208
|
-
// streamReaderFromEntries call produces a fresh map, so this is effectively per-reader).
|
|
182
|
+
// Per stream-reader cache of its column indexes. Keyed by the reader's byKey map (a fresh streamReaderFromEntries call produces a fresh map, so this is effectively per-reader).
|
|
209
183
|
const columnIndexCache = new WeakMap<Map<string, Record<string, unknown>>, Map<string, ColumnIndex>>();
|
|
@@ -1,14 +1,8 @@
|
|
|
1
|
-
// Browser-side cross-tab write sync for BulkDatabase2, over BroadcastChannel (one channel per
|
|
2
|
-
// collection, same-origin tabs and workers). It does NOT persist anything — each side writes to disk
|
|
3
|
-
// itself; this just relays live writes to other open tabs/workers and, when one starts up, asks peers
|
|
4
|
-
// for writes they've made recently that may not be on disk yet. No-op in Node / where
|
|
5
|
-
// BroadcastChannel is unavailable, so BulkDatabase2 can call these unconditionally. This is an
|
|
6
|
-
// optional feature, so there is no fallback.
|
|
1
|
+
// Browser-side cross-tab write sync for BulkDatabase2, over BroadcastChannel (one channel per collection, same-origin tabs and workers). It does NOT persist anything — each side writes to disk itself; this just relays live writes to other open tabs/workers and, when one starts up, asks peers for writes they've made recently that may not be on disk yet. No-op in Node / where BroadcastChannel is unavailable, so BulkDatabase2 can call these unconditionally. This is an optional feature, so there is no fallback.
|
|
7
2
|
|
|
8
3
|
export type RemoteWrite = { key: string; time: number; deleted?: boolean; value?: unknown };
|
|
9
4
|
|
|
10
|
-
// Writes older than this are assumed already flushed to disk (a freshly-opened tab gets those by
|
|
11
|
-
// reading disk), so a tab only replays writes newer than this when a peer says hello.
|
|
5
|
+
// Writes older than this are assumed already flushed to disk (a freshly-opened tab gets those by reading disk), so a tab only replays writes newer than this when a peer says hello.
|
|
12
6
|
const RECENT_WINDOW_MS = 60_000;
|
|
13
7
|
|
|
14
8
|
type Channel = {
|
|
@@ -23,8 +17,7 @@ type Channel = {
|
|
|
23
17
|
const channels = new Map<string, Channel>();
|
|
24
18
|
|
|
25
19
|
export function isSyncSupported(): boolean {
|
|
26
|
-
// window (main-thread browser) or WorkerGlobalScope (Web/Service/Shared worker) — but not Node
|
|
27
|
-
// (typesafecss's isNode is `typeof window === "undefined"`, which misclassifies workers)
|
|
20
|
+
// window (main-thread browser) or WorkerGlobalScope (Web/Service/Shared worker) — but not Node (typesafecss's isNode is `typeof window === "undefined"`, which misclassifies workers)
|
|
28
21
|
let isBrowserOrWorker = typeof window !== "undefined"
|
|
29
22
|
|| "WorkerGlobalScope" in globalThis;
|
|
30
23
|
return isBrowserOrWorker && typeof BroadcastChannel !== "undefined";
|
|
@@ -43,8 +36,7 @@ function ensure(collection: string): Channel | undefined {
|
|
|
43
36
|
if (!isSyncSupported()) return undefined;
|
|
44
37
|
let channel = channels.get(collection);
|
|
45
38
|
if (channel) return channel;
|
|
46
|
-
// BroadcastChannel never delivers a message back to the instance that sent it, so a tab never
|
|
47
|
-
// hears its own writes — only the other open tabs do.
|
|
39
|
+
// BroadcastChannel never delivers a message back to the instance that sent it, so a tab never hears its own writes — only the other open tabs do.
|
|
48
40
|
const bc = new BroadcastChannel(`bulkDatabase2:${collection}`);
|
|
49
41
|
const created: Channel = { bc, subscribers: [], sealSubscribers: [], recent: [] };
|
|
50
42
|
bc.onmessage = (event: MessageEvent) => {
|
|
@@ -55,13 +47,10 @@ function ensure(collection: string): Channel | undefined {
|
|
|
55
47
|
} else if (msg.type === "recent" && msg.writes) {
|
|
56
48
|
for (const write of msg.writes) deliver(created, write);
|
|
57
49
|
} else if (msg.type === "seal") {
|
|
58
|
-
// A peer is about to fold recent data; stop appending to our current stream file so it
|
|
59
|
-
// becomes complete and the merge can fold it whole. Our next write starts a fresh file.
|
|
50
|
+
// A peer is about to fold recent data; stop appending to our current stream file so it becomes complete and the merge can fold it whole. Our next write starts a fresh file.
|
|
60
51
|
for (const sub of created.sealSubscribers) sub();
|
|
61
52
|
} else if (msg.type === "hello") {
|
|
62
|
-
// Another tab just started; replay our recent writes so it doesn't miss any. The reply goes
|
|
63
|
-
// to every tab, but peers that already have a write ignore it (its timestamp isn't newer
|
|
64
|
-
// than what they hold), so the redundant broadcast is harmless.
|
|
53
|
+
// Another tab just started; replay our recent writes so it doesn't miss any. The reply goes to every tab, but peers that already have a write ignore it (its timestamp isn't newer than what they hold), so the redundant broadcast is harmless.
|
|
65
54
|
pruneRecent(created);
|
|
66
55
|
if (created.recent.length) created.bc.postMessage({ type: "recent", writes: created.recent });
|
|
67
56
|
}
|
|
@@ -70,9 +59,7 @@ function ensure(collection: string): Channel | undefined {
|
|
|
70
59
|
return created;
|
|
71
60
|
}
|
|
72
61
|
|
|
73
|
-
// Subscribe to remote writes for a collection. Recent writes from already-open tabs arrive through the
|
|
74
|
-
// same onWrite callback (as the reply to our hello), so the returned array is always empty — it's kept
|
|
75
|
-
// only for API compatibility with callers that await it.
|
|
62
|
+
// Subscribe to remote writes for a collection. Recent writes from already-open tabs arrive through the same onWrite callback (as the reply to our hello), so the returned array is always empty — it's kept only for API compatibility with callers that await it.
|
|
76
63
|
export function connect(collection: string, onWrite: (write: RemoteWrite) => void, onSeal?: () => void): Promise<RemoteWrite[]> {
|
|
77
64
|
const channel = ensure(collection);
|
|
78
65
|
if (!channel) return Promise.resolve([]);
|
|
@@ -90,10 +77,7 @@ export function broadcast(collection: string, write: RemoteWrite): void {
|
|
|
90
77
|
channel.bc.postMessage({ type: "write", write });
|
|
91
78
|
}
|
|
92
79
|
|
|
93
|
-
// Ask every other open tab of this collection to seal (stop appending to) its current stream file, so
|
|
94
|
-
// a merge can fold recent data up to the present without racing an append. Best-effort: no-op in Node,
|
|
95
|
-
// and a peer that misses it just keeps appending — worst case the merge folds a prefix and the rest
|
|
96
|
-
// folds later (duplication, resolved by write-time), never data loss.
|
|
80
|
+
// Ask every other open tab of this collection to seal (stop appending to) its current stream file, so a merge can fold recent data up to the present without racing an append. Best-effort: no-op in Node, and a peer that misses it just keeps appending — worst case the merge folds a prefix and the rest folds later (duplication, resolved by write-time), never data loss.
|
|
97
81
|
export function broadcastSeal(collection: string): void {
|
|
98
82
|
const channel = ensure(collection);
|
|
99
83
|
if (!channel) return;
|