sliftutils 1.7.29 → 1.7.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/builders/extensionBuild.ts +1 -1
- package/bundler/bundleEntry.ts +1 -3
- package/bundler/bundler.ts +2 -4
- package/bundler/sourceMaps.ts +1 -2
- package/examplestorage/browser.tsx +1 -2
- package/examplestorage/exampleserver.ts +1 -2
- package/index.d.ts +106 -99
- package/misc/dist/environment.ts.cache +2 -2
- package/misc/dist/getSecret.ts.cache +2 -2
- package/misc/dist/strings.ts.cache +2 -2
- package/misc/dist/zip.ts.cache +2 -2
- package/misc/https/dist/certs.ts.cache +2 -2
- package/misc/https/dist/dns.ts.cache +7 -16
- package/misc/https/dist/persistentLocalStorage.ts.cache +2 -2
- package/misc/https/dns.d.ts +1 -4
- package/misc/https/dns.ts +5 -14
- package/misc/https/hostServer.d.ts +2 -0
- package/misc/https/hostServer.ts +6 -7
- package/misc/https/httpsCerts.ts +4 -10
- package/misc/openrouter.ts +1 -2
- package/package.json +1 -1
- package/render-utils/FullscreenModal.tsx +1 -4
- package/render-utils/GenericFormat.tsx +2 -8
- package/render-utils/Input.tsx +5 -13
- package/render-utils/InputLabel.tsx +4 -6
- package/render-utils/InputPicker.tsx +1 -2
- package/render-utils/URLParam.ts +2 -3
- package/render-utils/dist/observer.tsx.cache +2 -2
- package/render-utils/mobxTyped.ts +2 -3
- package/render-utils/niceStringify.ts +2 -4
- package/storage/ArchivesDisk.ts +4 -14
- package/storage/BulkDatabase2/BulkDatabase2.d.ts +1 -2
- package/storage/BulkDatabase2/BulkDatabase2.ts +4 -11
- package/storage/BulkDatabase2/BulkDatabaseBase.ts +42 -114
- package/storage/BulkDatabase2/BulkDatabaseFormat.ts +31 -91
- package/storage/BulkDatabase2/BulkDatabaseMerge.ts +19 -57
- package/storage/BulkDatabase2/BulkDatabaseReader.ts +4 -12
- package/storage/BulkDatabase2/LoadedIndex.ts +6 -18
- package/storage/BulkDatabase2/WriteOverlay.ts +3 -8
- package/storage/BulkDatabase2/blockCache.ts +12 -28
- package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +40 -103
- package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +19 -50
- package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +18 -50
- package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +7 -15
- package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +9 -21
- package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +6 -11
- package/storage/BulkDatabase2/dist/blockCache.ts.cache +15 -31
- package/storage/BulkDatabase2/dist/mergeLock.ts.cache +10 -29
- package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +10 -18
- package/storage/BulkDatabase2/dist/streamLog.ts.cache +17 -43
- package/storage/BulkDatabase2/dist/syncClient.ts.cache +11 -27
- package/storage/BulkDatabase2/mergeLock.ts +7 -26
- package/storage/BulkDatabase2/mergeMarkers.ts +7 -15
- package/storage/BulkDatabase2/streamLog.ts +14 -40
- package/storage/BulkDatabase2/syncClient.ts +8 -24
- package/storage/FileFolderAPI.tsx +28 -75
- package/storage/IArchives.d.ts +3 -3
- package/storage/IArchives.ts +21 -50
- package/storage/IStorage.ts +2 -4
- package/storage/PendingManager.tsx +2 -3
- package/storage/TransactionStorage.ts +11 -24
- package/storage/backblaze.ts +12 -43
- package/storage/dist/ArchivesDisk.ts.cache +7 -17
- package/storage/dist/FileFolderAPI.tsx.cache +25 -63
- package/storage/dist/IArchives.ts.cache +10 -20
- package/storage/dist/IndexedDBFileFolderAPI.ts.cache +2 -2
- package/storage/dist/JSONStorage.ts.cache +2 -2
- package/storage/dist/PendingManager.tsx.cache +5 -6
- package/storage/dist/TransactionStorage.ts.cache +14 -27
- package/storage/dist/backblaze.ts.cache +15 -46
- package/storage/dist/embeddingFormats.ts.cache +11 -24
- package/storage/dist/fileSystemPointer.ts.cache +4 -7
- package/storage/dist/remoteFileStorage.ts.cache +17 -36
- package/storage/embeddingFormats.ts +11 -27
- package/storage/fileSystemPointer.ts +2 -7
- package/storage/proxydatabase/dist/ivfEmbeddingDatabase.ts.cache +8 -18
- package/storage/proxydatabase/dist/transactionSet.ts.cache +4 -7
- package/storage/proxydatabase/inMemoryDatabase.ts +1 -3
- package/storage/proxydatabase/ivfEmbeddingDatabase.ts +8 -23
- package/storage/proxydatabase/transactionSet.ts +2 -5
- package/storage/remoteFileServer.ts +20 -48
- package/storage/remoteFileStorage.ts +14 -33
- package/storage/remoteStorage/ArchivesRemote.d.ts +1 -4
- package/storage/remoteStorage/ArchivesRemote.ts +11 -26
- package/storage/remoteStorage/ArchivesUrl.ts +2 -4
- package/storage/remoteStorage/accessPage.tsx +1 -5
- package/storage/remoteStorage/blobStore.d.ts +5 -13
- package/storage/remoteStorage/blobStore.ts +52 -130
- package/storage/remoteStorage/createArchives.d.ts +9 -13
- package/storage/remoteStorage/createArchives.ts +43 -187
- package/storage/remoteStorage/deployTakeover.d.ts +18 -32
- package/storage/remoteStorage/deployTakeover.ts +70 -384
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +12 -24
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +5 -7
- package/storage/remoteStorage/dist/blobStore.ts.cache +47 -117
- package/storage/remoteStorage/dist/cliArgs.ts.cache +2 -2
- package/storage/remoteStorage/dist/createArchives.ts.cache +54 -186
- package/storage/remoteStorage/dist/deployTakeover.ts.cache +81 -387
- package/storage/remoteStorage/dist/intermediateSources.ts.cache +119 -0
- package/storage/remoteStorage/dist/remoteConfig.ts.cache +36 -24
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +20 -40
- package/storage/remoteStorage/dist/storageClientController.ts.cache +32 -0
- package/storage/remoteStorage/dist/storageController.ts.cache +30 -64
- package/storage/remoteStorage/dist/storageServerState.ts.cache +269 -186
- package/storage/remoteStorage/grantAccessCli.ts +1 -3
- package/storage/remoteStorage/intermediateSources.d.ts +21 -0
- package/storage/remoteStorage/intermediateSources.ts +117 -0
- package/storage/remoteStorage/remoteConfig.ts +33 -20
- package/storage/remoteStorage/sourceWrapper.d.ts +8 -19
- package/storage/remoteStorage/sourceWrapper.ts +19 -41
- package/storage/remoteStorage/storageClientController.ts +1 -3
- package/storage/remoteStorage/storageController.d.ts +5 -3
- package/storage/remoteStorage/storageController.ts +12 -83
- package/storage/remoteStorage/storageServer.ts +20 -34
- package/storage/remoteStorage/storageServerState.d.ts +28 -6
- package/storage/remoteStorage/storageServerState.ts +275 -221
|
@@ -123,8 +123,7 @@ export class InputPicker<T> extends preact.Component<{
|
|
|
123
123
|
<button
|
|
124
124
|
key={`add-${option.matchText}`}
|
|
125
125
|
className={css.hbox(5).button + greenButton}
|
|
126
|
-
// On mouse down, so we can add picked BEFORE we blur (otherwise
|
|
127
|
-
// this button disappears before it can be clicked)
|
|
126
|
+
// On mouse down, so we can add picked BEFORE we blur (otherwise this button disappears before it can be clicked)
|
|
128
127
|
onMouseDown={() => {
|
|
129
128
|
this.props.addPicked(option.value);
|
|
130
129
|
}}
|
package/render-utils/URLParam.ts
CHANGED
|
@@ -84,9 +84,8 @@ export function getCurrentUrl() {
|
|
|
84
84
|
let currentBatchedUrl: string | undefined;
|
|
85
85
|
function throttledUrlPush(url: string) {
|
|
86
86
|
history.pushState({}, "", url);
|
|
87
|
-
//currentBatchedUrl = url;
|
|
88
|
-
// NOTE: Stopped throttling, so when you click on links, it immediately updates the selected state.
|
|
89
|
-
//void throttledUrlPushBase(url);
|
|
87
|
+
// currentBatchedUrl = url;
|
|
88
|
+
// NOTE: Stopped throttling, so when you click on links, it immediately updates the selected state. void throttledUrlPushBase(url);
|
|
90
89
|
}
|
|
91
90
|
const throttledUrlPushBase = throttleFunction(1000, (url: string) => {
|
|
92
91
|
currentBatchedUrl = undefined;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"use strict";
|
|
1
|
+
"use strict"; var console = (function shimConsole() { let baseConsole = globalThis.console; if (typeof window !== "undefined") return baseConsole; let name = __filename.replaceAll("\\", "/").split("/").pop(); let dirName = __dirname.replaceAll("\\", "/").split("/").pop(); let context = { __FILE__: __filename, __DIR__: __dirname, __NAME__: name, __DIRNAME__: dirName }; let newConsole = { ...baseConsole }; for (let shim of ["log", "info", "warn", "error"]) { newConsole[shim] = (...args) => { args.push(context); baseConsole[shim](...args); }; } newConsole.__context = context; return newConsole; })(); "use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
|
|
3
3
|
//exports.observer = void 0;
|
|
4
4
|
module.allowclient = true;
|
|
@@ -34,4 +34,4 @@ function observer(Constructor) {
|
|
|
34
34
|
}
|
|
35
35
|
exports.observer = observer;
|
|
36
36
|
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib2JzZXJ2ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJvYnNlcnZlci50c3giXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBQUEsTUFBTSxDQUFDLFdBQVcsR0FBRyxJQUFJLENBQUM7QUFHMUIsK0JBQTRDO0FBQzVDLG1FQUFxRTtBQUVyRSxJQUFJLG9CQUFvQixHQUFHLENBQUMsQ0FBQztBQUM3QixTQUFnQixRQUFRLENBU3BCLFdBQWM7SUFFZCxJQUFJLElBQUksR0FBRyxXQUFXLENBQUMsSUFBSSxDQUFDO0lBQzVCLE9BQU8sS0FBTSxTQUFRLFdBQVc7UUFBekI7O1lBR0gsYUFBUSxHQUFHLElBQUksZUFBUSxDQUFDLFVBQVUsSUFBSSxJQUFJLG9CQUFvQixFQUFFLEVBQUUsRUFBRSxHQUFHLEVBQUU7Z0JBQ3JFLEtBQUssQ0FBQyxXQUFXLEVBQUUsQ0FBQztZQUN4QixDQUFDLENBQUMsQ0FBQztRQWNQLENBQUM7UUFsQkcsYUFBYTtRQUNiLE1BQU0sS0FBSyxJQUFJLEtBQUssT0FBTyxXQUFXLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQztRQUk5QyxvQkFBb0I7O1lBQ2hCLElBQUksQ0FBQyxRQUFRLENBQUMsT0FBTyxFQUFFLENBQUM7WUFDeEIsTUFBQSxLQUFLLENBQUMsb0JBQW9CLG9EQUFJLENBQUM7UUFDbkMsQ0FBQztRQUNELE1BQU0sQ0FBQyxHQUFHLElBQVc7WUFDakIsSUFBSSxNQUE2QixDQUFDO1lBQ2xDLElBQUksQ0FBQyxRQUFRLENBQUMsS0FBSyxDQUFDLEdBQUcsRUFBRTtnQkFDckIsSUFBQSxzQkFBWSxFQUFDLEdBQUcsRUFBRTtvQkFDZCxNQUFNLEdBQUksS0FBSyxDQUFDLE1BQWMsQ0FBQyxHQUFHLElBQUksQ0FBQyxDQUFDO2dCQUM1QyxDQUFDLEVBQUUsR0FBRyxJQUFJLFNBQVMsQ0FBQyxDQUFDO1lBQ3pCLENBQUMsQ0FBQyxDQUFDO1lBQ0gsT0FBTyxNQUFNLENBQUM7UUFDbEIsQ0FBQztLQUNKLENBQUM7QUFDTixDQUFDO0FBaENELDRCQWdDQyIsInNvdXJjZXNDb250ZW50IjpbIm1vZHVsZS5hbGxvd2NsaWVudCA9IHRydWU7XG5cbmltcG9ydCAqIGFzIHByZWFjdCBmcm9tIFwicHJlYWN0XCI7XG5pbXBvcnQgeyBvYnNlcnZhYmxlLCBSZWFjdGlvbiB9IGZyb20gXCJtb2J4XCI7XG5pbXBvcnQgeyBtZWFzdXJlQmxvY2sgfSBmcm9tIFwic29ja2V0LWZ1bmN0aW9uL3NyYy9wcm9maWxpbmcvbWVhc3VyZVwiO1xuXG5sZXQgZ2xvYmFsQ29uc3RydWN0T3JkZXIgPSAxO1xuZXhwb3J0IGZ1bmN0aW9uIG9ic2VydmVyPFxuICAgIFQgZXh0ZW5kcyB7XG4gICAgICAgIG5ldyguLi5hcmdzOiBhbnlbXSk6IHtcbiAgICAgICAgICAgIHJlbmRlcigpOiBwcmVhY3QuQ29tcG9uZW50Q2hpbGQ7XG4gICAgICAgICAgICBmb3JjZVVwZGF0ZShjYWxsYmFjaz86ICgpID0+IHZvaWQpOiB2b2lkO1xuICAgICAgICAgICAgY29tcG9uZW50V2lsbFVubW91bnQ/KCk6IHZvaWQ7XG4gICAgICAgIH1cbiAgICB9XG4+KFxuICAgIENvbnN0cnVjdG9yOiBUXG4pIHtcbiAgICBsZXQgbmFtZSA9IENvbnN0cnVjdG9yLm5hbWU7XG4gICAgcmV0dXJuIGNsYXNzIGV4dGVuZHMgQ29uc3RydWN0b3Ige1xuICAgICAgICAvLyBAdHMtaWdub3JlXG4gICAgICAgIHN0YXRpYyBnZXQgbmFtZSgpIHsgcmV0dXJuIENvbnN0cnVjdG9yLm5hbWU7IH1cbiAgICAgICAgcmVhY3Rpb24gPSBuZXcgUmVhY3Rpb24oYHJlbmRlci4ke25hbWV9LiR7Z2xvYmFsQ29uc3RydWN0T3JkZXIrK31gLCAoKSA9PiB7XG4gICAgICAgICAgICBzdXBlci5mb3JjZVVwZGF0ZSgpO1xuICAgICAgICB9KTtcbiAgICAgICAgY29tcG9uZW50V2lsbFVubW91bnQoKSB7XG4gICAgICAgICAgICB0aGlzLnJlYWN0aW9uLmRpc3Bvc2UoKTtcbiAgICAgICAgICAgIHN1cGVyLmNvbXBvbmVudFdpbGxVbm1vdW50Py4oKTtcbiAgICAgICAgfVxuICAgICAgICByZW5kZXIoLi4uYXJnczogYW55W10pIHtcbiAgICAgICAgICAgIGxldCBvdXRwdXQ6IHByZWFjdC5Db21wb25lbnRDaGlsZDtcbiAgICAgICAgICAgIHRoaXMucmVhY3Rpb24udHJhY2soKCkgPT4ge1xuICAgICAgICAgICAgICAgIG1lYXN1cmVCbG9jaygoKSA9PiB7XG4gICAgICAgICAgICAgICAgICAgIG91dHB1dCA9IChzdXBlci5yZW5kZXIgYXMgYW55KSguLi5hcmdzKTtcbiAgICAgICAgICAgICAgICB9LCBgJHtuYW1lfXxyZW5kZXJgKTtcbiAgICAgICAgICAgIH0pO1xuICAgICAgICAgICAgcmV0dXJuIG91dHB1dDtcbiAgICAgICAgfVxuICAgIH07XG59Il19
|
|
37
|
-
/* _JS_SOURCE_HASH = "
|
|
37
|
+
/* _JS_SOURCE_HASH = "87290663d59bacd34ed8b38a01f2dd7e2297988b42c431ea6651a529aa2e2527"; */
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import * as mobx from "mobx";
|
|
2
2
|
import { batchFunction } from "socket-function/src/batching";
|
|
3
3
|
|
|
4
|
-
// Re-export the core mobx primitives so downstream packages (which may have their own duplicate mobx
|
|
5
|
-
// in node_modules) can share THIS mobx instance, otherwise reactivity doesn't cross package boundaries.
|
|
4
|
+
// Re-export the core mobx primitives so downstream packages (which may have their own duplicate mobx in node_modules) can share THIS mobx instance, otherwise reactivity doesn't cross package boundaries.
|
|
6
5
|
export { observable, runInAction, computed, autorun, onBecomeObserved, onBecomeUnobserved } from "mobx";
|
|
7
6
|
export function configureMobxNextFrameScheduler() {
|
|
8
7
|
// NOTE: This makes a big difference if we do await calls in a loop which mutates observable state. BUT... we should probably just do those await calls before the loop?
|
|
@@ -10,7 +9,7 @@ export function configureMobxNextFrameScheduler() {
|
|
|
10
9
|
delay: 16,
|
|
11
10
|
name: "reactionScheduler",
|
|
12
11
|
}, (callbacks: (() => void)[]) => {
|
|
13
|
-
//console.log(`Triggering ${callbacks.length} reactions`);
|
|
12
|
+
// console.log(`Triggering ${callbacks.length} reactions`);
|
|
14
13
|
for (let callback of callbacks) {
|
|
15
14
|
callback();
|
|
16
15
|
}
|
|
@@ -8,8 +8,7 @@ export const niceStringifyNan = `{NaN}`;
|
|
|
8
8
|
export const niceStringifyUndefined = `{Undefined}`;
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
// BUG: This is actually broken for hex strings. Hex strings may sometimes be entirely numbers,
|
|
12
|
-
// which means they will randomly change type.
|
|
11
|
+
// BUG: This is actually broken for hex strings. Hex strings may sometimes be entirely numbers, which means they will randomly change type.
|
|
13
12
|
function looksLikeJSON(str: string) {
|
|
14
13
|
return (
|
|
15
14
|
str === "null"
|
|
@@ -32,8 +31,7 @@ export function niceStringify(value: unknown): string {
|
|
|
32
31
|
if (value === true) return niceStringifyTrue;
|
|
33
32
|
if (Number.isNaN(value)) return niceStringifyNan;
|
|
34
33
|
|
|
35
|
-
// Any strings that don't look like JSON, don't need to encoded as JSON, and can instead
|
|
36
|
-
// just be stored as strings.
|
|
34
|
+
// Any strings that don't look like JSON, don't need to encoded as JSON, and can instead just be stored as strings.
|
|
37
35
|
if (typeof value === "string" && !looksLikeJSON(value)) {
|
|
38
36
|
return value;
|
|
39
37
|
}
|
package/storage/ArchivesDisk.ts
CHANGED
|
@@ -5,11 +5,7 @@ import { runInfinitePoll } from "socket-function/src/batching";
|
|
|
5
5
|
import { sort, binarySearchBasic } from "socket-function/src/misc";
|
|
6
6
|
import { IArchives, ArchiveFileInfo, ArchivesConfig, assertValidLastModified } from "./IArchives";
|
|
7
7
|
|
|
8
|
-
// The base file-system IArchives: storage is one-to-one with the file system, every key is exactly
|
|
9
|
-
// one real file under <folder>/files, so the file system itself is the index. File handles are
|
|
10
|
-
// cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in
|
|
11
|
-
// serial, so they can't collide with each other or with handle closing. Used as the disk
|
|
12
|
-
// synchronization source of BlobStore (see remoteStorage/blobStore.ts).
|
|
8
|
+
// The base file-system IArchives: storage is one-to-one with the file system, every key is exactly one real file under <folder>/files, so the file system itself is the index. File handles are cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in serial, so they can't collide with each other or with handle closing. Used as the disk synchronization source of BlobStore (see remoteStorage/blobStore.ts).
|
|
13
9
|
|
|
14
10
|
const HANDLE_IDLE_TIMEOUT = 1000 * 60;
|
|
15
11
|
const HANDLE_SWEEP_INTERVAL = 1000 * 15;
|
|
@@ -20,11 +16,7 @@ type HandleEntry = {
|
|
|
20
16
|
lastUse: number;
|
|
21
17
|
};
|
|
22
18
|
|
|
23
|
-
// Caches open file handles, closing them once idle for HANDLE_IDLE_TIMEOUT. Instead of one
|
|
24
|
-
// setTimeout per handle, a list sorted by last use is swept periodically (entries are moved via
|
|
25
|
-
// binary search on access, and lastUse values only increase, so touched entries append at the
|
|
26
|
-
// end). Also serializes operations per file: each operation only starts once the previous one on
|
|
27
|
-
// the same file finished, and a handle is never closed while an operation is pending on it.
|
|
19
|
+
// Caches open file handles, closing them once idle for HANDLE_IDLE_TIMEOUT. Instead of one setTimeout per handle, a list sorted by last use is swept periodically (entries are moved via binary search on access, and lastUse values only increase, so touched entries append at the end). Also serializes operations per file: each operation only starts once the previous one on the same file finished, and a handle is never closed while an operation is pending on it.
|
|
28
20
|
class FileHandleCache {
|
|
29
21
|
private entries = new Map<string, HandleEntry>();
|
|
30
22
|
// Sorted by lastUse ascending (least recently used first)
|
|
@@ -278,8 +270,7 @@ export class ArchivesDisk implements IArchives {
|
|
|
278
270
|
}
|
|
279
271
|
}
|
|
280
272
|
|
|
281
|
-
// Large files stream into their own file under uploads/, then move into place on finish. No
|
|
282
|
-
// handle is opened until the first append actually happens.
|
|
273
|
+
// Large files stream into their own file under uploads/, then move into place on finish. No handle is opened until the first append actually happens.
|
|
283
274
|
public async startLargeUpload(): Promise<string> {
|
|
284
275
|
await this.init();
|
|
285
276
|
let id = `${Date.now()}_${this.nextLargeUploadId++}`;
|
|
@@ -344,8 +335,7 @@ async function statOrUndefined(filePath: string): Promise<fs.Stats | undefined>
|
|
|
344
335
|
}
|
|
345
336
|
}
|
|
346
337
|
|
|
347
|
-
// The folders/shallow post-processing shared by findInfo implementations that list flat files
|
|
348
|
-
// (used by ArchivesDisk on the raw disk walk, and by BlobStore on its index).
|
|
338
|
+
// The folders/shallow post-processing shared by findInfo implementations that list flat files (used by ArchivesDisk on the raw disk walk, and by BlobStore on its index).
|
|
349
339
|
export function applyFindInfoShape(files: ArchiveFileInfo[], prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): ArchiveFileInfo[] {
|
|
350
340
|
if (config?.type === "folders") {
|
|
351
341
|
let folders = new Map<string, ArchiveFileInfo>();
|
|
@@ -145,8 +145,7 @@ export interface IBulkDatabase2<T extends {
|
|
|
145
145
|
* "someone else is already merging".
|
|
146
146
|
*/
|
|
147
147
|
tryMergeNow(): Promise<MergeAttemptResult>;
|
|
148
|
-
/** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
|
|
149
|
-
* most callers want compact() or tryMergeNow(). */
|
|
148
|
+
/** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level; most callers want compact() or tryMergeNow(). */
|
|
150
149
|
merge(timeLo: number, timeHi: number): Promise<void>;
|
|
151
150
|
}
|
|
152
151
|
export declare class MobxReactiveDeps implements ReactiveDeps {
|
|
@@ -139,19 +139,14 @@ export interface IBulkDatabase2<T extends { key: string }> {
|
|
|
139
139
|
*/
|
|
140
140
|
tryMergeNow(): Promise<MergeAttemptResult>;
|
|
141
141
|
|
|
142
|
-
/** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
|
|
143
|
-
* most callers want compact() or tryMergeNow(). */
|
|
142
|
+
/** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level; most callers want compact() or tryMergeNow(). */
|
|
144
143
|
merge(timeLo: number, timeHi: number): Promise<void>;
|
|
145
144
|
}
|
|
146
145
|
|
|
147
|
-
// mobx-backed reactivity: each signal string gets its own observable box. observe() reads it (so an
|
|
148
|
-
// observer/autorun that calls it tracks that box) and invalidate() bumps it (so those reactions re-run).
|
|
149
|
-
// This reproduces the fine-grained per-key + load-version reactivity the class had when it used
|
|
150
|
-
// observable.map/observable.box directly, while keeping all that logic in the mobx-free base.
|
|
146
|
+
// mobx-backed reactivity: each signal string gets its own observable box. observe() reads it (so an observer/autorun that calls it tracks that box) and invalidate() bumps it (so those reactions re-run). This reproduces the fine-grained per-key + load-version reactivity the class had when it used observable.map/observable.box directly, while keeping all that logic in the mobx-free base.
|
|
151
147
|
export class MobxReactiveDeps implements ReactiveDeps {
|
|
152
148
|
private boxes = new Map<string, { get(): number; set(value: number): void }>();
|
|
153
|
-
// Signals that currently have at least one observer, kept current via mobx's onBecomeObserved/
|
|
154
|
-
// onBecomeUnobserved hooks. Lets isObserved() answer "is anything watching this row" cheaply.
|
|
149
|
+
// Signals that currently have at least one observer, kept current via mobx's onBecomeObserved/ onBecomeUnobserved hooks. Lets isObserved() answer "is anything watching this row" cheaply.
|
|
155
150
|
private observed = new Set<string>();
|
|
156
151
|
private box(signal: string) {
|
|
157
152
|
let box = this.boxes.get(signal);
|
|
@@ -178,9 +173,7 @@ export class MobxReactiveDeps implements ReactiveDeps {
|
|
|
178
173
|
}
|
|
179
174
|
}
|
|
180
175
|
|
|
181
|
-
// Backwards-compatible BulkDatabase2: the mobx-reactive flavor. All behavior lives in BulkDatabaseBase;
|
|
182
|
-
// this just supplies mobx reactivity and the default (getFileStorageNested2) storage backend, so the
|
|
183
|
-
// sync reads (getSingleFieldSync / getColumnSync) stay observable for mobx components.
|
|
176
|
+
// Backwards-compatible BulkDatabase2: the mobx-reactive flavor. All behavior lives in BulkDatabaseBase; this just supplies mobx reactivity and the default (getFileStorageNested2) storage backend, so the sync reads (getSingleFieldSync / getColumnSync) stay observable for mobx components.
|
|
184
177
|
export class BulkDatabase2<T extends { key: string }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
|
|
185
178
|
constructor(name: string, config?: BulkDatabase2Config) {
|
|
186
179
|
super(name, new MobxReactiveDeps(), getFileStorageNested2, config);
|
|
@@ -43,21 +43,16 @@ const MAX_INDEX_RELOAD_ATTEMPTS = 3;
|
|
|
43
43
|
const FIRST_MERGE_BYTES = TARGET_FILE_BYTES / 2;
|
|
44
44
|
const KEY_GROUP_BYTES = 800 * 1024 * 1024;
|
|
45
45
|
const DUP_THRESHOLD = 0.4;
|
|
46
|
-
// Whole-tier dedup short-circuit (after Pass 1): when the bulk tier is over this size AND the overall
|
|
47
|
-
// key duplication fraction is over this threshold, fold every bulk file in one merge instead of the
|
|
48
|
-
// per-key-group walk (which spaces merges 5 min apart — 16 h for 200 groups).
|
|
46
|
+
// Whole-tier dedup short-circuit (after Pass 1): when the bulk tier is over this size AND the overall key duplication fraction is over this threshold, fold every bulk file in one merge instead of the per-key-group walk (which spaces merges 5 min apart — 16 h for 200 groups).
|
|
49
47
|
const DEDUP_TRIGGER_BYTES = 512 * 1024 * 1024;
|
|
50
48
|
const DEDUP_TRIGGER_FRACTION = 0.5;
|
|
51
49
|
const WRITE_FLUSH_FIRST_STEP_MS = 250;
|
|
52
|
-
// Skip logs are throttled per collection: the background write path retries merges about once a second,
|
|
53
|
-
// so an unthrottled log would repeat for the whole duration of another tab's merge.
|
|
50
|
+
// Skip logs are throttled per collection: the background write path retries merges about once a second, so an unthrottled log would repeat for the whole duration of another tab's merge.
|
|
54
51
|
const MERGE_SKIP_LOG_INTERVAL_MS = 30 * 1000;
|
|
55
52
|
|
|
56
53
|
export const bulkDatabase2Timing = {
|
|
57
54
|
streamSealAgeMs: 10 * 60 * 60 * 1000,
|
|
58
|
-
// Wait this long after the tab becomes visible before the first merge check, then check this often
|
|
59
|
-
// afterwards. Tab being hidden cancels the timers (no merges while in background). Browser-only;
|
|
60
|
-
// Node compactors (e.g. remoteFileServer) drive their own polling.
|
|
55
|
+
// Wait this long after the tab becomes visible before the first merge check, then check this often afterwards. Tab being hidden cancels the timers (no merges while in background). Browser-only; Node compactors (e.g. remoteFileServer) drive their own polling.
|
|
61
56
|
visibleMergeIntervalMs: 5 * 60 * 1000,
|
|
62
57
|
mergeSpacingMs: 5 * 60 * 1000,
|
|
63
58
|
firstMergeTriggerFiles: 20,
|
|
@@ -66,8 +61,7 @@ export const bulkDatabase2Timing = {
|
|
|
66
61
|
streamFoldTriggerBytes: 64 * 1024 * 1024,
|
|
67
62
|
streamFileMaxBytes: 50 * 1024 * 1024,
|
|
68
63
|
streamFoldHardLimitBytes: 768 * 1024 * 1024,
|
|
69
|
-
// 0 = flush every write (Node — append is real and cheap); browser ramps to 15s to avoid rewriting
|
|
70
|
-
// the whole stream file per write.
|
|
64
|
+
// 0 = flush every write (Node — append is real and cheap); browser ramps to 15s to avoid rewriting the whole stream file per write.
|
|
71
65
|
writeFlushMaxDelayMs: isNode() ? 0 : 15 * 1000,
|
|
72
66
|
fileSetPollIntervalMs: 30 * 60 * 1000,
|
|
73
67
|
memoryFlushHeapBytes: 5 * 1024 * 1024 * 1024,
|
|
@@ -82,14 +76,12 @@ function fmtBytes(n: number): string {
|
|
|
82
76
|
return (n / 1024 / 1024 / 1024).toFixed(2) + "GB";
|
|
83
77
|
}
|
|
84
78
|
|
|
85
|
-
// Reactivity seam (no mobx dependency in this file). The mobx subclass supplies a ReactiveDeps that
|
|
86
|
-
// maps signal strings to its own dependency tracking; non-reactive callers pass noopReactiveDeps.
|
|
79
|
+
// Reactivity seam (no mobx dependency in this file). The mobx subclass supplies a ReactiveDeps that maps signal strings to its own dependency tracking; non-reactive callers pass noopReactiveDeps.
|
|
87
80
|
export interface ReactiveDeps {
|
|
88
81
|
observe(signal: string): void;
|
|
89
82
|
invalidate(signal: string): void;
|
|
90
83
|
batch(fn: () => void): void;
|
|
91
|
-
// Optional — lets writes skip per-key invalidation for rows nothing is watching. Undefined =
|
|
92
|
-
// "assume watched".
|
|
84
|
+
// Optional — lets writes skip per-key invalidation for rows nothing is watching. Undefined = "assume watched".
|
|
93
85
|
isObserved?(signal: string): boolean;
|
|
94
86
|
}
|
|
95
87
|
|
|
@@ -117,9 +109,7 @@ export type MergeSkipReason =
|
|
|
117
109
|
// compact() only: the collection has no files on disk.
|
|
118
110
|
| "nothingToMerge";
|
|
119
111
|
|
|
120
|
-
// What a compact()/tryMergeNow() call did. skipReason is set when the pass never ran; for the lock
|
|
121
|
-
// reasons, lockHolderId/lockExpiresInMs report who holds the lock and how long until it goes stale
|
|
122
|
-
// (so a scheduler knows when retrying could succeed).
|
|
112
|
+
// What a compact()/tryMergeNow() call did. skipReason is set when the pass never ran; for the lock reasons, lockHolderId/lockExpiresInMs report who holds the lock and how long until it goes stale (so a scheduler knows when retrying could succeed).
|
|
123
113
|
export type MergeAttemptResult = {
|
|
124
114
|
merged: boolean;
|
|
125
115
|
skipReason?: MergeSkipReason;
|
|
@@ -173,11 +163,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
173
163
|
private config: BulkDatabase2Config = {},
|
|
174
164
|
) { }
|
|
175
165
|
|
|
176
|
-
// The reader (and the background machinery that rides along with it) must NOT be set up just
|
|
177
|
-
// because the collection was constructed. Many collections are constructed and never touched, and
|
|
178
|
-
// in Node a merge tick would poke the storage factory (e.g. indexedDB) and throw. We build it
|
|
179
|
-
// lazily on first access to `this.reader`: every read, write, and merge goes through the reader,
|
|
180
|
-
// while pure construction never touches it.
|
|
166
|
+
// The reader (and the background machinery that rides along with it) must NOT be set up just because the collection was constructed. Many collections are constructed and never touched, and in Node a merge tick would poke the storage factory (e.g. indexedDB) and throw. We build it lazily on first access to `this.reader`: every read, write, and merge goes through the reader, while pure construction never touches it.
|
|
181
167
|
private _reader: BulkDatabaseReader<T> | undefined;
|
|
182
168
|
private get reader(): BulkDatabaseReader<T> {
|
|
183
169
|
if (this._reader) return this._reader;
|
|
@@ -213,11 +199,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
213
199
|
BulkDatabaseBase.startMemoryWatchdog();
|
|
214
200
|
}
|
|
215
201
|
|
|
216
|
-
// Every `visibleMergeIntervalMs` of being-visible time, run a merge check. First check fires
|
|
217
|
-
// `visibleMergeIntervalMs` after the tab becomes visible (not on initial load — gives the user a
|
|
218
|
-
// moment of session commitment before we start churning the FS), then every interval after. Tab
|
|
219
|
-
// hiding cancels the timers; visible-again restarts. Node has no document — there we consider
|
|
220
|
-
// ourselves always-visible and just install the interval directly.
|
|
202
|
+
// Every `visibleMergeIntervalMs` of being-visible time, run a merge check. First check fires `visibleMergeIntervalMs` after the tab becomes visible (not on initial load — gives the user a moment of session commitment before we start churning the FS), then every interval after. Tab hiding cancels the timers; visible-again restarts. Node has no document — there we consider ourselves always-visible and just install the interval directly.
|
|
221
203
|
private setupVisibilityMergeCheck(): void {
|
|
222
204
|
let firstTimer: ReturnType<typeof setTimeout> | undefined;
|
|
223
205
|
let intervalTimer: ReturnType<typeof setInterval> | undefined;
|
|
@@ -256,13 +238,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
256
238
|
private streamFileName: string | undefined;
|
|
257
239
|
private currentStreamFileName: string | undefined;
|
|
258
240
|
private currentStreamFileBytes = 0;
|
|
259
|
-
// In-process re-entry guard: if a merge is running, additional triggers (visibility timer, writes,
|
|
260
|
-
// tryMergeNow calls) return immediately instead of queueing. Keeps the visibility-timer firings
|
|
261
|
-
// from stacking behind a long merge.
|
|
241
|
+
// In-process re-entry guard: if a merge is running, additional triggers (visibility timer, writes, tryMergeNow calls) return immediately instead of queueing. Keeps the visibility-timer firings from stacking behind a long merge.
|
|
262
242
|
private mergeInFlight = false;
|
|
263
243
|
private lastMergeSkipLogMs = 0;
|
|
264
|
-
// Running counters of stream-tier rows + bytes on disk. Seeded from each LoadedIndex build, then
|
|
265
|
-
// incremented per flush so the fold-trigger checks current data without an extra directory listing.
|
|
244
|
+
// Running counters of stream-tier rows + bytes on disk. Seeded from each LoadedIndex build, then incremented per flush so the fold-trigger checks current data without an extra directory listing.
|
|
266
245
|
private streamRowsOnDisk = 0;
|
|
267
246
|
private streamBytesOnDisk = 0;
|
|
268
247
|
|
|
@@ -343,8 +322,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
343
322
|
return idx;
|
|
344
323
|
}
|
|
345
324
|
|
|
346
|
-
// Coalescing rebuild loop: triggers during a build set rebuildDirty so the loop iterates once more
|
|
347
|
-
// when it finishes — N rapid triggers cause at most ONE extra rebuild after the current one ends.
|
|
325
|
+
// Coalescing rebuild loop: triggers during a build set rebuildDirty so the loop iterates once more when it finishes — N rapid triggers cause at most ONE extra rebuild after the current one ends.
|
|
348
326
|
private triggerRebuild(opts: { dropStaleFallback?: boolean } = {}): Promise<void> {
|
|
349
327
|
if (opts.dropStaleFallback) this.rebuildOptions.dropStaleFallback = true;
|
|
350
328
|
if (this.rebuildPromise) {
|
|
@@ -449,8 +427,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
449
427
|
const stamped = rows.map(row => ({ time: getTimeUnique(), row }));
|
|
450
428
|
const framed = frameRows(stamped);
|
|
451
429
|
|
|
452
|
-
// Big batches skip the stream and become a bulk file directly — streaming thousands of rows
|
|
453
|
-
// one frame at a time would just churn.
|
|
430
|
+
// Big batches skip the stream and become a bulk file directly — streaming thousands of rows one frame at a time would just churn.
|
|
454
431
|
if (entries.length >= ROLLOVER_ROWS || framed.length >= ROLLOVER_BYTES) {
|
|
455
432
|
await this.writeBulkFile(rows);
|
|
456
433
|
return;
|
|
@@ -480,9 +457,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
480
457
|
void this.maybeMerge({ onlyIfStreamHeavy: true });
|
|
481
458
|
}
|
|
482
459
|
|
|
483
|
-
// Coalesce stream appends on a ramping per-collection schedule (the browser rewrites the whole
|
|
484
|
-
// file per append). The first write after a lull flushes immediately so a single edit-then-close
|
|
485
|
-
// is saved at once; sustained writes ramp toward writeFlushMaxDelayMs.
|
|
460
|
+
// Coalesce stream appends on a ramping per-collection schedule (the browser rewrites the whole file per append). The first write after a lull flushes immediately so a single edit-then-close is saved at once; sustained writes ramp toward writeFlushMaxDelayMs.
|
|
486
461
|
private async streamAppend(framed: Buffer, rows: number): Promise<void> {
|
|
487
462
|
this.pendingAppends.push({ framed, rows });
|
|
488
463
|
const max = bulkDatabase2Timing.writeFlushMaxDelayMs;
|
|
@@ -524,8 +499,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
524
499
|
}
|
|
525
500
|
// On failure the throw leaves pendingAppends intact so a later flush retries.
|
|
526
501
|
await storage.append(fileName, combined);
|
|
527
|
-
// New entries appended during the await are after `batch` — removing the front is exactly the
|
|
528
|
-
// flushed set.
|
|
502
|
+
// New entries appended during the await are after `batch` — removing the front is exactly the flushed set.
|
|
529
503
|
this.pendingAppends.splice(0, batch.length);
|
|
530
504
|
this.streamBytesOnDisk += combined.length;
|
|
531
505
|
for (const p of batch) this.streamRowsOnDisk += p.rows;
|
|
@@ -539,8 +513,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
539
513
|
}
|
|
540
514
|
|
|
541
515
|
private getStreamFileName(): string {
|
|
542
|
-
// Seal our current file once it ages past the seal threshold — no file is ever appended to past
|
|
543
|
-
// its seal age, which lets a consolidation safely fold it once aged.
|
|
516
|
+
// Seal our current file once it ages past the seal threshold — no file is ever appended to past its seal age, which lets a consolidation safely fold it once aged.
|
|
544
517
|
if (this.streamFileName) {
|
|
545
518
|
const info = parseStreamFileName(this.streamFileName);
|
|
546
519
|
if (info && Date.now() - info.timestamp >= bulkDatabase2Timing.streamSealAgeMs) this.streamFileName = undefined;
|
|
@@ -586,9 +559,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
586
559
|
private async listFiles(): Promise<{ bulkFiles: BulkFileInfo[]; streamFiles: StreamFileInfo[] }> {
|
|
587
560
|
const storage = await this.storage();
|
|
588
561
|
const names = await storage.getKeys();
|
|
589
|
-
// A consumed-merge input is hidden from reads the instant its deletion marker exists — that's
|
|
590
|
-
// what stops it from being re-read and re-merged. Physical deletion + marker cleanup runs on a
|
|
591
|
-
// throttle (not inline) so a read isn't slowed by it.
|
|
562
|
+
// A consumed-merge input is hidden from reads the instant its deletion marker exists — that's what stops it from being re-read and re-merged. Physical deletion + marker cleanup runs on a throttle (not inline) so a read isn't slowed by it.
|
|
592
563
|
const markers = await readDeleteMarkers(storage, names);
|
|
593
564
|
const excluded = markerExclusions(markers);
|
|
594
565
|
if (markers.length) void this.processMarkers();
|
|
@@ -607,9 +578,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
607
578
|
return { bulkFiles, streamFiles };
|
|
608
579
|
}
|
|
609
580
|
|
|
610
|
-
// Throttled marker housekeeping: delete inputs whose replacement outputs have landed (or whose
|
|
611
|
-
// marker has aged out) and retire fulfilled markers. Triggered from listFiles, so it's gated to
|
|
612
|
-
// avoid a delete storm under heavy reads.
|
|
581
|
+
// Throttled marker housekeeping: delete inputs whose replacement outputs have landed (or whose marker has aged out) and retire fulfilled markers. Triggered from listFiles, so it's gated to avoid a delete storm under heavy reads.
|
|
613
582
|
private processMarkers = throttleFunction(30 * 1000, async () => {
|
|
614
583
|
const storage = await this.storage();
|
|
615
584
|
const names = await storage.getKeys();
|
|
@@ -630,10 +599,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
630
599
|
void this.maybeMerge();
|
|
631
600
|
}
|
|
632
601
|
|
|
633
|
-
// ── merge policy ─────────────────────────────────────────────────────────────────────────────────
|
|
634
|
-
// Called both periodically (visibility timer) and on writes (with the streamNeedsFold gate so a
|
|
635
|
-
// small write doesn't drag the merge through). The mergeInFlight check in tryMergeNow skips any
|
|
636
|
-
// call that arrives while a previous one is still running — so 5-min timer ticks don't pile up.
|
|
602
|
+
// ── merge policy ───────────────────────────────────────────────────────────────────────────────── Called both periodically (visibility timer) and on writes (with the streamNeedsFold gate so a small write doesn't drag the merge through). The mergeInFlight check in tryMergeNow skips any call that arrives while a previous one is still running — so 5-min timer ticks don't pile up.
|
|
637
603
|
private async maybeMerge(opts: { onlyIfStreamHeavy?: boolean } = {}): Promise<void> {
|
|
638
604
|
if (!await this.automaticCompactionAllowed()) return;
|
|
639
605
|
if (opts.onlyIfStreamHeavy && !this.streamNeedsFold()) return;
|
|
@@ -644,8 +610,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
644
610
|
}
|
|
645
611
|
}
|
|
646
612
|
|
|
647
|
-
// Build the "we didn't merge" result and log it (throttled — see MERGE_SKIP_LOG_INTERVAL_MS). For
|
|
648
|
-
// lock reasons the log and result include the holder and time until the lock goes stale.
|
|
613
|
+
// Build the "we didn't merge" result and log it (throttled — see MERGE_SKIP_LOG_INTERVAL_MS). For lock reasons the log and result include the holder and time until the lock goes stale.
|
|
649
614
|
private async mergeSkip(reason: MergeSkipReason): Promise<MergeAttemptResult> {
|
|
650
615
|
let lock: MergeLockInfo | undefined;
|
|
651
616
|
if (reason === "tabLockHeld") {
|
|
@@ -661,8 +626,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
661
626
|
return { merged: false, skipReason: reason, lockHolderId: lock?.holderId, lockExpiresInMs: lock?.expiresInMs };
|
|
662
627
|
}
|
|
663
628
|
|
|
664
|
-
// Acquire both merge locks (same-tab mergeInFlight, cross-tab localStorage, cross-process file),
|
|
665
|
-
// run the pass, release. Skips (with the reason) instead of waiting when any of them is held.
|
|
629
|
+
// Acquire both merge locks (same-tab mergeInFlight, cross-tab localStorage, cross-process file), run the pass, release. Skips (with the reason) instead of waiting when any of them is held.
|
|
666
630
|
private async runLockedMerge(run: () => Promise<MergeAttemptResult>): Promise<MergeAttemptResult> {
|
|
667
631
|
if (this.mergeInFlight) return await this.mergeSkip("mergeInFlight");
|
|
668
632
|
if (!tryAcquireMergeLock(this.name, writerId)) return await this.mergeSkip("tabLockHeld");
|
|
@@ -684,8 +648,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
684
648
|
}
|
|
685
649
|
}
|
|
686
650
|
|
|
687
|
-
// The background path (maybeMerge) goes through this so write bursts coalesce; explicit callers use
|
|
688
|
-
// tryMergeNow directly and get the per-call result.
|
|
651
|
+
// The background path (maybeMerge) goes through this so write bursts coalesce; explicit callers use tryMergeNow directly and get the per-call result.
|
|
689
652
|
private tryMergeThrottled = throttleFunction(1000, async () => {
|
|
690
653
|
await this.tryMergeNow();
|
|
691
654
|
});
|
|
@@ -704,8 +667,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
704
667
|
this.streamFileName = undefined;
|
|
705
668
|
const { bulkFiles, streamFiles } = await this.listFiles();
|
|
706
669
|
if (bulkFiles.length + streamFiles.length < 1) return await this.mergeSkip("nothingToMerge");
|
|
707
|
-
// compact() folds every file → no older data survives outside it → surviving tombstones
|
|
708
|
-
// can be dropped (nothing left to suppress).
|
|
670
|
+
// compact() folds every file → no older data survives outside it → surviving tombstones can be dropped (nothing left to suppress).
|
|
709
671
|
const merged = await this.mergeFileSet(bulkFiles, streamFiles, true);
|
|
710
672
|
return { merged };
|
|
711
673
|
});
|
|
@@ -751,9 +713,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
751
713
|
}
|
|
752
714
|
}
|
|
753
715
|
|
|
754
|
-
// A bulk file that won't load is either an in-progress write (recent) or a crashed partial write
|
|
755
|
-
// (stale). Warn while recent, delete once clearly abandoned — deleting is safe because the write
|
|
756
|
-
// protocol always lands the replacement before removing the file it supersedes.
|
|
716
|
+
// A bulk file that won't load is either an in-progress write (recent) or a crashed partial write (stale). Warn while recent, delete once clearly abandoned — deleting is safe because the write protocol always lands the replacement before removing the file it supersedes.
|
|
757
717
|
private async handleUnreadableFile(file: BulkFileInfo, message: string): Promise<void> {
|
|
758
718
|
const ageMs = Date.now() - file.timestamp;
|
|
759
719
|
if (ageMs > STALE_DELETE_MS) {
|
|
@@ -769,14 +729,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
769
729
|
console.warn(`${this.name}: skipping unreadable bulk file ${file.fileName} (recent - may be in-progress): ${message}`);
|
|
770
730
|
}
|
|
771
731
|
|
|
772
|
-
// The one merge primitive. Reads + plans + writes outputs before deleting any input, so a crash
|
|
773
|
-
//
|
|
774
|
-
//
|
|
775
|
-
// entries are evicted (no consumer can ask for them now).
|
|
776
|
-
//
|
|
777
|
-
// Serialized: foldOwnStream + merge(timeLo, timeHi) + testMerge's runMerge all hit this; the lock
|
|
778
|
-
// covers tryMergeNow/compact but those two paths bypass it. runInSerial queues so they never
|
|
779
|
-
// collide on the file set.
|
|
732
|
+
// The one merge primitive. Reads + plans + writes outputs before deleting any input, so a crash leaves duplicates (next merge dedupes) rather than a gap. After the file set changes on disk, we trigger an index rebuild + atomic swap; once swap completes, the consumed files' block-cache entries are evicted (no consumer can ask for them now).
|
|
733
|
+
//
|
|
734
|
+
// Serialized: foldOwnStream + merge(timeLo, timeHi) + testMerge's runMerge all hit this; the lock covers tryMergeNow/compact but those two paths bypass it. runInSerial queues so they never collide on the file set.
|
|
780
735
|
private mergeFileSet = runInSerial(async (bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false, forceDeleteStreams = false): Promise<boolean> => {
|
|
781
736
|
this.reader.beginCompaction();
|
|
782
737
|
try {
|
|
@@ -790,11 +745,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
790
745
|
const storage = await this.storage();
|
|
791
746
|
const timestamp = nextFileTime();
|
|
792
747
|
|
|
793
|
-
// The caller's bulkFiles came from a `listFiles()` snapshot. Between then and now any file
|
|
794
|
-
// could be gone (another tab's deferred delete, manual cleanup). The subCaches reader for a
|
|
795
|
-
// deleted file is silently stale — its in-memory metadata still resolves, but the very next
|
|
796
|
-
// getRange against disk throws. Re-verify existence up-front and drop the cache entry for
|
|
797
|
-
// anything that vanished, so we don't even try to plan around a ghost file.
|
|
748
|
+
// The caller's bulkFiles came from a `listFiles()` snapshot. Between then and now any file could be gone (another tab's deferred delete, manual cleanup). The subCaches reader for a deleted file is silently stale — its in-memory metadata still resolves, but the very next getRange against disk throws. Re-verify existence up-front and drop the cache entry for anything that vanished, so we don't even try to plan around a ghost file.
|
|
798
749
|
const verifiedBulkFiles = (await Promise.all(bulkFiles.map(async f => {
|
|
799
750
|
try {
|
|
800
751
|
const info = await storage.getInfo(f.fileName);
|
|
@@ -835,8 +786,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
835
786
|
// expanded console.group so a merge takes one collapsible block, not a
|
|
836
787
|
// screenful (and never the per-file name dump it used to spew).
|
|
837
788
|
const steps: string[] = [];
|
|
838
|
-
// Each step records the time spent since the previous step in blue at the front, so callers can
|
|
839
|
-
// just describe what they did and not bother measuring/formatting elapsed times themselves.
|
|
789
|
+
// Each step records the time spent since the previous step in blue at the front, so callers can just describe what they did and not bother measuring/formatting elapsed times themselves.
|
|
840
790
|
let lastStepMs = mergeStartMs;
|
|
841
791
|
const log = (line: string) => {
|
|
842
792
|
const now = Date.now();
|
|
@@ -879,36 +829,25 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
879
829
|
for (const line of steps) console.log(line);
|
|
880
830
|
console.groupEnd();
|
|
881
831
|
|
|
882
|
-
// Only the sources runPlannedMerge actually used can be retired — a source it dropped mid-plan
|
|
883
|
-
// (file gone, corruption) still holds data we couldn't merge in, so leave it on disk and let a
|
|
884
|
-
// future merge re-attempt it.
|
|
832
|
+
// Only the sources runPlannedMerge actually used can be retired — a source it dropped mid-plan (file gone, corruption) still holds data we couldn't merge in, so leave it on disk and let a future merge re-attempt it.
|
|
885
833
|
const usedConsumedBulk = consumedBulk.filter(f => mergeResult.usedSourceNames.has(f.fileName));
|
|
886
834
|
const usedStreamFiles = streamFiles.filter(f => mergeResult.usedSourceNames.has(f.fileName) || mergeResult.usedSourceNames.has("(streams)"));
|
|
887
|
-
// A stream may still be appended to after we read it (the writer hasn't sealed/moved on); only
|
|
888
|
-
// retire streams canDeleteStream clears — the rest stay live and a later merge re-folds them
|
|
889
|
-
// once sealed. Bulk files are immutable, so a used one is always safe to retire.
|
|
835
|
+
// A stream may still be appended to after we read it (the writer hasn't sealed/moved on); only retire streams canDeleteStream clears — the rest stay live and a later merge re-folds them once sealed. Bulk files are immutable, so a used one is always safe to retire.
|
|
890
836
|
const deletableStreams: string[] = [];
|
|
891
837
|
for (const f of usedStreamFiles) {
|
|
892
838
|
if (await this.canDeleteStream(f, Date.now(), streamData.sizes, forceDeleteStreams)) deletableStreams.push(f.fileName);
|
|
893
839
|
}
|
|
894
840
|
const deleteFiles = [...usedConsumedBulk.map(f => f.fileName), ...deletableStreams];
|
|
895
841
|
|
|
896
|
-
// Write a deletion marker instead of deleting inline. From the next read on, listFiles hides
|
|
897
|
-
// these inputs (so they're never re-merged — the loop fix), and processMarkers removes them
|
|
898
|
-
// physically once the outputs have landed or the marker ages out. Marker BEFORE the rebuild so
|
|
899
|
-
// the freshly-swapped index already excludes the retired inputs.
|
|
842
|
+
// Write a deletion marker instead of deleting inline. From the next read on, listFiles hides these inputs (so they're never re-merged — the loop fix), and processMarkers removes them physically once the outputs have landed or the marker ages out. Marker BEFORE the rebuild so the freshly-swapped index already excludes the retired inputs.
|
|
900
843
|
if (deleteFiles.length) await writeDeleteMarker(storage, { deleteFiles, replacedBy: outNames });
|
|
901
844
|
|
|
902
|
-
// Rebuild + swap so the index sees the new outputs and drops the retired inputs. Block-cache
|
|
903
|
-
// eviction for the dropped inputs happens here (they're no longer in the new index's fileSet).
|
|
845
|
+
// Rebuild + swap so the index sees the new outputs and drops the retired inputs. Block-cache eviction for the dropped inputs happens here (they're no longer in the new index's fileSet).
|
|
904
846
|
await this.triggerRebuild();
|
|
905
847
|
return newNames.length > 0 || carriedDeletes > 0;
|
|
906
848
|
}
|
|
907
849
|
|
|
908
|
-
// A stream is safe to delete iff no writer will append to it again: it's aged past the seal age
|
|
909
|
-
// (writer has provably switched files) OR cross-tab sync is on AND its size didn't change while
|
|
910
|
-
// we read it. Else leave it — the data is also in the new bulk file; a later merge deletes it
|
|
911
|
-
// once aged.
|
|
850
|
+
// A stream is safe to delete iff no writer will append to it again: it's aged past the seal age (writer has provably switched files) OR cross-tab sync is on AND its size didn't change while we read it. Else leave it — the data is also in the new bulk file; a later merge deletes it once aged.
|
|
912
851
|
private async canDeleteStream(f: StreamFileInfo, now: number, sizes: Map<string, number>, force = false): Promise<boolean> {
|
|
913
852
|
if (now - f.timestamp >= bulkDatabase2Timing.streamSealAgeMs) return true;
|
|
914
853
|
if (!isSyncSupported() && !force) return false;
|
|
@@ -933,11 +872,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
933
872
|
}
|
|
934
873
|
|
|
935
874
|
// The merge policy (two passes, spaced by mergeSpacingMs):
|
|
936
|
-
// 1) Consolidate recent fragmentation: take newest files up to ~FIRST_MERGE_BYTES; merge them
|
|
937
|
-
//
|
|
938
|
-
// 2) Key-stratify: walk all keys in ~KEY_GROUP_BYTES groups; rewrite groups whose duplicate-key
|
|
939
|
-
// fraction passes DUP_THRESHOLD, highest first.
|
|
940
|
-
// Use tryMerge instead
|
|
875
|
+
// 1) Consolidate recent fragmentation: take newest files up to ~FIRST_MERGE_BYTES; merge them into one when they fragment or span too wide a time range.
|
|
876
|
+
// 2) Key-stratify: walk all keys in ~KEY_GROUP_BYTES groups; rewrite groups whose duplicate-key fraction passes DUP_THRESHOLD, highest first. Use tryMerge instead
|
|
941
877
|
private async testMergeINTERNAL_DO_NOT_CALL(): Promise<boolean> {
|
|
942
878
|
let merged = false;
|
|
943
879
|
await this.flushPending();
|
|
@@ -947,8 +883,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
947
883
|
return true;
|
|
948
884
|
};
|
|
949
885
|
|
|
950
|
-
// Hard stream limit: a stream this big makes every read pull a huge file → fold ALL of it now,
|
|
951
|
-
// force-delete (canDeleteStream still requires size-stable, so an active writer never loses data).
|
|
886
|
+
// Hard stream limit: a stream this big makes every read pull a huge file → fold ALL of it now, force-delete (canDeleteStream still requires size-stable, so an active writer never loses data).
|
|
952
887
|
{
|
|
953
888
|
const { streamFiles } = await this.listFiles();
|
|
954
889
|
if (streamFiles.length) {
|
|
@@ -962,8 +897,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
962
897
|
}
|
|
963
898
|
}
|
|
964
899
|
|
|
965
|
-
// Pass 1: consolidate recent. Only seal when cross-tab sync can fold recent streams — in Node
|
|
966
|
-
// canDeleteStream needs them aged anyway, so sealing would just fragment streams every pass.
|
|
900
|
+
// Pass 1: consolidate recent. Only seal when cross-tab sync can fold recent streams — in Node canDeleteStream needs them aged anyway, so sealing would just fragment streams every pass.
|
|
967
901
|
const foldRecentStreams = isSyncSupported();
|
|
968
902
|
if (foldRecentStreams) {
|
|
969
903
|
syncBroadcastSeal(this.name);
|
|
@@ -1004,9 +938,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
1004
938
|
}
|
|
1005
939
|
}
|
|
1006
940
|
|
|
1007
|
-
// Whole-tier dedup short-circuit (between Pass 1 and the per-group Pass 2): when the bulk tier
|
|
1008
|
-
// is big enough AND mostly duplicates, fold every bulk file in one merge. Avoids the per-group
|
|
1009
|
-
// walk's 5-min spacing × N-groups latency.
|
|
941
|
+
// Whole-tier dedup short-circuit (between Pass 1 and the per-group Pass 2): when the bulk tier is big enough AND mostly duplicates, fold every bulk file in one merge. Avoids the per-group walk's 5-min spacing × N-groups latency.
|
|
1010
942
|
{
|
|
1011
943
|
const { bulkFiles } = await this.listFiles();
|
|
1012
944
|
if (bulkFiles.length >= 2) {
|
|
@@ -1031,8 +963,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
1031
963
|
}
|
|
1032
964
|
}
|
|
1033
965
|
|
|
1034
|
-
// Pass 2: key-stratified deduplication. Disjoint key ranges → one group's merge doesn't
|
|
1035
|
-
// change another's duplication; re-select each group's files at merge time (set shifts).
|
|
966
|
+
// Pass 2: key-stratified deduplication. Disjoint key ranges → one group's merge doesn't change another's duplication; re-select each group's files at merge time (set shifts).
|
|
1036
967
|
const groups = await this.findDuplicateGroups();
|
|
1037
968
|
for (const g of groups) {
|
|
1038
969
|
const { bulkFiles } = await this.listFiles();
|
|
@@ -1131,9 +1062,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
1131
1062
|
return this.reader.isColumnLoadedSync(column);
|
|
1132
1063
|
}
|
|
1133
1064
|
|
|
1134
|
-
// Reactive: true while a merge is rewriting this collection's files. Use this in UI to show a
|
|
1135
|
-
// "compacting…" indicator. Counts both background merges (maybeMerge) and explicit compact/merge
|
|
1136
|
-
// calls. Becomes false once the new index is swapped in (the deferred-delete window is NOT counted).
|
|
1065
|
+
// Reactive: true while a merge is rewriting this collection's files. Use this in UI to show a "compacting…" indicator. Counts both background merges (maybeMerge) and explicit compact/merge calls. Becomes false once the new index is swapped in (the deferred-delete window is NOT counted).
|
|
1137
1066
|
public isCompactingSync(): boolean {
|
|
1138
1067
|
return this.reader.isCompactingSync();
|
|
1139
1068
|
}
|
|
@@ -1228,8 +1157,7 @@ export type BulkFileEntry = {
|
|
|
1228
1157
|
bytes: number;
|
|
1229
1158
|
// Filesystem mtime (ms since epoch) — 0 if the storage layer didn't return one.
|
|
1230
1159
|
lastModified: number;
|
|
1231
|
-
// Lazy: pulled from the cached sub-reader for bulk (free if loaded), or from a parse of the stream
|
|
1232
|
-
// file (small — tier-0 size-capped, also cached). Call only when you need the per-file detail.
|
|
1160
|
+
// Lazy: pulled from the cached sub-reader for bulk (free if loaded), or from a parse of the stream file (small — tier-0 size-capped, also cached). Call only when you need the per-file detail.
|
|
1233
1161
|
getDetails: () => Promise<BulkFileDetails>;
|
|
1234
1162
|
};
|
|
1235
1163
|
export type BulkFileInfoListing = { files: BulkFileEntry[]; count: number; totalBytes: number };
|