sliftutils 1.7.32 → 1.7.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/builders/extensionBuild.ts +1 -1
  2. package/bundler/bundleEntry.ts +1 -3
  3. package/bundler/bundler.ts +2 -4
  4. package/bundler/sourceMaps.ts +1 -2
  5. package/examplestorage/browser.tsx +1 -2
  6. package/examplestorage/exampleserver.ts +1 -2
  7. package/index.d.ts +105 -100
  8. package/misc/dist/environment.ts.cache +2 -2
  9. package/misc/dist/getSecret.ts.cache +2 -2
  10. package/misc/dist/strings.ts.cache +2 -2
  11. package/misc/dist/zip.ts.cache +2 -2
  12. package/misc/https/dist/certs.ts.cache +2 -2
  13. package/misc/https/dist/dns.ts.cache +7 -16
  14. package/misc/https/dist/persistentLocalStorage.ts.cache +2 -2
  15. package/misc/https/dns.d.ts +1 -4
  16. package/misc/https/dns.ts +5 -14
  17. package/misc/https/hostServer.d.ts +2 -0
  18. package/misc/https/hostServer.ts +6 -7
  19. package/misc/https/httpsCerts.ts +4 -10
  20. package/misc/openrouter.ts +1 -2
  21. package/package.json +1 -1
  22. package/render-utils/FullscreenModal.tsx +1 -4
  23. package/render-utils/GenericFormat.tsx +2 -8
  24. package/render-utils/Input.tsx +5 -13
  25. package/render-utils/InputLabel.tsx +4 -6
  26. package/render-utils/InputPicker.tsx +1 -2
  27. package/render-utils/URLParam.ts +2 -3
  28. package/render-utils/dist/observer.tsx.cache +2 -2
  29. package/render-utils/mobxTyped.ts +2 -3
  30. package/render-utils/niceStringify.ts +2 -4
  31. package/storage/ArchivesDisk.ts +4 -14
  32. package/storage/BulkDatabase2/BulkDatabase2.d.ts +1 -2
  33. package/storage/BulkDatabase2/BulkDatabase2.ts +4 -11
  34. package/storage/BulkDatabase2/BulkDatabaseBase.ts +42 -114
  35. package/storage/BulkDatabase2/BulkDatabaseFormat.ts +31 -91
  36. package/storage/BulkDatabase2/BulkDatabaseMerge.ts +19 -57
  37. package/storage/BulkDatabase2/BulkDatabaseReader.ts +4 -12
  38. package/storage/BulkDatabase2/LoadedIndex.ts +6 -18
  39. package/storage/BulkDatabase2/WriteOverlay.ts +3 -8
  40. package/storage/BulkDatabase2/blockCache.ts +12 -28
  41. package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +40 -103
  42. package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +19 -50
  43. package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +18 -50
  44. package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +7 -15
  45. package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +9 -21
  46. package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +6 -11
  47. package/storage/BulkDatabase2/dist/blockCache.ts.cache +15 -31
  48. package/storage/BulkDatabase2/dist/mergeLock.ts.cache +10 -29
  49. package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +10 -18
  50. package/storage/BulkDatabase2/dist/streamLog.ts.cache +17 -43
  51. package/storage/BulkDatabase2/dist/syncClient.ts.cache +11 -27
  52. package/storage/BulkDatabase2/mergeLock.ts +7 -26
  53. package/storage/BulkDatabase2/mergeMarkers.ts +7 -15
  54. package/storage/BulkDatabase2/streamLog.ts +14 -40
  55. package/storage/BulkDatabase2/syncClient.ts +8 -24
  56. package/storage/FileFolderAPI.tsx +28 -75
  57. package/storage/IArchives.d.ts +3 -3
  58. package/storage/IArchives.ts +21 -50
  59. package/storage/IStorage.ts +2 -4
  60. package/storage/PendingManager.tsx +2 -3
  61. package/storage/TransactionStorage.ts +11 -24
  62. package/storage/backblaze.ts +12 -43
  63. package/storage/dist/ArchivesDisk.ts.cache +7 -17
  64. package/storage/dist/FileFolderAPI.tsx.cache +25 -63
  65. package/storage/dist/IArchives.ts.cache +10 -20
  66. package/storage/dist/IndexedDBFileFolderAPI.ts.cache +2 -2
  67. package/storage/dist/JSONStorage.ts.cache +2 -2
  68. package/storage/dist/PendingManager.tsx.cache +5 -6
  69. package/storage/dist/TransactionStorage.ts.cache +14 -27
  70. package/storage/dist/backblaze.ts.cache +15 -46
  71. package/storage/dist/embeddingFormats.ts.cache +11 -24
  72. package/storage/dist/fileSystemPointer.ts.cache +4 -7
  73. package/storage/dist/remoteFileStorage.ts.cache +17 -36
  74. package/storage/embeddingFormats.ts +11 -27
  75. package/storage/fileSystemPointer.ts +2 -7
  76. package/storage/proxydatabase/dist/ivfEmbeddingDatabase.ts.cache +8 -18
  77. package/storage/proxydatabase/dist/transactionSet.ts.cache +4 -7
  78. package/storage/proxydatabase/inMemoryDatabase.ts +1 -3
  79. package/storage/proxydatabase/ivfEmbeddingDatabase.ts +8 -23
  80. package/storage/proxydatabase/transactionSet.ts +2 -5
  81. package/storage/remoteFileServer.ts +20 -48
  82. package/storage/remoteFileStorage.ts +14 -33
  83. package/storage/remoteStorage/ArchivesRemote.d.ts +1 -4
  84. package/storage/remoteStorage/ArchivesRemote.ts +11 -26
  85. package/storage/remoteStorage/ArchivesUrl.ts +2 -4
  86. package/storage/remoteStorage/accessPage.tsx +1 -5
  87. package/storage/remoteStorage/blobStore.d.ts +5 -13
  88. package/storage/remoteStorage/blobStore.ts +52 -130
  89. package/storage/remoteStorage/createArchives.d.ts +9 -13
  90. package/storage/remoteStorage/createArchives.ts +43 -187
  91. package/storage/remoteStorage/deployTakeover.d.ts +18 -32
  92. package/storage/remoteStorage/deployTakeover.ts +70 -384
  93. package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +12 -24
  94. package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +5 -7
  95. package/storage/remoteStorage/dist/blobStore.ts.cache +47 -117
  96. package/storage/remoteStorage/dist/cliArgs.ts.cache +2 -2
  97. package/storage/remoteStorage/dist/createArchives.ts.cache +48 -189
  98. package/storage/remoteStorage/dist/deployTakeover.ts.cache +81 -387
  99. package/storage/remoteStorage/dist/intermediateSources.ts.cache +119 -0
  100. package/storage/remoteStorage/dist/remoteConfig.ts.cache +36 -24
  101. package/storage/remoteStorage/dist/sourceWrapper.ts.cache +20 -40
  102. package/storage/remoteStorage/dist/storageClientController.ts.cache +4 -6
  103. package/storage/remoteStorage/dist/storageController.ts.cache +11 -73
  104. package/storage/remoteStorage/dist/storageServerState.ts.cache +265 -178
  105. package/storage/remoteStorage/grantAccessCli.ts +1 -3
  106. package/storage/remoteStorage/intermediateSources.d.ts +21 -0
  107. package/storage/remoteStorage/intermediateSources.ts +117 -0
  108. package/storage/remoteStorage/remoteConfig.ts +33 -20
  109. package/storage/remoteStorage/sourceWrapper.d.ts +8 -19
  110. package/storage/remoteStorage/sourceWrapper.ts +19 -41
  111. package/storage/remoteStorage/storageClientController.ts +1 -3
  112. package/storage/remoteStorage/storageController.d.ts +4 -5
  113. package/storage/remoteStorage/storageController.ts +10 -81
  114. package/storage/remoteStorage/storageServer.ts +20 -34
  115. package/storage/remoteStorage/storageServerState.d.ts +28 -5
  116. package/storage/remoteStorage/storageServerState.ts +271 -207
@@ -123,8 +123,7 @@ export class InputPicker<T> extends preact.Component<{
123
123
  <button
124
124
  key={`add-${option.matchText}`}
125
125
  className={css.hbox(5).button + greenButton}
126
- // On mouse down, so we can add picked BEFORE we blur (otherwise
127
- // this button disappears before it can be clicked)
126
+ // On mouse down, so we can add picked BEFORE we blur (otherwise this button disappears before it can be clicked)
128
127
  onMouseDown={() => {
129
128
  this.props.addPicked(option.value);
130
129
  }}
@@ -84,9 +84,8 @@ export function getCurrentUrl() {
84
84
  let currentBatchedUrl: string | undefined;
85
85
  function throttledUrlPush(url: string) {
86
86
  history.pushState({}, "", url);
87
- //currentBatchedUrl = url;
88
- // NOTE: Stopped throttling, so when you click on links, it immediately updates the selected state.
89
- //void throttledUrlPushBase(url);
87
+ // currentBatchedUrl = url;
88
+ // NOTE: Stopped throttling, so when you click on links, it immediately updates the selected state. void throttledUrlPushBase(url);
90
89
  }
91
90
  const throttledUrlPushBase = throttleFunction(1000, (url: string) => {
92
91
  currentBatchedUrl = undefined;
@@ -1,4 +1,4 @@
1
- "use strict";
1
+ "use strict"; var console = (function shimConsole() { let baseConsole = globalThis.console; if (typeof window !== "undefined") return baseConsole; let name = __filename.replaceAll("\\", "/").split("/").pop(); let dirName = __dirname.replaceAll("\\", "/").split("/").pop(); let context = { __FILE__: __filename, __DIR__: __dirname, __NAME__: name, __DIRNAME__: dirName }; let newConsole = { ...baseConsole }; for (let shim of ["log", "info", "warn", "error"]) { newConsole[shim] = (...args) => { args.push(context); baseConsole[shim](...args); }; } newConsole.__context = context; return newConsole; })(); "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
3
3
  //exports.observer = void 0;
4
4
  module.allowclient = true;
@@ -34,4 +34,4 @@ function observer(Constructor) {
34
34
  }
35
35
  exports.observer = observer;
36
36
  //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib2JzZXJ2ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJvYnNlcnZlci50c3giXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBQUEsTUFBTSxDQUFDLFdBQVcsR0FBRyxJQUFJLENBQUM7QUFHMUIsK0JBQTRDO0FBQzVDLG1FQUFxRTtBQUVyRSxJQUFJLG9CQUFvQixHQUFHLENBQUMsQ0FBQztBQUM3QixTQUFnQixRQUFRLENBU3BCLFdBQWM7SUFFZCxJQUFJLElBQUksR0FBRyxXQUFXLENBQUMsSUFBSSxDQUFDO0lBQzVCLE9BQU8sS0FBTSxTQUFRLFdBQVc7UUFBekI7O1lBR0gsYUFBUSxHQUFHLElBQUksZUFBUSxDQUFDLFVBQVUsSUFBSSxJQUFJLG9CQUFvQixFQUFFLEVBQUUsRUFBRSxHQUFHLEVBQUU7Z0JBQ3JFLEtBQUssQ0FBQyxXQUFXLEVBQUUsQ0FBQztZQUN4QixDQUFDLENBQUMsQ0FBQztRQWNQLENBQUM7UUFsQkcsYUFBYTtRQUNiLE1BQU0sS0FBSyxJQUFJLEtBQUssT0FBTyxXQUFXLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQztRQUk5QyxvQkFBb0I7O1lBQ2hCLElBQUksQ0FBQyxRQUFRLENBQUMsT0FBTyxFQUFFLENBQUM7WUFDeEIsTUFBQSxLQUFLLENBQUMsb0JBQW9CLG9EQUFJLENBQUM7UUFDbkMsQ0FBQztRQUNELE1BQU0sQ0FBQyxHQUFHLElBQVc7WUFDakIsSUFBSSxNQUE2QixDQUFDO1lBQ2xDLElBQUksQ0FBQyxRQUFRLENBQUMsS0FBSyxDQUFDLEdBQUcsRUFBRTtnQkFDckIsSUFBQSxzQkFBWSxFQUFDLEdBQUcsRUFBRTtvQkFDZCxNQUFNLEdBQUksS0FBSyxDQUFDLE1BQWMsQ0FBQyxHQUFHLElBQUksQ0FBQyxDQUFDO2dCQUM1QyxDQUFDLEVBQUUsR0FBRyxJQUFJLFNBQVMsQ0FBQyxDQUFDO1lBQ3pCLENBQUMsQ0FBQyxDQUFDO1lBQ0gsT0FBTyxNQUFNLENBQUM7UUFDbEIsQ0FBQztLQUNKLENBQUM7QUFDTixDQUFDO0FBaENELDRCQWdDQyIsInNvdXJjZXNDb250ZW50IjpbIm1vZHVsZS5hbGxvd2NsaWVudCA9IHRydWU7XG5cbmltcG9ydCAqIGFzIHByZWFjdCBmcm9tIFwicHJlYWN0XCI7XG5pbXBvcnQgeyBvYnNlcnZhYmxlLCBSZWFjdGlvbiB9IGZyb20gXCJtb2J4XCI7XG5pbXBvcnQgeyBtZWFzdXJlQmxvY2sgfSBmcm9tIFwic29ja2V0LWZ1bmN0aW9uL3NyYy9wcm9maWxpbmcvbWVhc3VyZVwiO1xuXG5sZXQgZ2xvYmFsQ29uc3RydWN0T3JkZXIgPSAxO1xuZXhwb3J0IGZ1bmN0aW9uIG9ic2VydmVyPFxuICAgIFQgZXh0ZW5kcyB7XG4gICAgICAgIG5ldyguLi5hcmdzOiBhbnlbXSk6IHtcbiAgICAgICAgICAgIHJlbmRlcigpOiBwcmVhY3QuQ29tcG9uZW50Q2hpbGQ7XG4gICAgICAgICAgICBmb3JjZVVwZGF0ZShjYWxsYmFjaz86ICgpID0+IHZvaWQpOiB2b2lkO1xuICAgICAgICAgICAgY29tcG9uZW50V2lsbFVubW91bnQ/KCk6IHZvaWQ7XG4gICAgICAgIH1cbiAgICB9XG4+KFxuICAgIENvbnN0cnVjdG9yOiBUXG4pIHtcbiAgICBsZXQgbmFtZSA9IENvbnN0cnVjdG9yLm5hbWU7XG4gICAgcmV0dXJuIGNsYXNzIGV4dGVuZHMgQ29uc3RydWN0b3Ige1xuICAgICAgICAvLyBAdHMtaWdub3JlXG4gICAgICAgIHN0YXRpYyBnZXQgbmFtZSgpIHsgcmV0dXJuIENvbnN0cnVjdG9yLm5hbWU7IH1cbiAgICAgICAgcmVhY3Rpb24gPSBuZXcgUmVhY3Rpb24oYHJlbmRlci4ke25hbWV9LiR7Z2xvYmFsQ29uc3RydWN0T3JkZXIrK31gLCAoKSA9PiB7XG4gICAgICAgICAgICBzdXBlci5mb3JjZVVwZGF0ZSgpO1xuICAgICAgICB9KTtcbiAgICAgICAgY29tcG9uZW50V2lsbFVubW91bnQoKSB7XG4gICAgICAgICAgICB0aGlzLnJlYWN0aW9uLmRpc3Bvc2UoKTtcbiAgICAgICAgICAgIHN1cGVyLmNvbXBvbmVudFdpbGxVbm1vdW50Py4oKTtcbiAgICAgICAgfVxuICAgICAgICByZW5kZXIoLi4uYXJnczogYW55W10pIHtcbiAgICAgICAgICAgIGxldCBvdXRwdXQ6IHByZWFjdC5Db21wb25lbnRDaGlsZDtcbiAgICAgICAgICAgIHRoaXMucmVhY3Rpb24udHJhY2soKCkgPT4ge1xuICAgICAgICAgICAgICAgIG1lYXN1cmVCbG9jaygoKSA9PiB7XG4gICAgICAgICAgICAgICAgICAgIG91dHB1dCA9IChzdXBlci5yZW5kZXIgYXMgYW55KSguLi5hcmdzKTtcbiAgICAgICAgICAgICAgICB9LCBgJHtuYW1lfXxyZW5kZXJgKTtcbiAgICAgICAgICAgIH0pO1xuICAgICAgICAgICAgcmV0dXJuIG91dHB1dDtcbiAgICAgICAgfVxuICAgIH07XG59Il19
37
- /* _JS_SOURCE_HASH = "9ea1e7947dad4dc649b606086b48cc67b99bda02fd8fbe7af03aa1185a7420d9"; */
37
+ /* _JS_SOURCE_HASH = "87290663d59bacd34ed8b38a01f2dd7e2297988b42c431ea6651a529aa2e2527"; */
@@ -1,8 +1,7 @@
1
1
  import * as mobx from "mobx";
2
2
  import { batchFunction } from "socket-function/src/batching";
3
3
 
4
- // Re-export the core mobx primitives so downstream packages (which may have their own duplicate mobx
5
- // in node_modules) can share THIS mobx instance, otherwise reactivity doesn't cross package boundaries.
4
+ // Re-export the core mobx primitives so downstream packages (which may have their own duplicate mobx in node_modules) can share THIS mobx instance, otherwise reactivity doesn't cross package boundaries.
6
5
  export { observable, runInAction, computed, autorun, onBecomeObserved, onBecomeUnobserved } from "mobx";
7
6
  export function configureMobxNextFrameScheduler() {
8
7
  // NOTE: This makes a big difference if we do await calls in a loop which mutates observable state. BUT... we should probably just do those await calls before the loop?
@@ -10,7 +9,7 @@ export function configureMobxNextFrameScheduler() {
10
9
  delay: 16,
11
10
  name: "reactionScheduler",
12
11
  }, (callbacks: (() => void)[]) => {
13
- //console.log(`Triggering ${callbacks.length} reactions`);
12
+ // console.log(`Triggering ${callbacks.length} reactions`);
14
13
  for (let callback of callbacks) {
15
14
  callback();
16
15
  }
@@ -8,8 +8,7 @@ export const niceStringifyNan = `{NaN}`;
8
8
  export const niceStringifyUndefined = `{Undefined}`;
9
9
 
10
10
 
11
- // BUG: This is actually broken for hex strings. Hex strings may sometimes be entirely numbers,
12
- // which means they will randomly change type.
11
+ // BUG: This is actually broken for hex strings. Hex strings may sometimes be entirely numbers, which means they will randomly change type.
13
12
  function looksLikeJSON(str: string) {
14
13
  return (
15
14
  str === "null"
@@ -32,8 +31,7 @@ export function niceStringify(value: unknown): string {
32
31
  if (value === true) return niceStringifyTrue;
33
32
  if (Number.isNaN(value)) return niceStringifyNan;
34
33
 
35
- // Any strings that don't look like JSON, don't need to encoded as JSON, and can instead
36
- // just be stored as strings.
34
+ // Any strings that don't look like JSON, don't need to encoded as JSON, and can instead just be stored as strings.
37
35
  if (typeof value === "string" && !looksLikeJSON(value)) {
38
36
  return value;
39
37
  }
@@ -5,11 +5,7 @@ import { runInfinitePoll } from "socket-function/src/batching";
5
5
  import { sort, binarySearchBasic } from "socket-function/src/misc";
6
6
  import { IArchives, ArchiveFileInfo, ArchivesConfig, assertValidLastModified } from "./IArchives";
7
7
 
8
- // The base file-system IArchives: storage is one-to-one with the file system, every key is exactly
9
- // one real file under <folder>/files, so the file system itself is the index. File handles are
10
- // cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in
11
- // serial, so they can't collide with each other or with handle closing. Used as the disk
12
- // synchronization source of BlobStore (see remoteStorage/blobStore.ts).
8
+ // The base file-system IArchives: storage is one-to-one with the file system, every key is exactly one real file under <folder>/files, so the file system itself is the index. File handles are cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in serial, so they can't collide with each other or with handle closing. Used as the disk synchronization source of BlobStore (see remoteStorage/blobStore.ts).
13
9
 
14
10
  const HANDLE_IDLE_TIMEOUT = 1000 * 60;
15
11
  const HANDLE_SWEEP_INTERVAL = 1000 * 15;
@@ -20,11 +16,7 @@ type HandleEntry = {
20
16
  lastUse: number;
21
17
  };
22
18
 
23
- // Caches open file handles, closing them once idle for HANDLE_IDLE_TIMEOUT. Instead of one
24
- // setTimeout per handle, a list sorted by last use is swept periodically (entries are moved via
25
- // binary search on access, and lastUse values only increase, so touched entries append at the
26
- // end). Also serializes operations per file: each operation only starts once the previous one on
27
- // the same file finished, and a handle is never closed while an operation is pending on it.
19
+ // Caches open file handles, closing them once idle for HANDLE_IDLE_TIMEOUT. Instead of one setTimeout per handle, a list sorted by last use is swept periodically (entries are moved via binary search on access, and lastUse values only increase, so touched entries append at the end). Also serializes operations per file: each operation only starts once the previous one on the same file finished, and a handle is never closed while an operation is pending on it.
28
20
  class FileHandleCache {
29
21
  private entries = new Map<string, HandleEntry>();
30
22
  // Sorted by lastUse ascending (least recently used first)
@@ -278,8 +270,7 @@ export class ArchivesDisk implements IArchives {
278
270
  }
279
271
  }
280
272
 
281
- // Large files stream into their own file under uploads/, then move into place on finish. No
282
- // handle is opened until the first append actually happens.
273
+ // Large files stream into their own file under uploads/, then move into place on finish. No handle is opened until the first append actually happens.
283
274
  public async startLargeUpload(): Promise<string> {
284
275
  await this.init();
285
276
  let id = `${Date.now()}_${this.nextLargeUploadId++}`;
@@ -344,8 +335,7 @@ async function statOrUndefined(filePath: string): Promise<fs.Stats | undefined>
344
335
  }
345
336
  }
346
337
 
347
- // The folders/shallow post-processing shared by findInfo implementations that list flat files
348
- // (used by ArchivesDisk on the raw disk walk, and by BlobStore on its index).
338
+ // The folders/shallow post-processing shared by findInfo implementations that list flat files (used by ArchivesDisk on the raw disk walk, and by BlobStore on its index).
349
339
  export function applyFindInfoShape(files: ArchiveFileInfo[], prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): ArchiveFileInfo[] {
350
340
  if (config?.type === "folders") {
351
341
  let folders = new Map<string, ArchiveFileInfo>();
@@ -145,8 +145,7 @@ export interface IBulkDatabase2<T extends {
145
145
  * "someone else is already merging".
146
146
  */
147
147
  tryMergeNow(): Promise<MergeAttemptResult>;
148
- /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
149
- * most callers want compact() or tryMergeNow(). */
148
+ /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level; most callers want compact() or tryMergeNow(). */
150
149
  merge(timeLo: number, timeHi: number): Promise<void>;
151
150
  }
152
151
  export declare class MobxReactiveDeps implements ReactiveDeps {
@@ -139,19 +139,14 @@ export interface IBulkDatabase2<T extends { key: string }> {
139
139
  */
140
140
  tryMergeNow(): Promise<MergeAttemptResult>;
141
141
 
142
- /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
143
- * most callers want compact() or tryMergeNow(). */
142
+ /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level; most callers want compact() or tryMergeNow(). */
144
143
  merge(timeLo: number, timeHi: number): Promise<void>;
145
144
  }
146
145
 
147
- // mobx-backed reactivity: each signal string gets its own observable box. observe() reads it (so an
148
- // observer/autorun that calls it tracks that box) and invalidate() bumps it (so those reactions re-run).
149
- // This reproduces the fine-grained per-key + load-version reactivity the class had when it used
150
- // observable.map/observable.box directly, while keeping all that logic in the mobx-free base.
146
+ // mobx-backed reactivity: each signal string gets its own observable box. observe() reads it (so an observer/autorun that calls it tracks that box) and invalidate() bumps it (so those reactions re-run). This reproduces the fine-grained per-key + load-version reactivity the class had when it used observable.map/observable.box directly, while keeping all that logic in the mobx-free base.
151
147
  export class MobxReactiveDeps implements ReactiveDeps {
152
148
  private boxes = new Map<string, { get(): number; set(value: number): void }>();
153
- // Signals that currently have at least one observer, kept current via mobx's onBecomeObserved/
154
- // onBecomeUnobserved hooks. Lets isObserved() answer "is anything watching this row" cheaply.
149
+ // Signals that currently have at least one observer, kept current via mobx's onBecomeObserved/ onBecomeUnobserved hooks. Lets isObserved() answer "is anything watching this row" cheaply.
155
150
  private observed = new Set<string>();
156
151
  private box(signal: string) {
157
152
  let box = this.boxes.get(signal);
@@ -178,9 +173,7 @@ export class MobxReactiveDeps implements ReactiveDeps {
178
173
  }
179
174
  }
180
175
 
181
- // Backwards-compatible BulkDatabase2: the mobx-reactive flavor. All behavior lives in BulkDatabaseBase;
182
- // this just supplies mobx reactivity and the default (getFileStorageNested2) storage backend, so the
183
- // sync reads (getSingleFieldSync / getColumnSync) stay observable for mobx components.
176
+ // Backwards-compatible BulkDatabase2: the mobx-reactive flavor. All behavior lives in BulkDatabaseBase; this just supplies mobx reactivity and the default (getFileStorageNested2) storage backend, so the sync reads (getSingleFieldSync / getColumnSync) stay observable for mobx components.
184
177
  export class BulkDatabase2<T extends { key: string }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
185
178
  constructor(name: string, config?: BulkDatabase2Config) {
186
179
  super(name, new MobxReactiveDeps(), getFileStorageNested2, config);
@@ -43,21 +43,16 @@ const MAX_INDEX_RELOAD_ATTEMPTS = 3;
43
43
  const FIRST_MERGE_BYTES = TARGET_FILE_BYTES / 2;
44
44
  const KEY_GROUP_BYTES = 800 * 1024 * 1024;
45
45
  const DUP_THRESHOLD = 0.4;
46
- // Whole-tier dedup short-circuit (after Pass 1): when the bulk tier is over this size AND the overall
47
- // key duplication fraction is over this threshold, fold every bulk file in one merge instead of the
48
- // per-key-group walk (which spaces merges 5 min apart — 16 h for 200 groups).
46
+ // Whole-tier dedup short-circuit (after Pass 1): when the bulk tier is over this size AND the overall key duplication fraction is over this threshold, fold every bulk file in one merge instead of the per-key-group walk (which spaces merges 5 min apart — 16 h for 200 groups).
49
47
  const DEDUP_TRIGGER_BYTES = 512 * 1024 * 1024;
50
48
  const DEDUP_TRIGGER_FRACTION = 0.5;
51
49
  const WRITE_FLUSH_FIRST_STEP_MS = 250;
52
- // Skip logs are throttled per collection: the background write path retries merges about once a second,
53
- // so an unthrottled log would repeat for the whole duration of another tab's merge.
50
+ // Skip logs are throttled per collection: the background write path retries merges about once a second, so an unthrottled log would repeat for the whole duration of another tab's merge.
54
51
  const MERGE_SKIP_LOG_INTERVAL_MS = 30 * 1000;
55
52
 
56
53
  export const bulkDatabase2Timing = {
57
54
  streamSealAgeMs: 10 * 60 * 60 * 1000,
58
- // Wait this long after the tab becomes visible before the first merge check, then check this often
59
- // afterwards. Tab being hidden cancels the timers (no merges while in background). Browser-only;
60
- // Node compactors (e.g. remoteFileServer) drive their own polling.
55
+ // Wait this long after the tab becomes visible before the first merge check, then check this often afterwards. Tab being hidden cancels the timers (no merges while in background). Browser-only; Node compactors (e.g. remoteFileServer) drive their own polling.
61
56
  visibleMergeIntervalMs: 5 * 60 * 1000,
62
57
  mergeSpacingMs: 5 * 60 * 1000,
63
58
  firstMergeTriggerFiles: 20,
@@ -66,8 +61,7 @@ export const bulkDatabase2Timing = {
66
61
  streamFoldTriggerBytes: 64 * 1024 * 1024,
67
62
  streamFileMaxBytes: 50 * 1024 * 1024,
68
63
  streamFoldHardLimitBytes: 768 * 1024 * 1024,
69
- // 0 = flush every write (Node — append is real and cheap); browser ramps to 15s to avoid rewriting
70
- // the whole stream file per write.
64
+ // 0 = flush every write (Node — append is real and cheap); browser ramps to 15s to avoid rewriting the whole stream file per write.
71
65
  writeFlushMaxDelayMs: isNode() ? 0 : 15 * 1000,
72
66
  fileSetPollIntervalMs: 30 * 60 * 1000,
73
67
  memoryFlushHeapBytes: 5 * 1024 * 1024 * 1024,
@@ -82,14 +76,12 @@ function fmtBytes(n: number): string {
82
76
  return (n / 1024 / 1024 / 1024).toFixed(2) + "GB";
83
77
  }
84
78
 
85
- // Reactivity seam (no mobx dependency in this file). The mobx subclass supplies a ReactiveDeps that
86
- // maps signal strings to its own dependency tracking; non-reactive callers pass noopReactiveDeps.
79
+ // Reactivity seam (no mobx dependency in this file). The mobx subclass supplies a ReactiveDeps that maps signal strings to its own dependency tracking; non-reactive callers pass noopReactiveDeps.
87
80
  export interface ReactiveDeps {
88
81
  observe(signal: string): void;
89
82
  invalidate(signal: string): void;
90
83
  batch(fn: () => void): void;
91
- // Optional — lets writes skip per-key invalidation for rows nothing is watching. Undefined =
92
- // "assume watched".
84
+ // Optional — lets writes skip per-key invalidation for rows nothing is watching. Undefined = "assume watched".
93
85
  isObserved?(signal: string): boolean;
94
86
  }
95
87
 
@@ -117,9 +109,7 @@ export type MergeSkipReason =
117
109
  // compact() only: the collection has no files on disk.
118
110
  | "nothingToMerge";
119
111
 
120
- // What a compact()/tryMergeNow() call did. skipReason is set when the pass never ran; for the lock
121
- // reasons, lockHolderId/lockExpiresInMs report who holds the lock and how long until it goes stale
122
- // (so a scheduler knows when retrying could succeed).
112
+ // What a compact()/tryMergeNow() call did. skipReason is set when the pass never ran; for the lock reasons, lockHolderId/lockExpiresInMs report who holds the lock and how long until it goes stale (so a scheduler knows when retrying could succeed).
123
113
  export type MergeAttemptResult = {
124
114
  merged: boolean;
125
115
  skipReason?: MergeSkipReason;
@@ -173,11 +163,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
173
163
  private config: BulkDatabase2Config = {},
174
164
  ) { }
175
165
 
176
- // The reader (and the background machinery that rides along with it) must NOT be set up just
177
- // because the collection was constructed. Many collections are constructed and never touched, and
178
- // in Node a merge tick would poke the storage factory (e.g. indexedDB) and throw. We build it
179
- // lazily on first access to `this.reader`: every read, write, and merge goes through the reader,
180
- // while pure construction never touches it.
166
+ // The reader (and the background machinery that rides along with it) must NOT be set up just because the collection was constructed. Many collections are constructed and never touched, and in Node a merge tick would poke the storage factory (e.g. indexedDB) and throw. We build it lazily on first access to `this.reader`: every read, write, and merge goes through the reader, while pure construction never touches it.
181
167
  private _reader: BulkDatabaseReader<T> | undefined;
182
168
  private get reader(): BulkDatabaseReader<T> {
183
169
  if (this._reader) return this._reader;
@@ -213,11 +199,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
213
199
  BulkDatabaseBase.startMemoryWatchdog();
214
200
  }
215
201
 
216
- // Every `visibleMergeIntervalMs` of being-visible time, run a merge check. First check fires
217
- // `visibleMergeIntervalMs` after the tab becomes visible (not on initial load — gives the user a
218
- // moment of session commitment before we start churning the FS), then every interval after. Tab
219
- // hiding cancels the timers; visible-again restarts. Node has no document — there we consider
220
- // ourselves always-visible and just install the interval directly.
202
+ // Every `visibleMergeIntervalMs` of being-visible time, run a merge check. First check fires `visibleMergeIntervalMs` after the tab becomes visible (not on initial load — gives the user a moment of session commitment before we start churning the FS), then every interval after. Tab hiding cancels the timers; visible-again restarts. Node has no document — there we consider ourselves always-visible and just install the interval directly.
221
203
  private setupVisibilityMergeCheck(): void {
222
204
  let firstTimer: ReturnType<typeof setTimeout> | undefined;
223
205
  let intervalTimer: ReturnType<typeof setInterval> | undefined;
@@ -256,13 +238,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
256
238
  private streamFileName: string | undefined;
257
239
  private currentStreamFileName: string | undefined;
258
240
  private currentStreamFileBytes = 0;
259
- // In-process re-entry guard: if a merge is running, additional triggers (visibility timer, writes,
260
- // tryMergeNow calls) return immediately instead of queueing. Keeps the visibility-timer firings
261
- // from stacking behind a long merge.
241
+ // In-process re-entry guard: if a merge is running, additional triggers (visibility timer, writes, tryMergeNow calls) return immediately instead of queueing. Keeps the visibility-timer firings from stacking behind a long merge.
262
242
  private mergeInFlight = false;
263
243
  private lastMergeSkipLogMs = 0;
264
- // Running counters of stream-tier rows + bytes on disk. Seeded from each LoadedIndex build, then
265
- // incremented per flush so the fold-trigger checks current data without an extra directory listing.
244
+ // Running counters of stream-tier rows + bytes on disk. Seeded from each LoadedIndex build, then incremented per flush so the fold-trigger checks current data without an extra directory listing.
266
245
  private streamRowsOnDisk = 0;
267
246
  private streamBytesOnDisk = 0;
268
247
 
@@ -343,8 +322,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
343
322
  return idx;
344
323
  }
345
324
 
346
- // Coalescing rebuild loop: triggers during a build set rebuildDirty so the loop iterates once more
347
- // when it finishes — N rapid triggers cause at most ONE extra rebuild after the current one ends.
325
+ // Coalescing rebuild loop: triggers during a build set rebuildDirty so the loop iterates once more when it finishes — N rapid triggers cause at most ONE extra rebuild after the current one ends.
348
326
  private triggerRebuild(opts: { dropStaleFallback?: boolean } = {}): Promise<void> {
349
327
  if (opts.dropStaleFallback) this.rebuildOptions.dropStaleFallback = true;
350
328
  if (this.rebuildPromise) {
@@ -449,8 +427,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
449
427
  const stamped = rows.map(row => ({ time: getTimeUnique(), row }));
450
428
  const framed = frameRows(stamped);
451
429
 
452
- // Big batches skip the stream and become a bulk file directly — streaming thousands of rows
453
- // one frame at a time would just churn.
430
+ // Big batches skip the stream and become a bulk file directly — streaming thousands of rows one frame at a time would just churn.
454
431
  if (entries.length >= ROLLOVER_ROWS || framed.length >= ROLLOVER_BYTES) {
455
432
  await this.writeBulkFile(rows);
456
433
  return;
@@ -480,9 +457,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
480
457
  void this.maybeMerge({ onlyIfStreamHeavy: true });
481
458
  }
482
459
 
483
- // Coalesce stream appends on a ramping per-collection schedule (the browser rewrites the whole
484
- // file per append). The first write after a lull flushes immediately so a single edit-then-close
485
- // is saved at once; sustained writes ramp toward writeFlushMaxDelayMs.
460
+ // Coalesce stream appends on a ramping per-collection schedule (the browser rewrites the whole file per append). The first write after a lull flushes immediately so a single edit-then-close is saved at once; sustained writes ramp toward writeFlushMaxDelayMs.
486
461
  private async streamAppend(framed: Buffer, rows: number): Promise<void> {
487
462
  this.pendingAppends.push({ framed, rows });
488
463
  const max = bulkDatabase2Timing.writeFlushMaxDelayMs;
@@ -524,8 +499,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
524
499
  }
525
500
  // On failure the throw leaves pendingAppends intact so a later flush retries.
526
501
  await storage.append(fileName, combined);
527
- // New entries appended during the await are after `batch` — removing the front is exactly the
528
- // flushed set.
502
+ // New entries appended during the await are after `batch` — removing the front is exactly the flushed set.
529
503
  this.pendingAppends.splice(0, batch.length);
530
504
  this.streamBytesOnDisk += combined.length;
531
505
  for (const p of batch) this.streamRowsOnDisk += p.rows;
@@ -539,8 +513,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
539
513
  }
540
514
 
541
515
  private getStreamFileName(): string {
542
- // Seal our current file once it ages past the seal threshold — no file is ever appended to past
543
- // its seal age, which lets a consolidation safely fold it once aged.
516
+ // Seal our current file once it ages past the seal threshold — no file is ever appended to past its seal age, which lets a consolidation safely fold it once aged.
544
517
  if (this.streamFileName) {
545
518
  const info = parseStreamFileName(this.streamFileName);
546
519
  if (info && Date.now() - info.timestamp >= bulkDatabase2Timing.streamSealAgeMs) this.streamFileName = undefined;
@@ -586,9 +559,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
586
559
  private async listFiles(): Promise<{ bulkFiles: BulkFileInfo[]; streamFiles: StreamFileInfo[] }> {
587
560
  const storage = await this.storage();
588
561
  const names = await storage.getKeys();
589
- // A consumed-merge input is hidden from reads the instant its deletion marker exists — that's
590
- // what stops it from being re-read and re-merged. Physical deletion + marker cleanup runs on a
591
- // throttle (not inline) so a read isn't slowed by it.
562
+ // A consumed-merge input is hidden from reads the instant its deletion marker exists — that's what stops it from being re-read and re-merged. Physical deletion + marker cleanup runs on a throttle (not inline) so a read isn't slowed by it.
592
563
  const markers = await readDeleteMarkers(storage, names);
593
564
  const excluded = markerExclusions(markers);
594
565
  if (markers.length) void this.processMarkers();
@@ -607,9 +578,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
607
578
  return { bulkFiles, streamFiles };
608
579
  }
609
580
 
610
- // Throttled marker housekeeping: delete inputs whose replacement outputs have landed (or whose
611
- // marker has aged out) and retire fulfilled markers. Triggered from listFiles, so it's gated to
612
- // avoid a delete storm under heavy reads.
581
+ // Throttled marker housekeeping: delete inputs whose replacement outputs have landed (or whose marker has aged out) and retire fulfilled markers. Triggered from listFiles, so it's gated to avoid a delete storm under heavy reads.
613
582
  private processMarkers = throttleFunction(30 * 1000, async () => {
614
583
  const storage = await this.storage();
615
584
  const names = await storage.getKeys();
@@ -630,10 +599,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
630
599
  void this.maybeMerge();
631
600
  }
632
601
 
633
- // ── merge policy ─────────────────────────────────────────────────────────────────────────────────
634
- // Called both periodically (visibility timer) and on writes (with the streamNeedsFold gate so a
635
- // small write doesn't drag the merge through). The mergeInFlight check in tryMergeNow skips any
636
- // call that arrives while a previous one is still running — so 5-min timer ticks don't pile up.
602
+ // ── merge policy ───────────────────────────────────────────────────────────────────────────────── Called both periodically (visibility timer) and on writes (with the streamNeedsFold gate so a small write doesn't drag the merge through). The mergeInFlight check in tryMergeNow skips any call that arrives while a previous one is still running — so 5-min timer ticks don't pile up.
637
603
  private async maybeMerge(opts: { onlyIfStreamHeavy?: boolean } = {}): Promise<void> {
638
604
  if (!await this.automaticCompactionAllowed()) return;
639
605
  if (opts.onlyIfStreamHeavy && !this.streamNeedsFold()) return;
@@ -644,8 +610,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
644
610
  }
645
611
  }
646
612
 
647
- // Build the "we didn't merge" result and log it (throttled — see MERGE_SKIP_LOG_INTERVAL_MS). For
648
- // lock reasons the log and result include the holder and time until the lock goes stale.
613
+ // Build the "we didn't merge" result and log it (throttled — see MERGE_SKIP_LOG_INTERVAL_MS). For lock reasons the log and result include the holder and time until the lock goes stale.
649
614
  private async mergeSkip(reason: MergeSkipReason): Promise<MergeAttemptResult> {
650
615
  let lock: MergeLockInfo | undefined;
651
616
  if (reason === "tabLockHeld") {
@@ -661,8 +626,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
661
626
  return { merged: false, skipReason: reason, lockHolderId: lock?.holderId, lockExpiresInMs: lock?.expiresInMs };
662
627
  }
663
628
 
664
- // Acquire both merge locks (same-tab mergeInFlight, cross-tab localStorage, cross-process file),
665
- // run the pass, release. Skips (with the reason) instead of waiting when any of them is held.
629
+ // Acquire both merge locks (same-tab mergeInFlight, cross-tab localStorage, cross-process file), run the pass, release. Skips (with the reason) instead of waiting when any of them is held.
666
630
  private async runLockedMerge(run: () => Promise<MergeAttemptResult>): Promise<MergeAttemptResult> {
667
631
  if (this.mergeInFlight) return await this.mergeSkip("mergeInFlight");
668
632
  if (!tryAcquireMergeLock(this.name, writerId)) return await this.mergeSkip("tabLockHeld");
@@ -684,8 +648,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
684
648
  }
685
649
  }
686
650
 
687
- // The background path (maybeMerge) goes through this so write bursts coalesce; explicit callers use
688
- // tryMergeNow directly and get the per-call result.
651
+ // The background path (maybeMerge) goes through this so write bursts coalesce; explicit callers use tryMergeNow directly and get the per-call result.
689
652
  private tryMergeThrottled = throttleFunction(1000, async () => {
690
653
  await this.tryMergeNow();
691
654
  });
@@ -704,8 +667,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
704
667
  this.streamFileName = undefined;
705
668
  const { bulkFiles, streamFiles } = await this.listFiles();
706
669
  if (bulkFiles.length + streamFiles.length < 1) return await this.mergeSkip("nothingToMerge");
707
- // compact() folds every file → no older data survives outside it → surviving tombstones
708
- // can be dropped (nothing left to suppress).
670
+ // compact() folds every file → no older data survives outside it → surviving tombstones can be dropped (nothing left to suppress).
709
671
  const merged = await this.mergeFileSet(bulkFiles, streamFiles, true);
710
672
  return { merged };
711
673
  });
@@ -751,9 +713,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
751
713
  }
752
714
  }
753
715
 
754
- // A bulk file that won't load is either an in-progress write (recent) or a crashed partial write
755
- // (stale). Warn while recent, delete once clearly abandoned — deleting is safe because the write
756
- // protocol always lands the replacement before removing the file it supersedes.
716
+ // A bulk file that won't load is either an in-progress write (recent) or a crashed partial write (stale). Warn while recent, delete once clearly abandoned — deleting is safe because the write protocol always lands the replacement before removing the file it supersedes.
757
717
  private async handleUnreadableFile(file: BulkFileInfo, message: string): Promise<void> {
758
718
  const ageMs = Date.now() - file.timestamp;
759
719
  if (ageMs > STALE_DELETE_MS) {
@@ -769,14 +729,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
769
729
  console.warn(`${this.name}: skipping unreadable bulk file ${file.fileName} (recent - may be in-progress): ${message}`);
770
730
  }
771
731
 
772
- // The one merge primitive. Reads + plans + writes outputs before deleting any input, so a crash
773
- // leaves duplicates (next merge dedupes) rather than a gap. After the file set changes on disk,
774
- // we trigger an index rebuild + atomic swap; once swap completes, the consumed files' block-cache
775
- // entries are evicted (no consumer can ask for them now).
776
- //
777
- // Serialized: foldOwnStream + merge(timeLo, timeHi) + testMerge's runMerge all hit this; the lock
778
- // covers tryMergeNow/compact but those two paths bypass it. runInSerial queues so they never
779
- // collide on the file set.
732
+ // The one merge primitive. Reads + plans + writes outputs before deleting any input, so a crash leaves duplicates (next merge dedupes) rather than a gap. After the file set changes on disk, we trigger an index rebuild + atomic swap; once swap completes, the consumed files' block-cache entries are evicted (no consumer can ask for them now).
733
+ //
734
+ // Serialized: foldOwnStream + merge(timeLo, timeHi) + testMerge's runMerge all hit this; the lock covers tryMergeNow/compact but those two paths bypass it. runInSerial queues so they never collide on the file set.
780
735
  private mergeFileSet = runInSerial(async (bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false, forceDeleteStreams = false): Promise<boolean> => {
781
736
  this.reader.beginCompaction();
782
737
  try {
@@ -790,11 +745,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
790
745
  const storage = await this.storage();
791
746
  const timestamp = nextFileTime();
792
747
 
793
- // The caller's bulkFiles came from a `listFiles()` snapshot. Between then and now any file
794
- // could be gone (another tab's deferred delete, manual cleanup). The subCaches reader for a
795
- // deleted file is silently stale — its in-memory metadata still resolves, but the very next
796
- // getRange against disk throws. Re-verify existence up-front and drop the cache entry for
797
- // anything that vanished, so we don't even try to plan around a ghost file.
748
+ // The caller's bulkFiles came from a `listFiles()` snapshot. Between then and now any file could be gone (another tab's deferred delete, manual cleanup). The subCaches reader for a deleted file is silently stale — its in-memory metadata still resolves, but the very next getRange against disk throws. Re-verify existence up-front and drop the cache entry for anything that vanished, so we don't even try to plan around a ghost file.
798
749
  const verifiedBulkFiles = (await Promise.all(bulkFiles.map(async f => {
799
750
  try {
800
751
  const info = await storage.getInfo(f.fileName);
@@ -835,8 +786,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
835
786
  // expanded console.group so a merge takes one collapsible block, not a
836
787
  // screenful (and never the per-file name dump it used to spew).
837
788
  const steps: string[] = [];
838
- // Each step records the time spent since the previous step in blue at the front, so callers can
839
- // just describe what they did and not bother measuring/formatting elapsed times themselves.
789
+ // Each step records the time spent since the previous step in blue at the front, so callers can just describe what they did and not bother measuring/formatting elapsed times themselves.
840
790
  let lastStepMs = mergeStartMs;
841
791
  const log = (line: string) => {
842
792
  const now = Date.now();
@@ -879,36 +829,25 @@ export class BulkDatabaseBase<T extends { key: string }> {
879
829
  for (const line of steps) console.log(line);
880
830
  console.groupEnd();
881
831
 
882
- // Only the sources runPlannedMerge actually used can be retired — a source it dropped mid-plan
883
- // (file gone, corruption) still holds data we couldn't merge in, so leave it on disk and let a
884
- // future merge re-attempt it.
832
+ // Only the sources runPlannedMerge actually used can be retired — a source it dropped mid-plan (file gone, corruption) still holds data we couldn't merge in, so leave it on disk and let a future merge re-attempt it.
885
833
  const usedConsumedBulk = consumedBulk.filter(f => mergeResult.usedSourceNames.has(f.fileName));
886
834
  const usedStreamFiles = streamFiles.filter(f => mergeResult.usedSourceNames.has(f.fileName) || mergeResult.usedSourceNames.has("(streams)"));
887
- // A stream may still be appended to after we read it (the writer hasn't sealed/moved on); only
888
- // retire streams canDeleteStream clears — the rest stay live and a later merge re-folds them
889
- // once sealed. Bulk files are immutable, so a used one is always safe to retire.
835
+ // A stream may still be appended to after we read it (the writer hasn't sealed/moved on); only retire streams canDeleteStream clears — the rest stay live and a later merge re-folds them once sealed. Bulk files are immutable, so a used one is always safe to retire.
890
836
  const deletableStreams: string[] = [];
891
837
  for (const f of usedStreamFiles) {
892
838
  if (await this.canDeleteStream(f, Date.now(), streamData.sizes, forceDeleteStreams)) deletableStreams.push(f.fileName);
893
839
  }
894
840
  const deleteFiles = [...usedConsumedBulk.map(f => f.fileName), ...deletableStreams];
895
841
 
896
- // Write a deletion marker instead of deleting inline. From the next read on, listFiles hides
897
- // these inputs (so they're never re-merged — the loop fix), and processMarkers removes them
898
- // physically once the outputs have landed or the marker ages out. Marker BEFORE the rebuild so
899
- // the freshly-swapped index already excludes the retired inputs.
842
+ // Write a deletion marker instead of deleting inline. From the next read on, listFiles hides these inputs (so they're never re-merged — the loop fix), and processMarkers removes them physically once the outputs have landed or the marker ages out. Marker BEFORE the rebuild so the freshly-swapped index already excludes the retired inputs.
900
843
  if (deleteFiles.length) await writeDeleteMarker(storage, { deleteFiles, replacedBy: outNames });
901
844
 
902
- // Rebuild + swap so the index sees the new outputs and drops the retired inputs. Block-cache
903
- // eviction for the dropped inputs happens here (they're no longer in the new index's fileSet).
845
+ // Rebuild + swap so the index sees the new outputs and drops the retired inputs. Block-cache eviction for the dropped inputs happens here (they're no longer in the new index's fileSet).
904
846
  await this.triggerRebuild();
905
847
  return newNames.length > 0 || carriedDeletes > 0;
906
848
  }
907
849
 
908
- // A stream is safe to delete iff no writer will append to it again: it's aged past the seal age
909
- // (writer has provably switched files) OR cross-tab sync is on AND its size didn't change while
910
- // we read it. Else leave it — the data is also in the new bulk file; a later merge deletes it
911
- // once aged.
850
+ // A stream is safe to delete iff no writer will append to it again: it's aged past the seal age (writer has provably switched files) OR cross-tab sync is on AND its size didn't change while we read it. Else leave it — the data is also in the new bulk file; a later merge deletes it once aged.
912
851
  private async canDeleteStream(f: StreamFileInfo, now: number, sizes: Map<string, number>, force = false): Promise<boolean> {
913
852
  if (now - f.timestamp >= bulkDatabase2Timing.streamSealAgeMs) return true;
914
853
  if (!isSyncSupported() && !force) return false;
@@ -933,11 +872,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
933
872
  }
934
873
 
935
874
  // The merge policy (two passes, spaced by mergeSpacingMs):
936
- // 1) Consolidate recent fragmentation: take newest files up to ~FIRST_MERGE_BYTES; merge them
937
- // into one when they fragment or span too wide a time range.
938
- // 2) Key-stratify: walk all keys in ~KEY_GROUP_BYTES groups; rewrite groups whose duplicate-key
939
- // fraction passes DUP_THRESHOLD, highest first.
940
- // Use tryMerge instead
875
+ // 1) Consolidate recent fragmentation: take newest files up to ~FIRST_MERGE_BYTES; merge them into one when they fragment or span too wide a time range.
876
+ // 2) Key-stratify: walk all keys in ~KEY_GROUP_BYTES groups; rewrite groups whose duplicate-key fraction passes DUP_THRESHOLD, highest first. Use tryMerge instead
941
877
  private async testMergeINTERNAL_DO_NOT_CALL(): Promise<boolean> {
942
878
  let merged = false;
943
879
  await this.flushPending();
@@ -947,8 +883,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
947
883
  return true;
948
884
  };
949
885
 
950
- // Hard stream limit: a stream this big makes every read pull a huge file → fold ALL of it now,
951
- // force-delete (canDeleteStream still requires size-stable, so an active writer never loses data).
886
+ // Hard stream limit: a stream this big makes every read pull a huge file → fold ALL of it now, force-delete (canDeleteStream still requires size-stable, so an active writer never loses data).
952
887
  {
953
888
  const { streamFiles } = await this.listFiles();
954
889
  if (streamFiles.length) {
@@ -962,8 +897,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
962
897
  }
963
898
  }
964
899
 
965
- // Pass 1: consolidate recent. Only seal when cross-tab sync can fold recent streams — in Node
966
- // canDeleteStream needs them aged anyway, so sealing would just fragment streams every pass.
900
+ // Pass 1: consolidate recent. Only seal when cross-tab sync can fold recent streams — in Node canDeleteStream needs them aged anyway, so sealing would just fragment streams every pass.
967
901
  const foldRecentStreams = isSyncSupported();
968
902
  if (foldRecentStreams) {
969
903
  syncBroadcastSeal(this.name);
@@ -1004,9 +938,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
1004
938
  }
1005
939
  }
1006
940
 
1007
- // Whole-tier dedup short-circuit (between Pass 1 and the per-group Pass 2): when the bulk tier
1008
- // is big enough AND mostly duplicates, fold every bulk file in one merge. Avoids the per-group
1009
- // walk's 5-min spacing × N-groups latency.
941
+ // Whole-tier dedup short-circuit (between Pass 1 and the per-group Pass 2): when the bulk tier is big enough AND mostly duplicates, fold every bulk file in one merge. Avoids the per-group walk's 5-min spacing × N-groups latency.
1010
942
  {
1011
943
  const { bulkFiles } = await this.listFiles();
1012
944
  if (bulkFiles.length >= 2) {
@@ -1031,8 +963,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
1031
963
  }
1032
964
  }
1033
965
 
1034
- // Pass 2: key-stratified deduplication. Disjoint key ranges → one group's merge doesn't
1035
- // change another's duplication; re-select each group's files at merge time (set shifts).
966
+ // Pass 2: key-stratified deduplication. Disjoint key ranges → one group's merge doesn't change another's duplication; re-select each group's files at merge time (set shifts).
1036
967
  const groups = await this.findDuplicateGroups();
1037
968
  for (const g of groups) {
1038
969
  const { bulkFiles } = await this.listFiles();
@@ -1131,9 +1062,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
1131
1062
  return this.reader.isColumnLoadedSync(column);
1132
1063
  }
1133
1064
 
1134
- // Reactive: true while a merge is rewriting this collection's files. Use this in UI to show a
1135
- // "compacting…" indicator. Counts both background merges (maybeMerge) and explicit compact/merge
1136
- // calls. Becomes false once the new index is swapped in (the deferred-delete window is NOT counted).
1065
+ // Reactive: true while a merge is rewriting this collection's files. Use this in UI to show a "compacting…" indicator. Counts both background merges (maybeMerge) and explicit compact/merge calls. Becomes false once the new index is swapped in (the deferred-delete window is NOT counted).
1137
1066
  public isCompactingSync(): boolean {
1138
1067
  return this.reader.isCompactingSync();
1139
1068
  }
@@ -1228,8 +1157,7 @@ export type BulkFileEntry = {
1228
1157
  bytes: number;
1229
1158
  // Filesystem mtime (ms since epoch) — 0 if the storage layer didn't return one.
1230
1159
  lastModified: number;
1231
- // Lazy: pulled from the cached sub-reader for bulk (free if loaded), or from a parse of the stream
1232
- // file (small — tier-0 size-capped, also cached). Call only when you need the per-file detail.
1160
+ // Lazy: pulled from the cached sub-reader for bulk (free if loaded), or from a parse of the stream file (small — tier-0 size-capped, also cached). Call only when you need the per-file detail.
1233
1161
  getDetails: () => Promise<BulkFileDetails>;
1234
1162
  };
1235
1163
  export type BulkFileInfoListing = { files: BulkFileEntry[]; count: number; totalBytes: number };