sliftutils 1.4.46 → 1.4.48

package/index.d.ts

@@ -715,7 +715,7 @@ declare module "sliftutils/render-utils/colors" {
 }
 
 declare module "sliftutils/render-utils/mobxTyped" {
-    export { observable, runInAction, computed, autorun } from "mobx";
+    export { observable, runInAction, computed, autorun, onBecomeObserved, onBecomeUnobserved } from "mobx";
     export declare function configureMobxNextFrameScheduler(): void;
 
 }
@@ -763,7 +763,7 @@ declare module "sliftutils/render-utils/observer" {
 }
 
 declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
-    import { BulkDatabaseBase, BulkDatabase2Config } from "./BulkDatabaseBase";
+    import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config } from "./BulkDatabaseBase";
     export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
     export type { ReactiveDeps, StorageFactory, BulkDatabase2Config } from "./BulkDatabaseBase";
     /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
@@ -853,6 +853,17 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
         isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
         /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
         isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+        /**
+         * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
+         * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;
+         * returns true if the backend can't tell.
+         */
+        isKeyWatched(key: string): boolean;
+        /**
+         * Drop all of this collection's in-memory loaded caches and re-trigger every watcher, which re-requests
+         * and reloads from disk. Pending un-flushed writes are kept. Per-collection.
+         */
+        reloadFromDisk(): void;
         /** The columns present on disk and their byte sizes (no row data read). */
         getColumnInfo(): Promise<BulkColumnInfo[]>;
         /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
@@ -901,6 +912,15 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
          * most callers want compact() or tryMergeNow(). */
         merge(timeLo: number, timeHi: number): Promise<void>;
     }
+    export declare class MobxReactiveDeps implements ReactiveDeps {
+        private boxes;
+        private observed;
+        private box;
+        observe(signal: string): void;
+        invalidate(signal: string): void;
+        batch(fn: () => void): void;
+        isObserved(signal: string): boolean;
+    }
     export declare class BulkDatabase2<T extends {
         key: string;
     }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
@@ -924,11 +944,15 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         streamFoldHardLimitBytes: number;
         writeFlushMaxDelayMs: number;
         fileSetPollIntervalMs: number;
+        memoryFlushHeapBytes: number;
+        memoryFlushMinCollectionBytes: number;
+        memoryFlushThrottleMs: number;
     };
     export interface ReactiveDeps {
         observe(signal: string): void;
         invalidate(signal: string): void;
         batch(fn: () => void): void;
+        isObserved?(signal: string): boolean;
     }
     export declare const noopReactiveDeps: ReactiveDeps;
     export type StorageFactory = (path: string) => Promise<FileStorage>;
@@ -943,6 +967,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private storageFactory;
         private config;
         constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+        private static liveInstances;
+        private static memoryWatchdogStarted;
+        private static lastMemoryFlushMs;
+        private static startMemoryWatchdog;
+        static checkMemoryPressure(usedHeapBytes: number): void;
         private pendingAppends;
         private flushTimer;
         private flushChain;
@@ -963,6 +992,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private columnCache;
         private readerKeys;
         private loadedFileSet;
+        private loadedTotalBytes;
         private readerEpoch;
         private fileSetPollTimer;
         private bulkReaderCache;
@@ -980,6 +1010,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private lastMergeCheck;
         private getStreamFileName;
         private invalidateOverlay;
+        isKeyWatched(key: string): boolean;
         private isLiveNow;
         private invalidateSignal;
         private flushSignals;
@@ -993,6 +1024,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private clearReaderState;
         private resetReader;
         private reloadReader;
+        reloadFromDisk(): void;
         private readWithReload;
         private pollFileSet;
         write(entry: T): Promise<void>;
@@ -1109,7 +1141,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseFormat" {
     export declare const EMPTY_BUFFER: Buffer;
     export declare const ABSENT: unique symbol;
     export declare const TARGET_FILE_BYTES: number;
-    export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): Buffer[];
+    export interface BuiltFile {
+        buffer: Buffer;
+        minKey: string;
+        maxKey: string;
+        rowCount: number;
+    }
+    export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): BuiltFile[];
     export type BaseBulkDatabaseReader = {
         rowCount: number;
         totalBytes: number;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.46",
+    "version": "1.4.48",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/render-utils/mobxTyped.d.ts

@@ -1,2 +1,2 @@
-export { observable, runInAction, computed, autorun } from "mobx";
+export { observable, runInAction, computed, autorun, onBecomeObserved, onBecomeUnobserved } from "mobx";
 export declare function configureMobxNextFrameScheduler(): void;

package/render-utils/mobxTyped.ts

@@ -3,7 +3,7 @@ import { batchFunction } from "socket-function/src/batching";
 
 // Re-export the core mobx primitives so downstream packages (which may have their own duplicate mobx
 // in node_modules) can share THIS mobx instance, otherwise reactivity doesn't cross package boundaries.
-export { observable, runInAction, computed, autorun } from "mobx";
+export { observable, runInAction, computed, autorun, onBecomeObserved, onBecomeUnobserved } from "mobx";
 export function configureMobxNextFrameScheduler() {
     // NOTE: This makes a big difference if we do await calls in a loop which mutates observable state. BUT... we should probably just do those await calls before the loop?
     let batchReactionScheduler = batchFunction({

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -1,4 +1,4 @@
-import { BulkDatabaseBase, BulkDatabase2Config } from "./BulkDatabaseBase";
+import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config } from "./BulkDatabaseBase";
 export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
 export type { ReactiveDeps, StorageFactory, BulkDatabase2Config } from "./BulkDatabaseBase";
 /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
@@ -88,6 +88,17 @@ export interface IBulkDatabase2<T extends {
     isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
     /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
     isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+    /**
+     * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
+     * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;
+     * returns true if the backend can't tell.
+     */
+    isKeyWatched(key: string): boolean;
+    /**
+     * Drop all of this collection's in-memory loaded caches and re-trigger every watcher, which re-requests
+     * and reloads from disk. Pending un-flushed writes are kept. Per-collection.
+     */
+    reloadFromDisk(): void;
     /** The columns present on disk and their byte sizes (no row data read). */
     getColumnInfo(): Promise<BulkColumnInfo[]>;
     /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
@@ -136,6 +147,15 @@ export interface IBulkDatabase2<T extends {
      * most callers want compact() or tryMergeNow(). */
     merge(timeLo: number, timeHi: number): Promise<void>;
 }
+export declare class MobxReactiveDeps implements ReactiveDeps {
+    private boxes;
+    private observed;
+    private box;
+    observe(signal: string): void;
+    invalidate(signal: string): void;
+    batch(fn: () => void): void;
+    isObserved(signal: string): boolean;
+}
 export declare class BulkDatabase2<T extends {
     key: string;
 }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -1,5 +1,5 @@
 import { getFileStorageNested2 } from "../FileFolderAPI";
-import { observable, runInAction } from "../../render-utils/mobxTyped";
+import { observable, runInAction, onBecomeObserved, onBecomeUnobserved } from "../../render-utils/mobxTyped";
 import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config } from "./BulkDatabaseBase";
 
 export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
@@ -74,6 +74,19 @@ export interface IBulkDatabase2<T extends { key: string }> {
     /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
     isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
 
+    /**
+     * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
+     * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;
+     * returns true if the backend can't tell.
+     */
+    isKeyWatched(key: string): boolean;
+
+    /**
+     * Drop all of this collection's in-memory loaded caches and re-trigger every watcher, which re-requests
+     * and reloads from disk. Pending un-flushed writes are kept. Per-collection.
+     */
+    reloadFromDisk(): void;
+
     /** The columns present on disk and their byte sizes (no row data read). */
     getColumnInfo(): Promise<BulkColumnInfo[]>;
     /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
@@ -121,13 +134,18 @@ export interface IBulkDatabase2<T extends { key: string }> {
 // observer/autorun that calls it tracks that box) and invalidate() bumps it (so those reactions re-run).
 // This reproduces the fine-grained per-key + load-version reactivity the class had when it used
 // observable.map/observable.box directly, while keeping all that logic in the mobx-free base.
-class MobxReactiveDeps implements ReactiveDeps {
+export class MobxReactiveDeps implements ReactiveDeps {
     private boxes = new Map<string, { get(): number; set(value: number): void }>();
+    // Signals that currently have at least one observer, kept current via mobx's onBecomeObserved/
+    // onBecomeUnobserved hooks. Lets isObserved() answer "is anything watching this row" cheaply.
+    private observed = new Set<string>();
     private box(signal: string) {
         let box = this.boxes.get(signal);
         if (!box) {
             box = observable.box(0);
             this.boxes.set(signal, box);
+            onBecomeObserved(box, () => this.observed.add(signal));
+            onBecomeUnobserved(box, () => this.observed.delete(signal));
         }
         return box;
     }
@@ -141,6 +159,9 @@ class MobxReactiveDeps implements ReactiveDeps {
     batch(fn: () => void) {
         runInAction(fn);
     }
+    isObserved(signal: string) {
+        return this.observed.has(signal);
+    }
 }
 
 // Backwards-compatible BulkDatabase2: the mobx-reactive flavor. All behavior lives in BulkDatabaseBase;

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -12,11 +12,15 @@ export declare const bulkDatabase2Timing: {
     streamFoldHardLimitBytes: number;
     writeFlushMaxDelayMs: number;
     fileSetPollIntervalMs: number;
+    memoryFlushHeapBytes: number;
+    memoryFlushMinCollectionBytes: number;
+    memoryFlushThrottleMs: number;
 };
 export interface ReactiveDeps {
     observe(signal: string): void;
     invalidate(signal: string): void;
     batch(fn: () => void): void;
+    isObserved?(signal: string): boolean;
 }
 export declare const noopReactiveDeps: ReactiveDeps;
 export type StorageFactory = (path: string) => Promise<FileStorage>;
@@ -31,6 +35,11 @@ export declare class BulkDatabaseBase<T extends {
     private storageFactory;
     private config;
     constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+    private static liveInstances;
+    private static memoryWatchdogStarted;
+    private static lastMemoryFlushMs;
+    private static startMemoryWatchdog;
+    static checkMemoryPressure(usedHeapBytes: number): void;
     private pendingAppends;
     private flushTimer;
     private flushChain;
@@ -51,6 +60,7 @@ export declare class BulkDatabaseBase<T extends {
     private columnCache;
     private readerKeys;
     private loadedFileSet;
+    private loadedTotalBytes;
     private readerEpoch;
     private fileSetPollTimer;
     private bulkReaderCache;
@@ -68,6 +78,7 @@ export declare class BulkDatabaseBase<T extends {
     private lastMergeCheck;
     private getStreamFileName;
     private invalidateOverlay;
+    isKeyWatched(key: string): boolean;
     private isLiveNow;
     private invalidateSignal;
     private flushSignals;
@@ -81,6 +92,7 @@ export declare class BulkDatabaseBase<T extends {
     private clearReaderState;
     private resetReader;
     private reloadReader;
+    reloadFromDisk(): void;
     private readWithReload;
     private pollFileSet;
     write(entry: T): Promise<void>;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -34,6 +34,10 @@ const FILE_EXTENSION = ".bulk";
 const ROLLOVER_ROWS = 5000;
 const ROLLOVER_BYTES = 5 * 1024 * 1024;
 
+// How often the global memory-pressure watchdog samples the heap (the ACTION it may take is throttled
+// separately, see bulkDatabase2Timing.memoryFlushThrottleMs).
+const MEMORY_WATCHDOG_INTERVAL_MS = 60 * 1000;
+
 // An unreadable (corrupt/torn, not merely missing) file might be a write still in progress, so we
 // can't delete it on sight. Once it's been unreadable for longer than this (by its name timestamp),
 // no writer is plausibly still working on it, so we delete it. Until then we just warn.
@@ -104,6 +108,13 @@ export const bulkDatabase2Timing = {
     // We proactively re-list files this often; if the set changed under us (another tab/process merged) we
     // reload the index, so reads pick up the change even without first hitting a missing-file error.
     fileSetPollIntervalMs: 30 * 60 * 1000,
+    // Memory-pressure auto-flush (browser only; needs performance.memory). When the JS heap exceeds
+    // memoryFlushHeapBytes, at most once per memoryFlushThrottleMs, drop the in-memory observable state of
+    // every collection whose loaded data exceeds memoryFlushMinCollectionBytes (they reload lazily). Tuned
+    // so it does nothing for a healthy app (heap fine) or small collections.
+    memoryFlushHeapBytes: 1500 * 1024 * 1024,
+    memoryFlushMinCollectionBytes: 100 * 1024 * 1024,
+    memoryFlushThrottleMs: 15 * 60 * 1000,
 };
 
 // Marks a key as deleted in the in-memory overlay.
@@ -139,14 +150,19 @@ export interface ReactiveDeps {
     invalidate(signal: string): void;
     // Run a group of mutations + invalidations as one batch, so observers re-run at most once.
     batch(fn: () => void): void;
+    // Whether `signal` currently has any observer watching it. Optional — a backend that can't tell
+    // returns undefined (treated as "assume watched"). Lets writes skip per-key notification work for rows
+    // nothing is watching (see isWatchedSync).
+    isObserved?(signal: string): boolean;
 }
 
 // A non-reactive ReactiveDeps: sync reads still return current values, they just never trigger
-// re-renders. Use this when you don't need a UI to react to writes.
+// re-renders. Use this when you don't need a UI to react to writes. Nothing is ever observed.
 export const noopReactiveDeps: ReactiveDeps = {
     observe() { },
     invalidate() { },
     batch(fn) { fn(); },
+    isObserved() { return false; },
 };
 
 // Provides the FileStorage for a given path (the caller decides where data physically lives, so this
@@ -264,6 +280,50 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // stay correct even without first hitting a missing-file error.
         this.fileSetPollTimer = setInterval(() => void this.pollFileSet(), bulkDatabase2Timing.fileSetPollIntervalMs);
         (this.fileSetPollTimer as { unref?: () => void }).unref?.();
+        // Register for the global memory-pressure watchdog (browser only — needs performance.memory).
+        BulkDatabaseBase.liveInstances.add(this);
+        BulkDatabaseBase.startMemoryWatchdog();
+    }
+
+    // ---- memory-pressure watchdog (global, browser-only) ----
+    // When the JS heap gets large, the biggest contributors are usually a few collections whose in-memory
+    // observable state (loaded reader + base column/field caches) has grown. Rather than cap every
+    // collection all the time, we watch the heap and, only when it's actually under pressure, drop the
+    // observable state of the large collections (they reload lazily on demand). Throttled hard so it never
+    // thrashes a healthy app.
+    private static liveInstances = new Set<BulkDatabaseBase<{ key: string }>>();
+    private static memoryWatchdogStarted = false;
+    private static lastMemoryFlushMs = 0;
+    private static startMemoryWatchdog() {
+        if (BulkDatabaseBase.memoryWatchdogStarted) return;
+        BulkDatabaseBase.memoryWatchdogStarted = true;
+        const usedHeap = (): number | undefined => {
+            try { return (performance as unknown as { memory?: { usedJSHeapSize?: number } })?.memory?.usedJSHeapSize; }
+            catch { return undefined; }
+        };
+        if (typeof performance === "undefined" || usedHeap() === undefined) return; // no heap API (Node / non-Chromium)
+        const timer = setInterval(() => {
+            const used = usedHeap();
+            if (used !== undefined) BulkDatabaseBase.checkMemoryPressure(used);
+        }, MEMORY_WATCHDOG_INTERVAL_MS);
+        (timer as { unref?: () => void }).unref?.();
+    }
+    // If the heap is over the limit (and we haven't flushed recently), drop the in-memory observable state
+    // of every collection whose loaded data exceeds the size threshold, so it reloads lazily and frees the
+    // rest. Exposed (with the heap value injected) so it's callable/testable without the real timer.
+    public static checkMemoryPressure(usedHeapBytes: number): void {
+        if (usedHeapBytes < bulkDatabase2Timing.memoryFlushHeapBytes) return;
+        const now = Date.now();
+        if (now - BulkDatabaseBase.lastMemoryFlushMs < bulkDatabase2Timing.memoryFlushThrottleMs) return;
+        BulkDatabaseBase.lastMemoryFlushMs = now;
+        const flushed: string[] = [];
+        for (const db of BulkDatabaseBase.liveInstances) {
+            if (db.loadedTotalBytes > bulkDatabase2Timing.memoryFlushMinCollectionBytes) {
+                flushed.push(`${db.name} (${fmtBytes(db.loadedTotalBytes)})`);
+                db.reloadFromDisk();
+            }
+        }
+        if (flushed.length) console.log(`[bulk2] heap ${fmtBytes(usedHeapBytes)} over ${fmtBytes(bulkDatabase2Timing.memoryFlushHeapBytes)} — flushed observable state of ${flushed.length} large collection(s) to reclaim memory: ${flushed.join(", ")}`);
     }
 
     // ---- buffered stream writes (per collection) ----
@@ -341,6 +401,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // under us; we notice either when a read hits a deleted file (readWithReload) or via the periodic poll
     // (pollFileSet), and reload the index — cheap, since a build only reads headers + key columns.
     private loadedFileSet: Set<string> | undefined; // file names the current reader was built from
+    // Total (uncompressed) size of the data the current reader spans, as a proxy for this collection's
+    // in-memory footprint — used by the memory-pressure watchdog. 0 when no reader is loaded.
+    private loadedTotalBytes = 0;
     // Bumped every time the index is cleared/reloaded. A failed read captures it before reading and only
     // triggers a reload if it's unchanged afterward — so N concurrent failures coalesce onto ONE rebuild
     // (the rest see the bumped epoch, skip their own reload, and just await the in-flight one).
@@ -406,10 +469,20 @@ export class BulkDatabaseBase<T extends { key: string }> {
         this.dataGen++;
         if (columns === "all") this.columnCache.clear();
         else for (const c of columns) this.columnCache.delete(c);
-        this.invalidateSignal(key);
+        // The per-key signal only re-runs getSingleFieldObjSync watchers of THIS key; skip it when nothing
+        // is watching the row (a new watcher reads the current overlay anyway). The overlay-wide signal +
+        // column-cache drop above still cover whole-column watchers and read correctness.
+        if (this.deps.isObserved?.(key) ?? true) this.invalidateSignal(key);
         this.invalidateSignal(OVERLAY_SIGNAL);
     }
 
+    // Whether a row (key) is currently being watched — i.e. some reactive observer is subscribed to it via
+    // getSingleFieldObjSync / getSingleFieldSync. Useful for skipping per-row work when nothing's watching.
+    // Returns true if the backend can't tell (conservative).
+    public isKeyWatched(key: string): boolean {
+        return this.deps.isObserved?.(key) ?? true;
+    }
+
     // Whether `key` is currently a live key in the resolved view (overlay wins over disk). A write to a key
     // that's already live only changes the columns it sets; a write to an absent key (or one currently
     // deleted in the overlay) makes it appear in every column, so every column's cache must drop.
@@ -526,6 +599,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
         this.readerKeys = new Set(joined.keys);
         // The files this index was built from, so the poll can detect a concurrent merge changing the set.
         this.loadedFileSet = new Set([...bulkFiles.map(f => f.fileName), ...streamFiles.map(f => f.fileName)]);
+        this.loadedTotalBytes = joined.totalBytes; // footprint proxy for the memory-pressure watchdog
+
         // Evict cached sub-readers for files a merge removed, so the caches track the live file set.
         this.pruneFileCaches(bulkFiles, streamFiles);
 
@@ -588,6 +663,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         this.columnCache.clear();
         this.readerKeys = undefined;     // the next build repopulates it
         this.loadedFileSet = undefined;  // ditto
+        this.loadedTotalBytes = 0;       // nothing loaded ⇒ no footprint for the memory watchdog
         // The next build re-measures the stream; clear the estimate so a just-folded stream doesn't keep
         // looking "heavy" and re-trigger a fold before then.
         this.streamRowsOnDisk = 0;
@@ -620,6 +696,23 @@ export class BulkDatabaseBase<T extends { key: string }> {
         });
     }
 
+    // Drop ALL of this collection's in-memory loaded caches (the resolved reader, the sync base column /
+    // field caches and their stale fallbacks, the per-file decoded readers) and re-trigger every watcher,
+    // so they re-request and reload from disk. Unlike the automatic reloads, this is a genuine full clear
+    // (no stale-fallback served — watchers go through a real loading state). Pending un-flushed writes (the
+    // overlay) are KEPT, so nothing not-yet-on-disk is lost. Per-collection (this instance only).
+    public reloadFromDisk(): void {
+        this.deps.batch(() => {
+            this.clearReaderState();       // drops reader + base caches (→ stale) + column cache; bumps gens
+            this.staleBaseColumns.clear(); // and the stale fallbacks — a genuine full clear, not a swap
+            this.staleBaseFields.clear();
+            this.bulkReaderCache.clear();  // force re-decode from disk
+            this.streamReaderCache.clear();
+            this.invalidateSignal(LOAD_SIGNAL);
+            this.invalidateSignal(OVERLAY_SIGNAL);
+        });
+    }
+
     // Run a read against the loaded index; if a file vanished mid-read (a concurrent merge deleted it),
     // reload the index and retry — looping until it succeeds or we've tried MAX_INDEX_RELOAD_ATTEMPTS times.
     // Reloads coalesce: a failing read only resets the index if nobody has reset it since the read grabbed
@@ -836,9 +929,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const timestamp = nextFileTime();
         const now = Date.now();
         const times = rows.map(() => now);
-        for (const buffer of buildFileBuffer(rows, times)) {
+        for (const built of buildFileBuffer(rows, times)) {
             const name = newFileName(timestamp);
-            await storage.set(name, encodeCompressedBlocks(buffer));
+            await storage.set(name, encodeCompressedBlocks(built.buffer));
         }
         this.resetReader();
         void this.maybeMerge();
@@ -1168,7 +1261,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
             ...streamFiles.map(f => ({ name: f.fileName, size: streamData.sizes.get(f.fileName) ?? 0 })),
         ];
         const inTotal = inputs.reduce((a, f) => a + f.size, 0);
-        console.log(`${blue(this.name)} merge: reading ${inputs.length} files (${fmtBytes(inTotal)})`);
+        const mergeStartMs = Date.now();
+        console.log(`${blue(this.name)} merge: reading ${inputs.length} files (${fmtBytes(inTotal)}) at ${new Date(mergeStartMs).toISOString()}`);
         for (const f of inputs) console.log(`    in  ${f.name}  ${fmtBytes(f.size)}`);
 
         const { rows, times, deletes } = await this.resolveReaders(readers);
@@ -1176,10 +1270,19 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // Write all outputs BEFORE deleting any input, so a throw mid-write just leaves duplicates.
         const newNames: string[] = [];
         if (rows.length) {
-            for (const buffer of buildFileBuffer(rows, times)) {
+            const built = buildFileBuffer(rows, times);
+            // When the result is big enough to split across several files, log each sub-write (its key
+            // range + start/finish), since each storage.set can be large/slow and we want to see progress.
+            const split = built.length > 1;
+            if (split) console.log(`${blue(this.name)} merge: output split into ${built.length} files (> ${fmtBytes(TARGET_FILE_BYTES)} each)`);
+            for (let i = 0; i < built.length; i++) {
+                const part = built[i];
                 const name = newFileName(timestamp);
-                await storage.set(name, encodeCompressedBlocks(buffer));
+                const subStart = Date.now();
+                if (split) console.log(`    [${i + 1}/${built.length}] writing ${formatNumber(part.rowCount)} rows [${part.minKey} .. ${part.maxKey}] → ${name} at ${new Date(subStart).toISOString()}`);
+                await storage.set(name, encodeCompressedBlocks(part.buffer));
                 newNames.push(name);
+                if (split) console.log(`    [${i + 1}/${built.length}] wrote ${name} (${fmtBytes((await storage.getInfo(name).catch(() => undefined))?.size ?? 0)}) in ${formatTime(Date.now() - subStart)}`);
             }
         }
         // Carry surviving tombstones forward only if older files exist outside this merge that they still
@@ -1195,7 +1298,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // Log the result (files + on-disk sizes), so the before→after of the merge is visible.
         const outputs = await Promise.all(outNames.map(async n => ({ name: n, size: (await storage.getInfo(n).catch(() => undefined))?.size ?? 0 })));
         const outTotal = outputs.reduce((a, f) => a + f.size, 0);
-        console.log(`${blue(this.name)} merge: wrote ${outputs.length} files (${fmtBytes(outTotal)}, from ${fmtBytes(inTotal)})${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""}`);
+        console.log(`${blue(this.name)} merge: wrote ${outputs.length} files (${fmtBytes(outTotal)}, from ${fmtBytes(inTotal)})${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""} at ${new Date().toISOString()} (took ${formatTime(Date.now() - mergeStartMs)})`);
         for (const f of outputs) console.log(`    out ${f.name}  ${fmtBytes(f.size)}`);
 
         const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };

package/storage/BulkDatabase2/BulkDatabaseFormat.d.ts

@@ -4,7 +4,13 @@ export declare const KEY_COLUMN = "key";
 export declare const EMPTY_BUFFER: Buffer;
 export declare const ABSENT: unique symbol;
 export declare const TARGET_FILE_BYTES: number;
-export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): Buffer[];
+export interface BuiltFile {
+    buffer: Buffer;
+    minKey: string;
+    maxKey: string;
+    rowCount: number;
+}
+export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): BuiltFile[];
 export type BaseBulkDatabaseReader = {
     rowCount: number;
     totalBytes: number;

package/storage/BulkDatabase2/BulkDatabaseFormat.ts

@@ -226,14 +226,25 @@ function buildOneFile(rows: Record<string, unknown>[], times: number[]): Buffer
     return Buffer.concat([lengthPrefix, headerBuf, ...blobs]);
 }
 
-// Returns one complete, independent file buffer per chunk of rows. Rows are first sorted by key, then
+// One complete, independent file: the encoded buffer plus its key range + row count (the caller logs the
+// range when a merge splits across several files).
+export interface BuiltFile { buffer: Buffer; minKey: string; maxKey: string; rowCount: number; }
+
+// Returns one complete, independent file per chunk of rows. Rows are first sorted by key, then
 // partitioned into key-contiguous chunks of ~targetBytes each — so every returned file is key-sorted
 // (tight minKey/maxKey for the read-skip + merge planner) and stays near the target size, and no
 // single column blob / Buffer.concat approaches the ~2GB limit. The chunks have disjoint key ranges,
-// so the caller just writes each as its own file. A normal-sized write returns a single buffer.
+// so the caller just writes each as its own file. A normal-sized write returns a single file.
 // `times[i]` is row i's write-time, stored per row so reads resolve a key to its latest value by time.
-export function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes = TARGET_FILE_BYTES): Buffer[] {
-    if (rows.length === 0) return [buildOneFile([], [])];
+export function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes = TARGET_FILE_BYTES): BuiltFile[] {
+    // A chunk is already key-sorted, so its first/last row are its min/max key.
+    const make = (rs: Record<string, unknown>[], ts: number[]): BuiltFile => ({
+        buffer: buildOneFile(rs, ts),
+        minKey: rs.length ? (rs[0][KEY_COLUMN] as string) : "",
+        maxKey: rs.length ? (rs[rs.length - 1][KEY_COLUMN] as string) : "",
+        rowCount: rs.length,
+    });
+    if (rows.length === 0) return [make([], [])];
     // Sort rows + their times together by key so each output file is key-contiguous.
     const order = rows.map((_, i) => i).sort((a, b) => {
         const ka = rows[a][KEY_COLUMN] as string;
@@ -242,19 +253,19 @@ export function buildFileBuffer(rows: Record<string, unknown>[], times: number[]
     });
     const sortedRows = order.map(i => rows[i]);
     const sortedTimes = order.map(i => times[i]);
-    const result: Buffer[] = [];
+    const result: BuiltFile[] = [];
     let chunkStart = 0;
     let chunkBytes = 0;
     for (let i = 0; i < sortedRows.length; i++) {
         const rowBytes = estimateRowBytes(sortedRows[i]);
         if (i > chunkStart && chunkBytes + rowBytes > targetBytes) {
-            result.push(buildOneFile(sortedRows.slice(chunkStart, i), sortedTimes.slice(chunkStart, i)));
+            result.push(make(sortedRows.slice(chunkStart, i), sortedTimes.slice(chunkStart, i)));
             chunkStart = i;
             chunkBytes = 0;
         }
         chunkBytes += rowBytes;
     }
-    result.push(buildOneFile(sortedRows.slice(chunkStart), sortedTimes.slice(chunkStart)));
+    result.push(make(sortedRows.slice(chunkStart), sortedTimes.slice(chunkStart)));
     return result;
 }