sliftutils 1.4.45 → 1.4.47

package/index.d.ts

@@ -715,7 +715,7 @@ declare module "sliftutils/render-utils/colors" {
 }
 
 declare module "sliftutils/render-utils/mobxTyped" {
-    export { observable, runInAction, computed, autorun } from "mobx";
+    export { observable, runInAction, computed, autorun, onBecomeObserved, onBecomeUnobserved } from "mobx";
     export declare function configureMobxNextFrameScheduler(): void;
 
 }
@@ -763,7 +763,7 @@ declare module "sliftutils/render-utils/observer" {
 }
 
 declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
-    import { BulkDatabaseBase, BulkDatabase2Config } from "./BulkDatabaseBase";
+    import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config } from "./BulkDatabaseBase";
     export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
     export type { ReactiveDeps, StorageFactory, BulkDatabase2Config } from "./BulkDatabaseBase";
     /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
@@ -845,6 +845,25 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
             value: T[Column];
             time: number;
         }[] | undefined;
+        /**
+         * Reactive: whether (key, column) is loaded yet — true once we know the answer (value, absent, or
+         * deleted), false while it's still loading. getSingleFieldObjSync returns undefined for BOTH "loading"
+         * and "absent", so use this to tell them apart (e.g. show a spinner only when this is false).
+         */
+        isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+        /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
+        isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+        /**
+         * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
+         * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;
+         * returns true if the backend can't tell.
+         */
+        isKeyWatched(key: string): boolean;
+        /**
+         * Drop all of this collection's in-memory loaded caches and re-trigger every watcher, which re-requests
+         * and reloads from disk. Pending un-flushed writes are kept. Per-collection.
+         */
+        reloadFromDisk(): void;
         /** The columns present on disk and their byte sizes (no row data read). */
         getColumnInfo(): Promise<BulkColumnInfo[]>;
         /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
@@ -893,6 +912,15 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
          * most callers want compact() or tryMergeNow(). */
         merge(timeLo: number, timeHi: number): Promise<void>;
     }
+    export declare class MobxReactiveDeps implements ReactiveDeps {
+        private boxes;
+        private observed;
+        private box;
+        observe(signal: string): void;
+        invalidate(signal: string): void;
+        batch(fn: () => void): void;
+        isObserved(signal: string): boolean;
+    }
     export declare class BulkDatabase2<T extends {
         key: string;
     }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
@@ -916,11 +944,15 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         streamFoldHardLimitBytes: number;
         writeFlushMaxDelayMs: number;
         fileSetPollIntervalMs: number;
+        memoryFlushHeapBytes: number;
+        memoryFlushMinCollectionBytes: number;
+        memoryFlushThrottleMs: number;
     };
     export interface ReactiveDeps {
         observe(signal: string): void;
         invalidate(signal: string): void;
         batch(fn: () => void): void;
+        isObserved?(signal: string): boolean;
     }
     export declare const noopReactiveDeps: ReactiveDeps;
     export type StorageFactory = (path: string) => Promise<FileStorage>;
@@ -935,6 +967,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private storageFactory;
         private config;
         constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+        private static liveInstances;
+        private static memoryWatchdogStarted;
+        private static lastMemoryFlushMs;
+        private static startMemoryWatchdog;
+        static checkMemoryPressure(usedHeapBytes: number): void;
         private pendingAppends;
         private flushTimer;
         private flushChain;
@@ -955,6 +992,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private columnCache;
         private readerKeys;
         private loadedFileSet;
+        private loadedTotalBytes;
         private readerEpoch;
         private fileSetPollTimer;
         private bulkReaderCache;
@@ -972,6 +1010,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private lastMergeCheck;
         private getStreamFileName;
         private invalidateOverlay;
+        isKeyWatched(key: string): boolean;
         private isLiveNow;
         private invalidateSignal;
         private flushSignals;
@@ -985,6 +1024,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private clearReaderState;
         private resetReader;
         private reloadReader;
+        reloadFromDisk(): void;
         private readWithReload;
         private pollFileSet;
         write(entry: T): Promise<void>;
@@ -1058,6 +1098,8 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
             value: T[Column];
             time: number;
         }[] | undefined;
+        isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+        isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
         getColumnInfo(): Promise<{
             column: string;
             byteSize: number;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.45",
+    "version": "1.4.47",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/render-utils/mobxTyped.d.ts

@@ -1,2 +1,2 @@
-export { observable, runInAction, computed, autorun } from "mobx";
+export { observable, runInAction, computed, autorun, onBecomeObserved, onBecomeUnobserved } from "mobx";
 export declare function configureMobxNextFrameScheduler(): void;

package/render-utils/mobxTyped.ts

@@ -3,7 +3,7 @@ import { batchFunction } from "socket-function/src/batching";
 
 // Re-export the core mobx primitives so downstream packages (which may have their own duplicate mobx
 // in node_modules) can share THIS mobx instance, otherwise reactivity doesn't cross package boundaries.
-export { observable, runInAction, computed, autorun } from "mobx";
+export { observable, runInAction, computed, autorun, onBecomeObserved, onBecomeUnobserved } from "mobx";
 export function configureMobxNextFrameScheduler() {
     // NOTE: This makes a big difference if we do await calls in a loop which mutates observable state. BUT... we should probably just do those await calls before the loop?
     let batchReactionScheduler = batchFunction({

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -1,4 +1,4 @@
-import { BulkDatabaseBase, BulkDatabase2Config } from "./BulkDatabaseBase";
+import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config } from "./BulkDatabaseBase";
 export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
 export type { ReactiveDeps, StorageFactory, BulkDatabase2Config } from "./BulkDatabaseBase";
 /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
@@ -80,6 +80,25 @@ export interface IBulkDatabase2<T extends {
         value: T[Column];
         time: number;
     }[] | undefined;
+    /**
+     * Reactive: whether (key, column) is loaded yet — true once we know the answer (value, absent, or
+     * deleted), false while it's still loading. getSingleFieldObjSync returns undefined for BOTH "loading"
+     * and "absent", so use this to tell them apart (e.g. show a spinner only when this is false).
+     */
+    isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+    /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
+    isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+    /**
+     * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
+     * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;
+     * returns true if the backend can't tell.
+     */
+    isKeyWatched(key: string): boolean;
+    /**
+     * Drop all of this collection's in-memory loaded caches and re-trigger every watcher, which re-requests
+     * and reloads from disk. Pending un-flushed writes are kept. Per-collection.
+     */
+    reloadFromDisk(): void;
     /** The columns present on disk and their byte sizes (no row data read). */
     getColumnInfo(): Promise<BulkColumnInfo[]>;
     /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
@@ -128,6 +147,15 @@ export interface IBulkDatabase2<T extends {
      * most callers want compact() or tryMergeNow(). */
     merge(timeLo: number, timeHi: number): Promise<void>;
 }
+export declare class MobxReactiveDeps implements ReactiveDeps {
+    private boxes;
+    private observed;
+    private box;
+    observe(signal: string): void;
+    invalidate(signal: string): void;
+    batch(fn: () => void): void;
+    isObserved(signal: string): boolean;
+}
 export declare class BulkDatabase2<T extends {
     key: string;
 }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -1,5 +1,5 @@
 import { getFileStorageNested2 } from "../FileFolderAPI";
-import { observable, runInAction } from "../../render-utils/mobxTyped";
+import { observable, runInAction, onBecomeObserved, onBecomeUnobserved } from "../../render-utils/mobxTyped";
 import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config } from "./BulkDatabaseBase";
 
 export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
@@ -65,6 +65,28 @@ export interface IBulkDatabase2<T extends { key: string }> {
     /** Synchronous, reactive read of a whole column ({ key, value, time }). undefined while still loading. */
     getColumnSync<Column extends keyof T>(column: Column): { key: string; value: T[Column]; time: number }[] | undefined;
 
+    /**
+     * Reactive: whether (key, column) is loaded yet — true once we know the answer (value, absent, or
+     * deleted), false while it's still loading. getSingleFieldObjSync returns undefined for BOTH "loading"
+     * and "absent", so use this to tell them apart (e.g. show a spinner only when this is false).
+     */
+    isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+    /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
+    isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+
+    /**
+     * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
+     * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;
+     * returns true if the backend can't tell.
+     */
+    isKeyWatched(key: string): boolean;
+
+    /**
+     * Drop all of this collection's in-memory loaded caches and re-trigger every watcher, which re-requests
+     * and reloads from disk. Pending un-flushed writes are kept. Per-collection.
+     */
+    reloadFromDisk(): void;
+
     /** The columns present on disk and their byte sizes (no row data read). */
     getColumnInfo(): Promise<BulkColumnInfo[]>;
     /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
@@ -112,13 +134,18 @@ export interface IBulkDatabase2<T extends { key: string }> {
 // observer/autorun that calls it tracks that box) and invalidate() bumps it (so those reactions re-run).
 // This reproduces the fine-grained per-key + load-version reactivity the class had when it used
 // observable.map/observable.box directly, while keeping all that logic in the mobx-free base.
-class MobxReactiveDeps implements ReactiveDeps {
+export class MobxReactiveDeps implements ReactiveDeps {
     private boxes = new Map<string, { get(): number; set(value: number): void }>();
+    // Signals that currently have at least one observer, kept current via mobx's onBecomeObserved/
+    // onBecomeUnobserved hooks. Lets isObserved() answer "is anything watching this row" cheaply.
+    private observed = new Set<string>();
     private box(signal: string) {
         let box = this.boxes.get(signal);
         if (!box) {
             box = observable.box(0);
             this.boxes.set(signal, box);
+            onBecomeObserved(box, () => this.observed.add(signal));
+            onBecomeUnobserved(box, () => this.observed.delete(signal));
         }
         return box;
     }
@@ -132,6 +159,9 @@ class MobxReactiveDeps implements ReactiveDeps {
     batch(fn: () => void) {
         runInAction(fn);
     }
+    isObserved(signal: string) {
+        return this.observed.has(signal);
+    }
 }
 
 // Backwards-compatible BulkDatabase2: the mobx-reactive flavor. All behavior lives in BulkDatabaseBase;

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -12,11 +12,15 @@ export declare const bulkDatabase2Timing: {
     streamFoldHardLimitBytes: number;
     writeFlushMaxDelayMs: number;
     fileSetPollIntervalMs: number;
+    memoryFlushHeapBytes: number;
+    memoryFlushMinCollectionBytes: number;
+    memoryFlushThrottleMs: number;
 };
 export interface ReactiveDeps {
     observe(signal: string): void;
     invalidate(signal: string): void;
     batch(fn: () => void): void;
+    isObserved?(signal: string): boolean;
 }
 export declare const noopReactiveDeps: ReactiveDeps;
 export type StorageFactory = (path: string) => Promise<FileStorage>;
@@ -31,6 +35,11 @@ export declare class BulkDatabaseBase<T extends {
     private storageFactory;
     private config;
     constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+    private static liveInstances;
+    private static memoryWatchdogStarted;
+    private static lastMemoryFlushMs;
+    private static startMemoryWatchdog;
+    static checkMemoryPressure(usedHeapBytes: number): void;
     private pendingAppends;
     private flushTimer;
     private flushChain;
@@ -51,6 +60,7 @@ export declare class BulkDatabaseBase<T extends {
     private columnCache;
     private readerKeys;
     private loadedFileSet;
+    private loadedTotalBytes;
     private readerEpoch;
     private fileSetPollTimer;
     private bulkReaderCache;
@@ -68,6 +78,7 @@ export declare class BulkDatabaseBase<T extends {
     private lastMergeCheck;
     private getStreamFileName;
     private invalidateOverlay;
+    isKeyWatched(key: string): boolean;
     private isLiveNow;
     private invalidateSignal;
     private flushSignals;
@@ -81,6 +92,7 @@ export declare class BulkDatabaseBase<T extends {
     private clearReaderState;
     private resetReader;
     private reloadReader;
+    reloadFromDisk(): void;
     private readWithReload;
     private pollFileSet;
     write(entry: T): Promise<void>;
@@ -154,6 +166,8 @@ export declare class BulkDatabaseBase<T extends {
         value: T[Column];
         time: number;
     }[] | undefined;
+    isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
+    isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
     getColumnInfo(): Promise<{
         column: string;
         byteSize: number;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -34,6 +34,10 @@ const FILE_EXTENSION = ".bulk";
 const ROLLOVER_ROWS = 5000;
 const ROLLOVER_BYTES = 5 * 1024 * 1024;
 
+// How often the global memory-pressure watchdog samples the heap (the ACTION it may take is throttled
+// separately, see bulkDatabase2Timing.memoryFlushThrottleMs).
+const MEMORY_WATCHDOG_INTERVAL_MS = 60 * 1000;
+
 // An unreadable (corrupt/torn, not merely missing) file might be a write still in progress, so we
 // can't delete it on sight. Once it's been unreadable for longer than this (by its name timestamp),
 // no writer is plausibly still working on it, so we delete it. Until then we just warn.
@@ -104,6 +108,13 @@ export const bulkDatabase2Timing = {
     // We proactively re-list files this often; if the set changed under us (another tab/process merged) we
     // reload the index, so reads pick up the change even without first hitting a missing-file error.
     fileSetPollIntervalMs: 30 * 60 * 1000,
+    // Memory-pressure auto-flush (browser only; needs performance.memory). When the JS heap exceeds
+    // memoryFlushHeapBytes, at most once per memoryFlushThrottleMs, drop the in-memory observable state of
+    // every collection whose loaded data exceeds memoryFlushMinCollectionBytes (they reload lazily). Tuned
+    // so it does nothing for a healthy app (heap fine) or small collections.
+    memoryFlushHeapBytes: 1500 * 1024 * 1024,
+    memoryFlushMinCollectionBytes: 100 * 1024 * 1024,
+    memoryFlushThrottleMs: 15 * 60 * 1000,
 };
 
 // Marks a key as deleted in the in-memory overlay.
@@ -139,14 +150,19 @@ export interface ReactiveDeps {
     invalidate(signal: string): void;
     // Run a group of mutations + invalidations as one batch, so observers re-run at most once.
     batch(fn: () => void): void;
+    // Whether `signal` currently has any observer watching it. Optional — a backend that can't tell
+    // returns undefined (treated as "assume watched"). Lets writes skip per-key notification work for rows
+    // nothing is watching (see isWatchedSync).
+    isObserved?(signal: string): boolean;
 }
 
 // A non-reactive ReactiveDeps: sync reads still return current values, they just never trigger
-// re-renders. Use this when you don't need a UI to react to writes.
+// re-renders. Use this when you don't need a UI to react to writes. Nothing is ever observed.
 export const noopReactiveDeps: ReactiveDeps = {
     observe() { },
     invalidate() { },
     batch(fn) { fn(); },
+    isObserved() { return false; },
 };
 
 // Provides the FileStorage for a given path (the caller decides where data physically lives, so this
@@ -264,6 +280,50 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // stay correct even without first hitting a missing-file error.
         this.fileSetPollTimer = setInterval(() => void this.pollFileSet(), bulkDatabase2Timing.fileSetPollIntervalMs);
         (this.fileSetPollTimer as { unref?: () => void }).unref?.();
+        // Register for the global memory-pressure watchdog (browser only — needs performance.memory).
+        BulkDatabaseBase.liveInstances.add(this);
+        BulkDatabaseBase.startMemoryWatchdog();
+    }
+
+    // ---- memory-pressure watchdog (global, browser-only) ----
+    // When the JS heap gets large, the biggest contributors are usually a few collections whose in-memory
+    // observable state (loaded reader + base column/field caches) has grown. Rather than cap every
+    // collection all the time, we watch the heap and, only when it's actually under pressure, drop the
+    // observable state of the large collections (they reload lazily on demand). Throttled hard so it never
+    // thrashes a healthy app.
+    private static liveInstances = new Set<BulkDatabaseBase<{ key: string }>>();
+    private static memoryWatchdogStarted = false;
+    private static lastMemoryFlushMs = 0;
+    private static startMemoryWatchdog() {
+        if (BulkDatabaseBase.memoryWatchdogStarted) return;
+        BulkDatabaseBase.memoryWatchdogStarted = true;
+        const usedHeap = (): number | undefined => {
+            try { return (performance as unknown as { memory?: { usedJSHeapSize?: number } })?.memory?.usedJSHeapSize; }
+            catch { return undefined; }
+        };
+        if (typeof performance === "undefined" || usedHeap() === undefined) return; // no heap API (Node / non-Chromium)
+        const timer = setInterval(() => {
+            const used = usedHeap();
+            if (used !== undefined) BulkDatabaseBase.checkMemoryPressure(used);
+        }, MEMORY_WATCHDOG_INTERVAL_MS);
+        (timer as { unref?: () => void }).unref?.();
+    }
+    // If the heap is over the limit (and we haven't flushed recently), drop the in-memory observable state
+    // of every collection whose loaded data exceeds the size threshold, so it reloads lazily and frees the
+    // rest. Exposed (with the heap value injected) so it's callable/testable without the real timer.
+    public static checkMemoryPressure(usedHeapBytes: number): void {
+        if (usedHeapBytes < bulkDatabase2Timing.memoryFlushHeapBytes) return;
+        const now = Date.now();
+        if (now - BulkDatabaseBase.lastMemoryFlushMs < bulkDatabase2Timing.memoryFlushThrottleMs) return;
+        BulkDatabaseBase.lastMemoryFlushMs = now;
+        const flushed: string[] = [];
+        for (const db of BulkDatabaseBase.liveInstances) {
+            if (db.loadedTotalBytes > bulkDatabase2Timing.memoryFlushMinCollectionBytes) {
+                flushed.push(`${db.name} (${fmtBytes(db.loadedTotalBytes)})`);
+                db.reloadFromDisk();
+            }
+        }
+        if (flushed.length) console.log(`[bulk2] heap ${fmtBytes(usedHeapBytes)} over ${fmtBytes(bulkDatabase2Timing.memoryFlushHeapBytes)} — flushed observable state of ${flushed.length} large collection(s) to reclaim memory: ${flushed.join(", ")}`);
     }
 
     // ---- buffered stream writes (per collection) ----
@@ -341,6 +401,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // under us; we notice either when a read hits a deleted file (readWithReload) or via the periodic poll
     // (pollFileSet), and reload the index — cheap, since a build only reads headers + key columns.
     private loadedFileSet: Set<string> | undefined; // file names the current reader was built from
+    // Total (uncompressed) size of the data the current reader spans, as a proxy for this collection's
+    // in-memory footprint — used by the memory-pressure watchdog. 0 when no reader is loaded.
+    private loadedTotalBytes = 0;
     // Bumped every time the index is cleared/reloaded. A failed read captures it before reading and only
     // triggers a reload if it's unchanged afterward — so N concurrent failures coalesce onto ONE rebuild
     // (the rest see the bumped epoch, skip their own reload, and just await the in-flight one).
@@ -406,10 +469,20 @@ export class BulkDatabaseBase<T extends { key: string }> {
         this.dataGen++;
         if (columns === "all") this.columnCache.clear();
         else for (const c of columns) this.columnCache.delete(c);
-        this.invalidateSignal(key);
+        // The per-key signal only re-runs getSingleFieldObjSync watchers of THIS key; skip it when nothing
+        // is watching the row (a new watcher reads the current overlay anyway). The overlay-wide signal +
+        // column-cache drop above still cover whole-column watchers and read correctness.
+        if (this.deps.isObserved?.(key) ?? true) this.invalidateSignal(key);
         this.invalidateSignal(OVERLAY_SIGNAL);
     }
 
+    // Whether a row (key) is currently being watched — i.e. some reactive observer is subscribed to it via
+    // getSingleFieldObjSync / getSingleFieldSync. Useful for skipping per-row work when nothing's watching.
+    // Returns true if the backend can't tell (conservative).
+    public isKeyWatched(key: string): boolean {
+        return this.deps.isObserved?.(key) ?? true;
+    }
+
     // Whether `key` is currently a live key in the resolved view (overlay wins over disk). A write to a key
     // that's already live only changes the columns it sets; a write to an absent key (or one currently
     // deleted in the overlay) makes it appear in every column, so every column's cache must drop.
@@ -526,6 +599,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
         this.readerKeys = new Set(joined.keys);
         // The files this index was built from, so the poll can detect a concurrent merge changing the set.
         this.loadedFileSet = new Set([...bulkFiles.map(f => f.fileName), ...streamFiles.map(f => f.fileName)]);
+        this.loadedTotalBytes = joined.totalBytes; // footprint proxy for the memory-pressure watchdog
+
         // Evict cached sub-readers for files a merge removed, so the caches track the live file set.
         this.pruneFileCaches(bulkFiles, streamFiles);
 
@@ -588,6 +663,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         this.columnCache.clear();
         this.readerKeys = undefined;     // the next build repopulates it
         this.loadedFileSet = undefined;  // ditto
+        this.loadedTotalBytes = 0;       // nothing loaded ⇒ no footprint for the memory watchdog
         // The next build re-measures the stream; clear the estimate so a just-folded stream doesn't keep
         // looking "heavy" and re-trigger a fold before then.
         this.streamRowsOnDisk = 0;
@@ -620,6 +696,23 @@ export class BulkDatabaseBase<T extends { key: string }> {
         });
     }
 
+    // Drop ALL of this collection's in-memory loaded caches (the resolved reader, the sync base column /
+    // field caches and their stale fallbacks, the per-file decoded readers) and re-trigger every watcher,
+    // so they re-request and reload from disk. Unlike the automatic reloads, this is a genuine full clear
+    // (no stale-fallback served — watchers go through a real loading state). Pending un-flushed writes (the
+    // overlay) are KEPT, so nothing not-yet-on-disk is lost. Per-collection (this instance only).
+    public reloadFromDisk(): void {
+        this.deps.batch(() => {
+            this.clearReaderState();       // drops reader + base caches (→ stale) + column cache; bumps gens
+            this.staleBaseColumns.clear(); // and the stale fallbacks — a genuine full clear, not a swap
+            this.staleBaseFields.clear();
+            this.bulkReaderCache.clear();  // force re-decode from disk
+            this.streamReaderCache.clear();
+            this.invalidateSignal(LOAD_SIGNAL);
+            this.invalidateSignal(OVERLAY_SIGNAL);
+        });
+    }
+
     // Run a read against the loaded index; if a file vanished mid-read (a concurrent merge deleted it),
     // reload the index and retry — looping until it succeeds or we've tried MAX_INDEX_RELOAD_ATTEMPTS times.
     // Reloads coalesce: a failing read only resets the index if nobody has reset it since the read grabbed
@@ -1168,7 +1261,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
             ...streamFiles.map(f => ({ name: f.fileName, size: streamData.sizes.get(f.fileName) ?? 0 })),
         ];
         const inTotal = inputs.reduce((a, f) => a + f.size, 0);
-        console.log(`${blue(this.name)} merge: reading ${inputs.length} files (${fmtBytes(inTotal)})`);
+        const mergeStartMs = Date.now();
+        console.log(`${blue(this.name)} merge: reading ${inputs.length} files (${fmtBytes(inTotal)}) at ${new Date(mergeStartMs).toISOString()}`);
         for (const f of inputs) console.log(`    in  ${f.name}  ${fmtBytes(f.size)}`);
 
         const { rows, times, deletes } = await this.resolveReaders(readers);
@@ -1195,7 +1289,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // Log the result (files + on-disk sizes), so the before→after of the merge is visible.
         const outputs = await Promise.all(outNames.map(async n => ({ name: n, size: (await storage.getInfo(n).catch(() => undefined))?.size ?? 0 })));
         const outTotal = outputs.reduce((a, f) => a + f.size, 0);
-        console.log(`${blue(this.name)} merge: wrote ${outputs.length} files (${fmtBytes(outTotal)}, from ${fmtBytes(inTotal)})${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""}`);
+        console.log(`${blue(this.name)} merge: wrote ${outputs.length} files (${fmtBytes(outTotal)}, from ${fmtBytes(inTotal)})${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""} at ${new Date().toISOString()} (took ${formatTime(Date.now() - mergeStartMs)})`);
         for (const f of outputs) console.log(`    out ${f.name}  ${fmtBytes(f.size)}`);
 
         const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };
@@ -1604,6 +1698,38 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return result;
     }
 
+    // Reactive: whether (key, column) is available to read synchronously yet. true once it's loaded — we
+    // know the answer, whether that's a value, absent, or deleted; false while it's still loading from disk.
+    // Pairs with getSingleFieldObjSync, which returns undefined for BOTH "loading" and "absent" — use this
+    // to tell them apart (e.g. show a spinner only when this is false). Triggers the load if not started,
+    // and (like the sync reads) counts the last-known value served during a reload as loaded.
+    public isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean {
+        void this.syncSetup();
+        this.deps.observe(LOAD_SIGNAL);
+        this.deps.observe(key);
+        const entry = this.overlay.get(key);
+        if (entry !== undefined) {
+            if (entry.value === DELETED) return true;          // known: deleted
+            if (String(column) in entry.value) return true;    // known: overlay holds this column
+            // else: this column falls through to disk — check the base caches below
+        }
+        const cacheKey = nullJoin(String(column), key);
+        if (this.baseFields.has(cacheKey) || this.staleBaseFields.has(cacheKey)) return true;
+        this.ensureBaseField(key, String(column));
+        return false;
+    }
+
+    // Reactive: whether a whole column is available to read synchronously yet (see isFieldLoadedSync).
+    public isColumnLoadedSync<Column extends keyof T>(column: Column): boolean {
+        void this.syncSetup();
+        this.deps.observe(LOAD_SIGNAL);
+        this.deps.observe(OVERLAY_SIGNAL);
+        const col = String(column);
+        if (this.columnCache.has(col) || this.baseColumns.has(col) || this.staleBaseColumns.has(col)) return true;
+        this.ensureBaseColumn(col);
+        return false;
+    }
+
     public async getColumnInfo() {
         let reader = await this.reader();
         return reader.columns;