sliftutils 1.4.15 → 1.4.17

package/index.d.ts

@@ -763,9 +763,9 @@ declare module "sliftutils/render-utils/observer" {
 }
 
 declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
-    import { BulkDatabaseBase } from "./BulkDatabaseBase";
+    import { BulkDatabaseBase, BulkDatabase2Config } from "./BulkDatabaseBase";
     export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
-    export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+    export type { ReactiveDeps, StorageFactory, BulkDatabase2Config } from "./BulkDatabaseBase";
     /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
     export type BulkColumnInfo = {
         column: string;
@@ -896,7 +896,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
     export declare class BulkDatabase2<T extends {
         key: string;
     }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
-        constructor(name: string);
+        constructor(name: string, config?: BulkDatabase2Config);
     }
 
 }
@@ -920,13 +920,17 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
     }
     export declare const noopReactiveDeps: ReactiveDeps;
     export type StorageFactory = (path: string) => Promise<FileStorage>;
+    export type BulkDatabase2Config = {
+        triggerThrottleMs?: number;
+    };
     export declare class BulkDatabaseBase<T extends {
         key: string;
     }> {
         readonly name: string;
         protected deps: ReactiveDeps;
         private storageFactory;
-        constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+        private config;
+        constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
         private pendingAppends;
         private flushTimer;
         private flushChain;
@@ -944,12 +948,22 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private automaticCompactionAllowed;
         private overlay;
         private streamTimes;
+        private columnCache;
+        private readerKeys;
+        private dataGen;
+        private pendingSignals;
+        private triggerTimer;
+        private currentTriggerDelay;
+        private lastTriggerTime;
         private streamRowsOnDisk;
         private streamBytesOnDisk;
         private streamFileName;
         private lastMergeCheck;
         private getStreamFileName;
         private invalidateOverlay;
+        private isLiveNow;
+        private invalidateSignal;
+        private flushSignals;
         private setOverlayRow;
         private setOverlayDeleted;
         private reader;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.15",
+    "version": "1.4.17",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -1,6 +1,6 @@
-import { BulkDatabaseBase } from "./BulkDatabaseBase";
+import { BulkDatabaseBase, BulkDatabase2Config } from "./BulkDatabaseBase";
 export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
-export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+export type { ReactiveDeps, StorageFactory, BulkDatabase2Config } from "./BulkDatabaseBase";
 /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
 export type BulkColumnInfo = {
     column: string;
@@ -131,5 +131,5 @@ export interface IBulkDatabase2<T extends {
 export declare class BulkDatabase2<T extends {
     key: string;
 }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
-    constructor(name: string);
+    constructor(name: string, config?: BulkDatabase2Config);
 }

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -1,9 +1,9 @@
 import { getFileStorageNested2 } from "../FileFolderAPI";
 import { observable, runInAction } from "../../render-utils/mobxTyped";
-import { BulkDatabaseBase, ReactiveDeps } from "./BulkDatabaseBase";
+import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config } from "./BulkDatabaseBase";
 
 export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
-export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+export type { ReactiveDeps, StorageFactory, BulkDatabase2Config } from "./BulkDatabaseBase";
 
 /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
 export type BulkColumnInfo = { column: string; byteSize: number };
@@ -138,7 +138,7 @@ class MobxReactiveDeps implements ReactiveDeps {
 // this just supplies mobx reactivity and the default (getFileStorageNested2) storage backend, so the
 // sync reads (getSingleFieldSync / getColumnSync) stay observable for mobx components.
 export class BulkDatabase2<T extends { key: string }> extends BulkDatabaseBase<T> implements IBulkDatabase2<T> {
-    constructor(name: string) {
-        super(name, new MobxReactiveDeps(), getFileStorageNested2);
+    constructor(name: string, config?: BulkDatabase2Config) {
+        super(name, new MobxReactiveDeps(), getFileStorageNested2, config);
     }
 }

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -16,13 +16,17 @@ export interface ReactiveDeps {
 }
 export declare const noopReactiveDeps: ReactiveDeps;
 export type StorageFactory = (path: string) => Promise<FileStorage>;
+export type BulkDatabase2Config = {
+    triggerThrottleMs?: number;
+};
 export declare class BulkDatabaseBase<T extends {
     key: string;
 }> {
     readonly name: string;
     protected deps: ReactiveDeps;
     private storageFactory;
-    constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+    private config;
+    constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
     private pendingAppends;
     private flushTimer;
     private flushChain;
@@ -40,12 +44,22 @@ export declare class BulkDatabaseBase<T extends {
     private automaticCompactionAllowed;
     private overlay;
     private streamTimes;
+    private columnCache;
+    private readerKeys;
+    private dataGen;
+    private pendingSignals;
+    private triggerTimer;
+    private currentTriggerDelay;
+    private lastTriggerTime;
     private streamRowsOnDisk;
     private streamBytesOnDisk;
     private streamFileName;
     private lastMergeCheck;
     private getStreamFileName;
     private invalidateOverlay;
+    private isLiveNow;
+    private invalidateSignal;
+    private flushSignals;
     private setOverlayRow;
     private setOverlayDeleted;
     private reader;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -129,6 +129,22 @@ export const noopReactiveDeps: ReactiveDeps = {
 // file needn't know about getFileStorageNested2 / the browser-vs-node storage details).
 export type StorageFactory = (path: string) => Promise<FileStorage>;
 
+// Optional per-collection configuration.
+export type BulkDatabase2Config = {
+    // When set (> 0), the reactive change notifications writes/loads emit are throttled and BATCHED
+    // globally (across all keys), so a high-frequency write source doesn't re-run watchers on every single
+    // change. The throttle RAMPS like the write-flush one: the first change after a lull notifies
+    // immediately, then under sustained changes the delay doubles up to triggerThrottleMs, coalescing the
+    // burst into one notification. In-memory/async reads are always current — only the OBSERVABLE
+    // notification (the mobx re-render trigger) is delayed and merged.
+    triggerThrottleMs?: number;
+};
+
+// Trigger-throttle ramp: the first deferred notification waits this long, then the delay doubles on each
+// further change up to BulkDatabase2Config.triggerThrottleMs. ~16ms ≈ one animation frame, so an isolated
+// burst still notifies within a frame.
+const TRIGGER_THROTTLE_FIRST_STEP_MS = 16;
+
 // The load/reset lifecycle shares one signal; every sync read observes it so it re-renders when the
 // reader resets or a base column/field finishes loading. The overlay's per-key signal is the key
 // itself (a point read observes just its key), plus one overlay-wide signal that whole-column reads
@@ -205,6 +221,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         public readonly name: string,
         protected deps: ReactiveDeps,
         private storageFactory: StorageFactory,
+        private config: BulkDatabase2Config = {},
     ) {
         // Best-effort: persist buffered stream writes when the tab is hidden/closing. visibilitychange →
         // hidden fires early enough that the (async) flush usually completes; pagehide is the backstop.
@@ -278,6 +295,29 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // overlay to decide whether an incoming remote write is actually newer than what we have.
     private streamTimes = new Map<string, number>();
 
+    // Cache of fully-resolved (overlay-patched) column results, keyed by column name. The result only
+    // changes when the overlay mutates or the reader resets, so we keep it until then — repeat whole-column
+    // reads (common in a UI re-render) are then free, however large the column. Each cached array is frozen
+    // SHALLOWLY: Object.freeze locks the array itself (callers can't mutate a shared result) but never the
+    // element objects or their values, so typed-array column values keep their fast representation.
+    private columnCache = new Map<string, { key: string; value: unknown; time: number }[]>();
+    // Live keys of the currently-loaded reader (disk only, no overlay), so a write can tell a partial
+    // update of an existing key (only the written columns change) from adding/removing a key (which appears
+    // in / disappears from EVERY column). Set on reader build, cleared on reset.
+    private readerKeys: Set<string> | undefined;
+    // Bumped on every overlay mutation / reader reset. An async column build captures it before its awaits
+    // and only caches its result if it hasn't changed since — so a write or reset mid-build (which clears
+    // the cache and may swap the reader) can never leave a stale entry behind.
+    private dataGen = 0;
+
+    // ---- trigger throttle (see BulkDatabase2Config.triggerThrottleMs) ----
+    // Signals whose notification is deferred, the pending flush timer, and the ramping delay. Data state is
+    // already updated synchronously; only these observable notifications are batched/delayed.
+    private pendingSignals = new Set<string>();
+    private triggerTimer: ReturnType<typeof setTimeout> | undefined;
+    private currentTriggerDelay = 0;
+    private lastTriggerTime = 0;
+
     // Approximate size of the tier-0 stream data on disk: set accurately from the last reader build, then
     // kept current as each flush appends more. Drives the stream-fold trigger (see streamNeedsFold);
     // reset on resetReader, since after a merge/reset the next reader build re-measures it.
@@ -303,24 +343,71 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return this.streamFileName;
     }
 
-    private invalidateOverlay(key: string) {
-        this.deps.invalidate(key);
-        this.deps.invalidate(OVERLAY_SIGNAL);
+    // Invalidate after an overlay change. `columns` is the set of columns whose resolved result actually
+    // changed — only those cached columns are dropped — or "all" when the key set itself changed (a key
+    // added or removed appears in / disappears from every column).
+    private invalidateOverlay(key: string, columns: Iterable<string> | "all") {
+        this.dataGen++;
+        if (columns === "all") this.columnCache.clear();
+        else for (const c of columns) this.columnCache.delete(c);
+        this.invalidateSignal(key);
+        this.invalidateSignal(OVERLAY_SIGNAL);
+    }
+
+    // Whether `key` is currently a live key in the resolved view (overlay wins over disk). A write to a key
+    // that's already live only changes the columns it sets; a write to an absent key (or one currently
+    // deleted in the overlay) makes it appear in every column, so every column's cache must drop.
+    private isLiveNow(key: string): boolean {
+        const e = this.overlay.get(key);
+        if (e) return e.value !== DELETED;
+        return this.readerKeys?.has(key) ?? false;
+    }
+
+    // Notify observers of `signal`. With triggerThrottleMs set, notifications are batched and delayed on a
+    // ramping schedule so a high-frequency source can't re-run watchers on every change: a change after a
+    // lull notifies on the next tick (no real delay, but all of one change's signals batch together);
+    // under sustained changes the delay doubles up to the max, coalescing the burst into one notification.
+    // Only the OBSERVABLE notification is delayed — the underlying data was already updated, so a read in
+    // the meantime still sees current values.
+    private invalidateSignal(signal: string) {
+        const maxMs = this.config.triggerThrottleMs ?? 0;
+        if (maxMs <= 0) { this.deps.invalidate(signal); return; }
+        this.pendingSignals.add(signal);
+        const now = Date.now();
+        const lull = now - this.lastTriggerTime > maxMs;
+        this.lastTriggerTime = now;
+        if (this.triggerTimer !== undefined) return; // a flush is already scheduled; it will pick this up
+        // Lull resets the ramp (notify next tick); an active burst ramps the delay toward the max.
+        this.currentTriggerDelay = lull ? 0 : Math.min(maxMs, Math.max(TRIGGER_THROTTLE_FIRST_STEP_MS, this.currentTriggerDelay * 2));
+        this.triggerTimer = setTimeout(() => { this.triggerTimer = undefined; this.flushSignals(); }, this.currentTriggerDelay);
+        (this.triggerTimer as { unref?: () => void }).unref?.();
+    }
+
+    private flushSignals() {
+        if (this.pendingSignals.size === 0) return;
+        const signals = [...this.pendingSignals];
+        this.pendingSignals.clear();
+        this.deps.batch(() => { for (const s of signals) this.deps.invalidate(s); });
     }
 
     // Merges a (possibly partial) row onto the key's current overlay value, so a partial write/update
     // only changes the columns it includes — the rest fall through to disk on read. A prior delete is
     // reset (the key is being re-created).
     private setOverlayRow(key: string, row: Record<string, unknown>, time: number) {
+        // Decide BEFORE mutating: if the key is already live, only the columns this write sets change;
+        // otherwise it's a new/re-created key that joins every column.
+        const wasLive = this.isLiveNow(key);
         const existing = this.overlay.get(key);
         const value = existing && existing.value !== DELETED ? { ...existing.value, ...row } : { ...row };
         this.overlay.set(key, { time, value });
-        this.invalidateOverlay(key);
+        this.invalidateOverlay(key, wasLive ? Object.keys(row) : "all");
     }
 
     private setOverlayDeleted(key: string, time: number) {
+        // Deleting a live key removes it from every column; deleting an absent key changes no column.
+        const columns = this.isLiveNow(key) ? "all" : [];
         this.overlay.set(key, { time, value: DELETED });
-        this.invalidateOverlay(key);
+        this.invalidateOverlay(key, columns);
     }
 
     private reader = lazy(async (): Promise<ResolvedReader> => {
@@ -378,6 +465,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
         }
         readers.push(...bulkReaders);
         const joined = await joinBulkDatabases(readers);
+        // Live keys of this reader, so per-column cache invalidation can tell an update of an existing key
+        // (only its set columns change) from a key add/remove (which touches every column).
+        this.readerKeys = new Set(joined.keys);
 
         let time = Date.now() - start;
         if (time > 50) {
@@ -435,13 +525,16 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.baseFields.clear();
             this.baseFieldsLoading.clear();
             this.overlay.clear();
+            this.dataGen++;
+            this.columnCache.clear();
+            this.readerKeys = undefined; // the next reader build repopulates it
             // The next reader build re-measures the stream; clear the estimate so a just-folded stream
             // doesn't keep looking "heavy" and re-trigger a fold before then.
             this.streamRowsOnDisk = 0;
             this.streamBytesOnDisk = 0;
             for (const p of this.pendingAppends) p.apply();
-            this.deps.invalidate(LOAD_SIGNAL);
-            this.deps.invalidate(OVERLAY_SIGNAL);
+            this.invalidateSignal(LOAD_SIGNAL);
+            this.invalidateSignal(OVERLAY_SIGNAL);
         });
     }
 
@@ -1134,14 +1227,21 @@ export class BulkDatabaseBase<T extends { key: string }> {
 
     public async getColumn<Column extends keyof T>(column: Column): Promise<{ key: string; value: T[Column]; time: number }[]> {
         void this.syncSetup();
+        const col = String(column);
+        const cached = this.columnCache.get(col);
+        if (cached) return cached as { key: string; value: T[Column]; time: number }[];
+        const gen = this.dataGen;
         let time = Date.now();
         let reader = await this.reader();
-        let base = await reader.getColumn(String(column));
-        let result = this.patchColumn(base, String(column)) as { key: string; value: T[Column]; time: number }[];
+        let base = await reader.getColumn(col);
+        let result = this.patchColumn(base, col) as { key: string; value: T[Column]; time: number }[];
         time = Date.now() - time;
         if (time > 50) {
             console.log(`${blue(`${this.name}.getColumn(${JSON.stringify(column)})`)} took ${red(formatTime(time))} ${this.formatInfo(reader)}`);
         }
+        Object.freeze(result);
+        // Only cache if no write/reset happened during the awaits above (else this result may be stale).
+        if (this.dataGen === gen) this.columnCache.set(col, result);
         return result;
     }
 
@@ -1179,7 +1279,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.deps.batch(() => {
                 this.baseColumns.set(column, base);
                 this.baseColumnsLoading.delete(column);
-                this.deps.invalidate(LOAD_SIGNAL);
+                this.invalidateSignal(LOAD_SIGNAL);
             });
         })();
     }
@@ -1194,7 +1294,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.deps.batch(() => {
                 this.baseFields.set(cacheKey, resolved);
                 this.baseFieldsLoading.delete(cacheKey);
-                this.deps.invalidate(LOAD_SIGNAL);
+                this.invalidateSignal(LOAD_SIGNAL);
             });
         })();
     }
@@ -1236,12 +1336,19 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // Observe the overlay-wide signal so we recompute once the base arrives or the overlay changes.
         this.deps.observe(OVERLAY_SIGNAL);
         let col = String(column);
+        const cached = this.columnCache.get(col);
+        if (cached) return cached as { key: string; value: T[Column]; time: number }[];
         let base = this.baseColumns.get(col);
         if (!base) {
             this.ensureBaseColumn(col);
             return undefined;
         }
-        return this.patchColumn(base, col) as { key: string; value: T[Column]; time: number }[];
+        // Synchronous (no awaits) → reads the current overlay and caches it atomically; an observer re-runs
+        // (and the cache is cleared) on any later change, so the frozen result is safe to share.
+        let result = this.patchColumn(base, col) as { key: string; value: T[Column]; time: number }[];
+        Object.freeze(result);
+        this.columnCache.set(col, result);
+        return result;
     }
 
     public async getColumnInfo() {