sliftutils 1.4.1 → 1.4.3

package/index.d.ts

@@ -813,20 +813,37 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
         getKeys(): Promise<string[]>;
         /** One field's value for a key, or undefined if the key/column isn't set or the key is deleted. */
         getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
-        /** A whole column as {key, value} for every live key. */
+        /**
+         * Like getSingleField but returns { key, value, time } (the same shape a getColumn entry has), where
+         * time is roughly when the value last changed. undefined only when the key isn't present/live.
+         */
+        getSingleFieldObj<Column extends keyof T>(key: string, column: Column): Promise<{
+            key: string;
+            value: T[Column];
+            time: number;
+        } | undefined>;
+        /** A whole column as { key, value, time } for every live key (time ≈ when each value last changed). */
         getColumn<Column extends keyof T>(column: Column): Promise<{
             key: string;
             value: T[Column];
+            time: number;
         }[]>;
         /**
          * Synchronous, reactive read of one field. Returns undefined while the base value is still loading
          * (and re-renders once it arrives, under a mobx observer); reflects pending writes immediately.
          */
         getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
-        /** Synchronous, reactive read of a whole column. undefined while still loading. */
+        /** Sync, reactive counterpart of getSingleFieldObj: { key, value, time } once loaded, else undefined. */
+        getSingleFieldObjSync<Column extends keyof T>(key: string, column: Column): {
+            key: string;
+            value: T[Column];
+            time: number;
+        } | undefined;
+        /** Synchronous, reactive read of a whole column ({ key, value, time }). undefined while still loading. */
         getColumnSync<Column extends keyof T>(column: Column): {
             key: string;
             value: T[Column];
+            time: number;
         }[] | undefined;
         /** The columns present on disk and their byte sizes (no row data read). */
         getColumnInfo(): Promise<BulkColumnInfo[]>;
@@ -834,6 +851,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
         getReaderInfo(): Promise<BulkReaderInfo>;
         /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
         compact(): Promise<void>;
+        /**
+         * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
+         * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
+         * "accepted" (in memory + cross-tab), not necessarily "on disk". Call this to force durability — it's
+         * also run automatically on tab hide/close and before every merge.
+         */
+        flush(): Promise<void>;
         /**
          * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
          * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
@@ -861,8 +885,10 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
     export declare const bulkDatabase2Timing: {
         streamSealAgeMs: number;
         mergeCheckIntervalMs: number;
+        mergeSpacingMs: number;
         firstMergeTriggerFiles: number;
         firstMergeTriggerRangeMs: number;
+        writeFlushMaxDelayMs: number;
     };
     export interface ReactiveDeps {
         observe(signal: string): void;
@@ -878,6 +904,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         protected deps: ReactiveDeps;
         private storageFactory;
         constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+        private pendingAppends;
+        private flushTimer;
+        private flushChain;
+        private currentFlushDelay;
+        private lastWriteTime;
         static clearCache(): void;
         storage: {
             (): Promise<FileStorage>;
@@ -902,6 +933,10 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         writeBatch(entries: T[]): Promise<void>;
         delete(key: string): Promise<void>;
         deleteBatch(keys: string[]): Promise<void>;
+        private streamAppend;
+        flush(): Promise<void>;
+        private flushPending;
+        private doFlush;
         update(entry: Partial<T> & {
             key: string;
         }): Promise<void>;
@@ -927,13 +962,21 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private resolveReaders;
         private mergeFileSet;
         private canDeleteStream;
+        private mergeSpacingDelay;
         private testMerge;
+        private findDuplicateGroups;
         private formatInfo;
         private patchColumn;
         getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+        getSingleFieldObj<Column extends keyof T>(key: string, column: Column): Promise<{
+            key: string;
+            value: T[Column];
+            time: number;
+        } | undefined>;
         getColumn<Column extends keyof T>(column: Column): Promise<{
             key: string;
             value: T[Column];
+            time: number;
         }[]>;
         getKeys(): Promise<string[]>;
         private baseColumns;
@@ -943,9 +986,15 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private ensureBaseColumn;
         private ensureBaseField;
         getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+        getSingleFieldObjSync<Column extends keyof T>(key: string, column: Column): {
+            key: string;
+            value: T[Column];
+            time: number;
+        } | undefined;
         getColumnSync<Column extends keyof T>(column: Column): {
             key: string;
             value: T[Column];
+            time: number;
         }[] | undefined;
         getColumnInfo(): Promise<{
             column: string;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.1",
+    "version": "1.4.3",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -48,20 +48,37 @@ export interface IBulkDatabase2<T extends {
     getKeys(): Promise<string[]>;
     /** One field's value for a key, or undefined if the key/column isn't set or the key is deleted. */
     getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
-    /** A whole column as {key, value} for every live key. */
+    /**
+     * Like getSingleField but returns { key, value, time } (the same shape a getColumn entry has), where
+     * time is roughly when the value last changed. undefined only when the key isn't present/live.
+     */
+    getSingleFieldObj<Column extends keyof T>(key: string, column: Column): Promise<{
+        key: string;
+        value: T[Column];
+        time: number;
+    } | undefined>;
+    /** A whole column as { key, value, time } for every live key (time ≈ when each value last changed). */
     getColumn<Column extends keyof T>(column: Column): Promise<{
         key: string;
         value: T[Column];
+        time: number;
     }[]>;
     /**
      * Synchronous, reactive read of one field. Returns undefined while the base value is still loading
      * (and re-renders once it arrives, under a mobx observer); reflects pending writes immediately.
      */
     getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
-    /** Synchronous, reactive read of a whole column. undefined while still loading. */
+    /** Sync, reactive counterpart of getSingleFieldObj: { key, value, time } once loaded, else undefined. */
+    getSingleFieldObjSync<Column extends keyof T>(key: string, column: Column): {
+        key: string;
+        value: T[Column];
+        time: number;
+    } | undefined;
+    /** Synchronous, reactive read of a whole column ({ key, value, time }). undefined while still loading. */
     getColumnSync<Column extends keyof T>(column: Column): {
         key: string;
         value: T[Column];
+        time: number;
     }[] | undefined;
     /** The columns present on disk and their byte sizes (no row data read). */
     getColumnInfo(): Promise<BulkColumnInfo[]>;
@@ -69,6 +86,13 @@ export interface IBulkDatabase2<T extends {
     getReaderInfo(): Promise<BulkReaderInfo>;
     /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
     compact(): Promise<void>;
+    /**
+     * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
+     * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
+     * "accepted" (in memory + cross-tab), not necessarily "on disk". Call this to force durability — it's
+     * also run automatically on tab hide/close and before every merge.
+     */
+    flush(): Promise<void>;
     /**
      * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
      * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -47,16 +47,23 @@ export interface IBulkDatabase2<T extends { key: string }> {
     getKeys(): Promise<string[]>;
     /** One field's value for a key, or undefined if the key/column isn't set or the key is deleted. */
     getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
-    /** A whole column as {key, value} for every live key. */
-    getColumn<Column extends keyof T>(column: Column): Promise<{ key: string; value: T[Column] }[]>;
+    /**
+     * Like getSingleField but returns { key, value, time } (the same shape a getColumn entry has), where
+     * time is roughly when the value last changed. undefined only when the key isn't present/live.
+     */
+    getSingleFieldObj<Column extends keyof T>(key: string, column: Column): Promise<{ key: string; value: T[Column]; time: number } | undefined>;
+    /** A whole column as { key, value, time } for every live key (time ≈ when each value last changed). */
+    getColumn<Column extends keyof T>(column: Column): Promise<{ key: string; value: T[Column]; time: number }[]>;
 
     /**
      * Synchronous, reactive read of one field. Returns undefined while the base value is still loading
      * (and re-renders once it arrives, under a mobx observer); reflects pending writes immediately.
      */
     getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
-    /** Synchronous, reactive read of a whole column. undefined while still loading. */
-    getColumnSync<Column extends keyof T>(column: Column): { key: string; value: T[Column] }[] | undefined;
+    /** Sync, reactive counterpart of getSingleFieldObj: { key, value, time } once loaded, else undefined. */
+    getSingleFieldObjSync<Column extends keyof T>(key: string, column: Column): { key: string; value: T[Column]; time: number } | undefined;
+    /** Synchronous, reactive read of a whole column ({ key, value, time }). undefined while still loading. */
+    getColumnSync<Column extends keyof T>(column: Column): { key: string; value: T[Column]; time: number }[] | undefined;
 
     /** The columns present on disk and their byte sizes (no row data read). */
     getColumnInfo(): Promise<BulkColumnInfo[]>;
@@ -66,6 +73,14 @@ export interface IBulkDatabase2<T extends { key: string }> {
     /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
     compact(): Promise<void>;
 
+    /**
+     * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
+     * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
+     * "accepted" (in memory + cross-tab), not necessarily "on disk". Call this to force durability — it's
+     * also run automatically on tab hide/close and before every merge.
+     */
+    flush(): Promise<void>;
+
     /**
      * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
      * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -2,8 +2,10 @@ import type { FileStorage } from "../FileFolderAPI";
 export declare const bulkDatabase2Timing: {
     streamSealAgeMs: number;
     mergeCheckIntervalMs: number;
+    mergeSpacingMs: number;
     firstMergeTriggerFiles: number;
     firstMergeTriggerRangeMs: number;
+    writeFlushMaxDelayMs: number;
 };
 export interface ReactiveDeps {
     observe(signal: string): void;
@@ -19,6 +21,11 @@ export declare class BulkDatabaseBase<T extends {
     protected deps: ReactiveDeps;
     private storageFactory;
     constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+    private pendingAppends;
+    private flushTimer;
+    private flushChain;
+    private currentFlushDelay;
+    private lastWriteTime;
     static clearCache(): void;
     storage: {
         (): Promise<FileStorage>;
@@ -43,6 +50,10 @@ export declare class BulkDatabaseBase<T extends {
     writeBatch(entries: T[]): Promise<void>;
     delete(key: string): Promise<void>;
     deleteBatch(keys: string[]): Promise<void>;
+    private streamAppend;
+    flush(): Promise<void>;
+    private flushPending;
+    private doFlush;
     update(entry: Partial<T> & {
         key: string;
     }): Promise<void>;
@@ -68,13 +79,21 @@ export declare class BulkDatabaseBase<T extends {
     private resolveReaders;
     private mergeFileSet;
     private canDeleteStream;
+    private mergeSpacingDelay;
     private testMerge;
+    private findDuplicateGroups;
     private formatInfo;
     private patchColumn;
     getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+    getSingleFieldObj<Column extends keyof T>(key: string, column: Column): Promise<{
+        key: string;
+        value: T[Column];
+        time: number;
+    } | undefined>;
     getColumn<Column extends keyof T>(column: Column): Promise<{
         key: string;
         value: T[Column];
+        time: number;
     }[]>;
     getKeys(): Promise<string[]>;
     private baseColumns;
@@ -84,9 +103,15 @@ export declare class BulkDatabaseBase<T extends {
     private ensureBaseColumn;
     private ensureBaseField;
     getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+    getSingleFieldObjSync<Column extends keyof T>(key: string, column: Column): {
+        key: string;
+        value: T[Column];
+        time: number;
+    } | undefined;
     getColumnSync<Column extends keyof T>(column: Column): {
         key: string;
         value: T[Column];
+        time: number;
     }[] | undefined;
     getColumnInfo(): Promise<{
         column: string;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -8,6 +8,7 @@ import { blockCache, encodeCompressedBlocks, GetRange } from "./blockCache";
 import { STREAM_EXTENSION, StreamEntry, frameRows, frameDeletes, parseStream, streamReaderFromEntries } from "./streamLog";
 import { connect as syncConnect, broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, isSyncSupported, RemoteWrite } from "./syncClient";
 import { tryAcquireMergeLock, releaseMergeLock } from "./mergeLock";
+import { isNode } from "typesafecss";
 import type { FileStorage } from "../FileFolderAPI";
 
 // ───────────────────────────────────────────────────────────────────────────────────────────────
@@ -52,6 +53,13 @@ const FIRST_MERGE_BYTES = TARGET_FILE_BYTES / 2;
 const KEY_GROUP_BYTES = 800 * 1024 * 1024;
 const DUP_THRESHOLD = 0.4;
 
+// The browser File System Access API has no real append — every "append" rewrites the whole file — so
+// streaming one write at a time is quadratic. We coalesce stream writes and flush on a RAMPING delay:
+// the first write after a lull flushes immediately (a single checkbox-then-close is saved at once), and
+// as writes keep coming the delay doubles from this step up to writeFlushMaxDelayMs, batching a burst
+// into one rewrite. (Node has a real append, so there the default delay is 0 = flush every write.)
+const WRITE_FLUSH_FIRST_STEP_MS = 250;
+
 // Time thresholds, mutable so tests can shrink them from hours to milliseconds.
 export const bulkDatabase2Timing = {
     // A writer stops appending to its current stream file once it's this old (starts a fresh one). A
@@ -60,10 +68,20 @@ export const bulkDatabase2Timing = {
     streamSealAgeMs: 10 * 60 * 60 * 1000,
     // Per-instance throttle: a write triggers at most one background testMerge scan per this interval.
     mergeCheckIntervalMs: 30 * 60 * 1000,
+    // A single testMerge can do several merges (one pass-1 + several pass-2 groups). We wait this long
+    // after each one before the next, so a burst of writes doesn't rewrite every index on disk at once —
+    // it spreads the work out to keep peak lag low (important in the browser). The merge lock is kept
+    // alive across the wait. Set to 0 to disable spacing (tests).
+    mergeSpacingMs: 5 * 60 * 1000,
     // The first merge fires when the recent (up to FIRST_MERGE_BYTES) files number more than this...
     firstMergeTriggerFiles: 20,
     // ...or span a wider write-time range than this (data trickling in slowly still gets consolidated).
     firstMergeTriggerRangeMs: 3 * 24 * 60 * 60 * 1000,
+    // Max delay (per collection) before buffered stream writes are flushed to disk; the delay ramps up to
+    // this under sustained writing (see WRITE_FLUSH_FIRST_STEP_MS). 0 = flush every write immediately —
+    // the default in Node, where append is real and cheap; the browser ramps to 15s to avoid rewriting
+    // the whole stream file per write.
+    writeFlushMaxDelayMs: isNode() ? 0 : 15 * 1000,
 };
 
 // Marks a key as deleted in the in-memory overlay.
@@ -177,7 +195,32 @@ export class BulkDatabaseBase<T extends { key: string }> {
         public readonly name: string,
         protected deps: ReactiveDeps,
         private storageFactory: StorageFactory,
-    ) { }
+    ) {
+        // Best-effort: persist buffered stream writes when the tab is hidden/closing. visibilitychange →
+        // hidden fires early enough that the (async) flush usually completes; pagehide is the backstop.
+        // No-op outside a browser window (Node).
+        if (typeof window !== "undefined") {
+            try {
+                window.addEventListener("pagehide", () => void this.flushPending());
+                if (typeof document !== "undefined") {
+                    document.addEventListener("visibilitychange", () => {
+                        if (document.visibilityState === "hidden") void this.flushPending();
+                    });
+                }
+            } catch { /* not in a DOM context */ }
+        }
+    }
+
+    // ---- buffered stream writes (per collection) ----
+    // The browser rewrites the whole stream file on every append, so we coalesce writes and flush on a
+    // ramping delay (see WRITE_FLUSH_FIRST_STEP_MS / writeFlushMaxDelayMs). Each buffered item keeps an
+    // `apply` that re-applies its overlay mutation, so resetReader (which clears the overlay) can restore
+    // writes that aren't on disk yet. Removed from the buffer only once their append succeeds.
+    private pendingAppends: { framed: Buffer; apply: () => void }[] = [];
+    private flushTimer: ReturnType<typeof setTimeout> | undefined;
+    private flushChain: Promise<void> = Promise.resolve();
+    private currentFlushDelay = 0;
+    private lastWriteTime = 0;
 
     // Block range cache is global and immutable-file-safe; clear it to simulate a cold page load
     // (e.g. between an untimed prep step and the timed benchmark).
@@ -331,7 +374,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
     }
 
     // Reset the loaded reader and all derived in-memory caches/overlay. Used only on structural
-    // changes (large direct-bulk write, rollover, compact) after the data has been persisted.
+    // changes (large direct-bulk write, rollover, compact) after the data has been persisted. Writes
+    // still buffered (not yet on disk) are re-applied so the reset doesn't drop them from reads — they
+    // aren't in the reloaded reader until their append lands.
     private resetReader() {
         this.deps.batch(() => {
             this.reader.reset();
@@ -340,6 +385,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.baseFields.clear();
             this.baseFieldsLoading.clear();
             this.overlay.clear();
+            for (const p of this.pendingAppends) p.apply();
             this.deps.invalidate(LOAD_SIGNAL);
             this.deps.invalidate(OVERLAY_SIGNAL);
         });
@@ -367,14 +413,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
             return;
         }
 
-        // Otherwise append to this thread's stream file (one cheap append) and reflect it in the
-        // overlay immediately — no reader reset.
-        const storage = await this.storage();
-        await storage.append(this.getStreamFileName(), framed);
-        this.deps.batch(() => {
-            for (const { time, row } of stamped) this.setOverlayRow(row.key as string, row, time);
-        });
+        // Reflect in the overlay + broadcast to other tabs IMMEDIATELY (in-memory + cross-tab are always
+        // current), but buffer the disk append and flush it on a ramping schedule.
+        const apply = () => { for (const { time, row } of stamped) this.setOverlayRow(row.key as string, row, time); };
+        this.deps.batch(apply);
         for (const { time, row } of stamped) syncBroadcast(this.name, { key: row.key as string, time, value: row });
+        await this.streamAppend(framed, apply);
         void this.maybeMerge();
     }
 
@@ -386,15 +430,61 @@ export class BulkDatabaseBase<T extends { key: string }> {
         if (!keys.length) return;
         void this.syncSetup();
         const stamped = keys.map(key => ({ time: getTimeUnique(), key }));
-        const storage = await this.storage();
-        await storage.append(this.getStreamFileName(), frameDeletes(stamped));
-        this.deps.batch(() => {
-            for (const { time, key } of stamped) this.setOverlayDeleted(key, time);
-        });
+        const apply = () => { for (const { time, key } of stamped) this.setOverlayDeleted(key, time); };
+        this.deps.batch(apply);
         for (const { time, key } of stamped) syncBroadcast(this.name, { key, time, deleted: true });
+        await this.streamAppend(frameDeletes(stamped), apply);
         void this.maybeMerge();
     }
 
+    // Buffers framed stream bytes and flushes on a ramping per-collection schedule (see the field block).
+    // `apply` re-applies this write's overlay mutation if resetReader clears the overlay before it's on
+    // disk. Awaits durability only on an immediate (idle/Node) flush, so a single action — then close — is
+    // saved at once; a burst returns fast and is flushed in the background.
+    private async streamAppend(framed: Buffer, apply: () => void): Promise<void> {
+        this.pendingAppends.push({ framed, apply });
+        const max = bulkDatabase2Timing.writeFlushMaxDelayMs;
+        const now = Date.now();
+        // Immediate when batching is off (Node / real append), or for the first write after a lull.
+        if (max <= 0 || this.currentFlushDelay <= 0 || now - this.lastWriteTime > max) {
+            this.lastWriteTime = now;
+            this.currentFlushDelay = max > 0 ? Math.min(max, WRITE_FLUSH_FIRST_STEP_MS) : 0;
+            await this.flushPending();
+            return;
+        }
+        // Active burst: coalesce into one scheduled flush and ramp the delay toward max.
+        this.lastWriteTime = now;
+        if (this.flushTimer === undefined) {
+            this.flushTimer = setTimeout(() => { this.flushTimer = undefined; void this.flushPending(); }, this.currentFlushDelay);
+        }
+        this.currentFlushDelay = Math.min(max, this.currentFlushDelay * 2);
+    }
+
+    // Flushes all buffered stream writes to disk as one append. Serialized (so two flushes never write the
+    // same file concurrently) and best-effort: a failed append keeps the data buffered for the next try.
+    public async flush(): Promise<void> {
+        await this.flushPending();
+    }
+
+    private async flushPending(): Promise<void> {
+        if (this.flushTimer !== undefined) { clearTimeout(this.flushTimer); this.flushTimer = undefined; }
+        this.flushChain = this.flushChain.then(() => this.doFlush()).catch(e => {
+            console.warn(`${this.name}: stream flush failed, will retry: ${(e as Error).message}`);
+        });
+        return this.flushChain;
+    }
+
+    private async doFlush(): Promise<void> {
+        if (!this.pendingAppends.length) return;
+        const batch = this.pendingAppends.slice();
+        const combined = Buffer.concat(batch.map(p => p.framed));
+        const storage = await this.storage();
+        // Throws on failure -> we don't splice, so the data stays buffered and a later flush retries it.
+        await storage.append(this.getStreamFileName(), combined);
+        // New writes added during the await are after `batch`, so removing the front is exactly the flushed set.
+        this.pendingAppends.splice(0, batch.length);
+    }
+
     public async update(entry: Partial<T> & { key: string }): Promise<void> {
         return this.updateBatch([entry]);
     }
@@ -539,6 +629,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
     public async compact(): Promise<void> {
         if (!tryAcquireMergeLock(this.name, writerId)) return; // someone else is already merging; fine
         try {
+            await this.flushPending(); // get buffered writes on disk so they're folded in
             syncBroadcastSeal(this.name);
             this.streamFileName = undefined;
             const { bulkFiles, streamFiles } = await this.listFiles();
@@ -773,17 +864,44 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return !!info && info.size === readSize;
     }
 
-    // The merge policy. Up to two passes:
-    //  1) Consolidate recent fragmentation: take the newest files up to ~FIRST_MERGE_BYTES and, if they
-    //     number more than firstMergeTriggerFiles or span more than firstMergeTriggerRangeMs, merge them
-    //     into one file. Seals first so recent stream data is complete; in Node (no cross-tab seal) only
-    //     aged streams are folded, so we never re-fold the same un-deletable stream forever.
-    //  2) Key-stratify: sort all keys, walk them in ~KEY_GROUP_BYTES groups, and rewrite the single group
-    //     whose fraction of duplicate (multi-file) keys is highest above DUP_THRESHOLD — merging every
-    //     bulk file overlapping that key range. Over time this sorts the data into key-disjoint files.
-    // Returns whether either pass merged anything.
+    // Waits mergeSpacingMs between merges so a burst of work doesn't rewrite every index at once (keeps
+    // peak lag low for the browser). Heartbeats the merge lock through the wait; returns false if another
+    // tab took the lock over, so the caller stops doing further merges.
+    private async mergeSpacingDelay(): Promise<boolean> {
+        const total = bulkDatabase2Timing.mergeSpacingMs;
+        if (total <= 0) return tryAcquireMergeLock(this.name, writerId);
+        const step = 15 * 1000; // re-stamp the lock well within its TTL
+        let waited = 0;
+        while (waited < total) {
+            await new Promise<void>(r => setTimeout(r, Math.min(step, total - waited)));
+            waited += step;
+            if (!tryAcquireMergeLock(this.name, writerId)) return false; // another tab took over
+        }
+        return true;
+    }
+
+    // The merge policy. Two passes, with a mergeSpacingMs pause after every merge (so a write burst
+    // doesn't rewrite all indexes at once):
+    //  1) Consolidate recent fragmentation (one merge): take the newest files up to ~FIRST_MERGE_BYTES
+    //     and, if they number more than firstMergeTriggerFiles or span more than firstMergeTriggerRangeMs,
+    //     merge them into one file. Seals first so recent stream data is complete; in Node (no cross-tab
+    //     seal) only aged streams are folded, so we never re-fold the same un-deletable stream forever.
+    //  2) Key-stratify (possibly several merges): sort all keys, walk them in ~KEY_GROUP_BYTES groups, and
+    //     rewrite EVERY group whose fraction of duplicate (multi-file) keys exceeds DUP_THRESHOLD —
+    //     highest-duplication first — merging the bulk files overlapping that key range. Groups have
+    //     disjoint key ranges, so one group's merge doesn't change another's duplication; we re-select
+    //     each group's files at merge time (the file set shifts as we go). Over time this sorts the data
+    //     into key-disjoint files. Returns whether any merge happened.
     private async testMerge(): Promise<boolean> {
         let merged = false;
+        await this.flushPending(); // get buffered writes on disk so this pass can fold/consider them
+        // Run a merge, pausing first if we've already merged this pass (so the pause is BETWEEN merges,
+        // never before the first or after the last). Returns false if we lost the lock — stop entirely.
+        const runMerge = async (bulk: BulkFileInfo[], stream: StreamFileInfo[]): Promise<boolean> => {
+            if (merged && !await this.mergeSpacingDelay()) return false;
+            if (await this.mergeFileSet(bulk, stream)) merged = true;
+            return true;
+        };
 
         // ── Pass 1: consolidate recent files. ──
         // Only seal (ask peers + ourselves to abandon current stream files) when cross-tab sync can
@@ -822,106 +940,128 @@ export class BulkDatabaseBase<T extends { key: string }> {
             if (triggered) {
                 const rb = recent.filter(i => i.kind === "bulk").map(i => (i.file as BulkFileInfo));
                 const rs = recent.filter(i => i.kind === "stream").map(i => (i.file as StreamFileInfo));
-                if (await this.mergeFileSet(rb, rs)) merged = true;
+                if (!await runMerge(rb, rs)) return merged;
             }
         }
 
         // ── Pass 2: key-stratify the bulk files to remove duplication. ──
-        {
+        // Compute the qualifying groups' key ranges once (over a post-pass-1 listing); their key ranges
+        // are disjoint, so they stay valid as we merge each in turn.
+        const groups = await this.findDuplicateGroups();
+        for (const g of groups) {
+            // Re-select the files overlapping this group's key range now (earlier merges shifted the set).
             const { bulkFiles } = await this.listFiles();
-            if (bulkFiles.length >= 2) {
-                const infos = await Promise.all(bulkFiles.map(async f => {
-                    try {
-                        const reader = await this.loadFileReader(f.fileName);
-                        const keys = reader.keys;
-                        let min = keys[0], max = keys[0];
-                        for (const k of keys) { if (k < min) min = k; if (k > max) max = k; }
-                        return { file: f, keys, bytes: reader.totalBytes, min, max };
-                    } catch {
-                        return { file: f, keys: [] as string[], bytes: 0, min: undefined as string | undefined, max: undefined as string | undefined };
-                    }
-                }));
-                const usable = infos.filter(i => i.keys.length > 0);
-                const keyCount = new Map<string, number>();
-                let totalSlots = 0, totalBytes = 0;
-                for (const i of usable) {
-                    totalBytes += i.bytes;
-                    for (const k of i.keys) { keyCount.set(k, (keyCount.get(k) || 0) + 1); totalSlots++; }
-                }
-                if (totalSlots > 0) {
-                    const bytesPerSlot = totalBytes / totalSlots;
-                    const sortedKeys = [...keyCount.keys()].sort();
-                    // Walk sorted keys forming ~KEY_GROUP_BYTES groups; remember the group with the highest
-                    // duplicate fraction over the threshold (the most benefit), then merge its files.
-                    let best: { lo: string; hi: string; dup: number } | undefined;
-                    let gStart = 0, gBytes = 0, gSlots = 0, gUnique = 0;
-                    for (let i = 0; i < sortedKeys.length; i++) {
-                        const c = keyCount.get(sortedKeys[i])!;
-                        gBytes += c * bytesPerSlot; gSlots += c; gUnique += 1;
-                        if (gBytes >= KEY_GROUP_BYTES || i === sortedKeys.length - 1) {
-                            const dup = (gSlots - gUnique) / gSlots;
-                            if (dup > DUP_THRESHOLD && (!best || dup > best.dup)) best = { lo: sortedKeys[gStart], hi: sortedKeys[i], dup };
-                            gStart = i + 1; gBytes = 0; gSlots = 0; gUnique = 0;
-                        }
-                    }
-                    if (best) {
-                        const lo = best.lo, hi = best.hi;
-                        const groupFiles = usable
-                            .filter(i => i.min !== undefined && i.max !== undefined && i.min <= hi && i.max >= lo)
-                            .map(i => i.file);
-                        if (groupFiles.length >= 2 && await this.mergeFileSet(groupFiles, [])) merged = true;
-                    }
-                }
-            }
+            const headers = await Promise.all(bulkFiles.map(f => this.readBulkHeader(f.fileName)));
+            const groupFiles = bulkFiles.filter((f, i) => {
+                const h = headers[i];
+                if (!h) return false;
+                // Old files without a key range are treated as spanning all keys (so always overlap).
+                if (h.minKey === undefined || h.maxKey === undefined) return true;
+                return h.minKey <= g.hi && h.maxKey >= g.lo;
+            });
+            if (groupFiles.length >= 2) { if (!await runMerge(groupFiles, [])) return merged; }
         }
 
         return merged;
     }
 
+    // Finds key-range groups worth deduping: sorts all bulk keys, walks them in ~KEY_GROUP_BYTES groups,
+    // and returns the [lo, hi] of every group whose duplicate (multi-file) key fraction exceeds
+    // DUP_THRESHOLD, highest-duplication first (most benefit). Empty when nothing is worth it.
+    private async findDuplicateGroups(): Promise<{ lo: string; hi: string; dup: number }[]> {
+        const { bulkFiles } = await this.listFiles();
+        if (bulkFiles.length < 2) return [];
+        const infos = await Promise.all(bulkFiles.map(async f => {
+            try {
+                const reader = await this.loadFileReader(f.fileName);
+                return { keys: reader.keys, bytes: reader.totalBytes };
+            } catch {
+                return { keys: [] as string[], bytes: 0 };
+            }
+        }));
+        const keyCount = new Map<string, number>();
+        let totalSlots = 0, totalBytes = 0;
+        for (const i of infos) {
+            totalBytes += i.bytes;
+            for (const k of i.keys) { keyCount.set(k, (keyCount.get(k) || 0) + 1); totalSlots++; }
+        }
+        if (totalSlots === 0) return [];
+        const bytesPerSlot = totalBytes / totalSlots;
+        const sortedKeys = [...keyCount.keys()].sort();
+        const groups: { lo: string; hi: string; dup: number }[] = [];
+        let gStart = 0, gBytes = 0, gSlots = 0, gUnique = 0;
+        for (let i = 0; i < sortedKeys.length; i++) {
+            const c = keyCount.get(sortedKeys[i])!;
+            gBytes += c * bytesPerSlot; gSlots += c; gUnique += 1;
+            if (gBytes >= KEY_GROUP_BYTES || i === sortedKeys.length - 1) {
+                const dup = (gSlots - gUnique) / gSlots;
+                if (dup > DUP_THRESHOLD) groups.push({ lo: sortedKeys[gStart], hi: sortedKeys[i], dup });
+                gStart = i + 1; gBytes = 0; gSlots = 0; gUnique = 0;
+            }
+        }
+        groups.sort((a, b) => b.dup - a.dup);
+        return groups;
+    }
+
     private formatInfo(reader: ResolvedReader): string {
         return `(collection has ${blue(formatNumber(reader.rowCount))} rows, ${blue(formatNumber(reader.totalBytes))}B)`;
     }
 
     // Applies the overlay (pending writes/deletes) on top of a base column. No-op when empty. An
-    // overlay entry that doesn't include this column leaves the base (disk) value in place — a partial
-    // write/update only overrides the columns it set; everything else falls through.
-    private patchColumn(base: { key: string; value: unknown }[], column: string): { key: string; value: unknown }[] {
+    // overlay entry that doesn't include this column leaves the base (disk) value+time in place — a
+    // partial write/update only overrides the columns it set; everything else falls through. An overlay
+    // override carries the overlay write's time (when that pending write happened).
+    private patchColumn(base: { key: string; value: unknown; time: number }[], column: string): { key: string; value: unknown; time: number }[] {
         if (this.overlay.size === 0) return base;
-        const map = new Map(base.map(e => [e.key, e.value]));
+        const map = new Map(base.map(e => [e.key, { value: e.value, time: e.time }]));
         for (const [key, entry] of this.overlay) {
             if (entry.value === DELETED) { map.delete(key); continue; }
-            if (column in entry.value) map.set(key, entry.value[column]);
-            else if (!map.has(key)) map.set(key, undefined);
+            if (column in entry.value) map.set(key, { value: entry.value[column], time: entry.time });
+            else if (!map.has(key)) map.set(key, { value: undefined, time: entry.time });
         }
-        return [...map].map(([key, value]) => ({ key, value }));
+        return [...map].map(([key, v]) => ({ key, value: v.value, time: v.time }));
     }
 
     // ---- async reads (overlay-aware) ----
 
     public async getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined> {
+        return (await this.getSingleFieldObj(key, column))?.value;
+    }
+
+    // Like getSingleField, but returns the same shape a getColumn entry has: { key, value, time }, where
+    // time is roughly when that value last changed (the resolved write-time; for a row-merged value it's
+    // the newest contributing write). Returns undefined only when the key isn't present/live.
+    public async getSingleFieldObj<Column extends keyof T>(key: string, column: Column): Promise<{ key: string; value: T[Column]; time: number } | undefined> {
         void this.syncSetup();
+        const col = String(column);
         const entry = this.overlay.get(key);
         if (entry !== undefined) {
             if (entry.value === DELETED) return undefined;
-            if (String(column) in entry.value) return entry.value[String(column)] as T[Column];
-            // column not set in the overlay entry — fall through to disk
+            if (col in entry.value) return { key, value: entry.value[col] as T[Column], time: entry.time };
+            // column not set in the overlay entry — fall through to disk for this column
         }
         let time = Date.now();
         let reader = await this.reader();
-        let result = await reader.getSingleField(key, String(column)) as T[Column] | undefined;
+        const r = await reader.getSingleField(key, col);
         time = Date.now() - time;
         if (time > 50) {
-            console.log(`${blue(`${this.name}.getSingleField(${JSON.stringify(key)}, ${JSON.stringify(column)})`)} took ${red(formatTime(time))} ${this.formatInfo(reader)}`);
+            console.log(`${blue(`${this.name}.getSingleFieldObj(${JSON.stringify(key)}, ${JSON.stringify(column)})`)} took ${red(formatTime(time))} ${this.formatInfo(reader)}`);
         }
-        return result;
+        if (r === undefined) {
+            // Not live on disk; but if the overlay holds the key (a partial write of a not-yet-on-disk
+            // key) it's live with this column unset.
+            if (entry !== undefined && entry.value !== DELETED) return { key, value: undefined as T[Column], time: entry.time };
+            return undefined;
+        }
+        return { key, value: r.value as T[Column], time: r.time };
     }
 
-    public async getColumn<Column extends keyof T>(column: Column): Promise<{ key: string; value: T[Column] }[]> {
+    public async getColumn<Column extends keyof T>(column: Column): Promise<{ key: string; value: T[Column]; time: number }[]> {
         void this.syncSetup();
         let time = Date.now();
         let reader = await this.reader();
-        let base = await reader.getColumn(String(column)) as { key: string; value: T[Column] }[];
-        let result = this.patchColumn(base, String(column)) as { key: string; value: T[Column] }[];
+        let base = await reader.getColumn(String(column));
+        let result = this.patchColumn(base, String(column)) as { key: string; value: T[Column]; time: number }[];
         time = Date.now() - time;
         if (time > 50) {
             console.log(`${blue(`${this.name}.getColumn(${JSON.stringify(column)})`)} took ${red(formatTime(time))} ${this.formatInfo(reader)}`);
@@ -947,9 +1087,11 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // loaded once and cached; the overlay is layered on top (we can't async-cache the combined result
     // because the overlay mutates).
 
-    private baseColumns = new Map<string, { key: string; value: unknown }[]>();
+    private baseColumns = new Map<string, { key: string; value: unknown; time: number }[]>();
     private baseColumnsLoading = new Set<string>();
-    private baseFields = new Map<string, unknown>();
+    // The disk-resolved field: { value, time } when the key is live on disk, or undefined when it isn't
+    // (Map.has distinguishes "loaded" from "not loaded yet").
+    private baseFields = new Map<string, { value: unknown; time: number } | undefined>();
     private baseFieldsLoading = new Set<string>();
 
     private ensureBaseColumn(column: string) {
@@ -972,9 +1114,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
         this.baseFieldsLoading.add(cacheKey);
         void (async () => {
             let reader = await this.reader();
-            let value = await reader.getSingleField(key, column);
+            let resolved = await reader.getSingleField(key, column);
             this.deps.batch(() => {
-                this.baseFields.set(cacheKey, value);
+                this.baseFields.set(cacheKey, resolved);
                 this.baseFieldsLoading.delete(cacheKey);
                 this.deps.invalidate(LOAD_SIGNAL);
             });
@@ -982,6 +1124,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
     }
 
     public getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined {
+        return this.getSingleFieldObjSync(key, column)?.value;
+    }
+
+    // Sync (reactive) counterpart of getSingleFieldObj: { key, value, time } once loaded, undefined while
+    // loading or when the key isn't present/live. time is roughly when the value last changed.
+    public getSingleFieldObjSync<Column extends keyof T>(key: string, column: Column): { key: string; value: T[Column]; time: number } | undefined {
         void this.syncSetup();
         this.deps.observe(LOAD_SIGNAL);
         this.deps.observe(key);
@@ -989,18 +1137,24 @@ export class BulkDatabaseBase<T extends { key: string }> {
         let entry = this.overlay.get(key);
         if (entry !== undefined) {
             if (entry.value === DELETED) return undefined;
-            if (col in entry.value) return entry.value[col] as T[Column];
-            // column not set in the overlay entry — fall through to the base field cache
+            if (col in entry.value) return { key, value: entry.value[col] as T[Column], time: entry.time };
+            // column not set in the overlay entry — fall through to the base field cache for this column
         }
         let cacheKey = nullJoin(col, key);
         if (!this.baseFields.has(cacheKey)) {
             this.ensureBaseField(key, col);
             return undefined;
         }
-        return this.baseFields.get(cacheKey) as T[Column] | undefined;
+        const base = this.baseFields.get(cacheKey);
+        if (base === undefined) {
+            // Not live on disk; but an overlay entry for the key (partial write) makes it live, column unset.
+            if (entry !== undefined && entry.value !== DELETED) return { key, value: undefined as T[Column], time: entry.time };
+            return undefined;
+        }
+        return { key, value: base.value as T[Column], time: base.time };
     }
 
-    public getColumnSync<Column extends keyof T>(column: Column): { key: string; value: T[Column] }[] | undefined {
+    public getColumnSync<Column extends keyof T>(column: Column): { key: string; value: T[Column]; time: number }[] | undefined {
         void this.syncSetup();
         this.deps.observe(LOAD_SIGNAL);
         // Observe the overlay-wide signal so we recompute once the base arrives or the overlay changes.
@@ -1011,7 +1165,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.ensureBaseColumn(col);
             return undefined;
         }
-        return this.patchColumn(base, col) as { key: string; value: T[Column] }[];
+        return this.patchColumn(base, col) as { key: string; value: T[Column]; time: number }[];
     }
 
     public async getColumnInfo() {
@@ -1031,15 +1185,17 @@ export class BulkDatabaseBase<T extends { key: string }> {
     }
 }
 
-// The merged, time-resolved view over all readers. getColumn/getSingleField return plain resolved
-// values (not {value,time}); the base layers the overlay on top of these.
+// The merged, time-resolved view over all readers. getColumn/getSingleField return the resolved value
+// AND its write-time (so reads can expose roughly when a value last changed); the base layers the
+// overlay on top of these. getSingleField returns undefined only when the key isn't live (deleted or
+// absent) — a live key whose column is merely unset returns { value: undefined, time: rowTime }.
 type ResolvedReader = {
     rowCount: number;
     totalBytes: number;
     keys: string[];
     columns: { column: string; byteSize: number }[];
-    getColumn: (column: string) => Promise<{ key: string; value: unknown }[]>;
-    getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+    getColumn: (column: string) => Promise<{ key: string; value: unknown; time: number }[]>;
+    getSingleField: (key: string, column: string) => Promise<{ value: unknown; time: number } | undefined>;
 };
 
 // Resolve every read by ACTUAL write-time across all readers (stream + bulk), per key and per column:
@@ -1097,10 +1253,15 @@ async function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): Promise<R
                     if (!cell || cell.value === ABSENT) continue;
                     if (cell.time > bestTime) { bestTime = cell.time; bestVal = cell.value; found = true; }
                 }
-                return { key, value: (found && bestTime > delOf(key)) ? bestVal : undefined };
+                // time = the column value's write-time when this column has one, else the row's last
+                // write-time (the key is live but never set this column).
+                const live = found && bestTime > delOf(key);
+                return { key, value: live ? bestVal : undefined, time: live ? bestTime : (keyTime.get(key) ?? 0) };
             });
         },
         async getSingleField(key, column) {
+            const kt = keyTime.get(key);
+            if (kt === undefined || kt <= delOf(key)) return undefined; // key not live
             let bestTime = -Infinity;
             let bestVal: unknown;
             let found = false;
@@ -1110,7 +1271,8 @@ async function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): Promise<R
                 if (r === ABSENT) continue;
                 if (r.time > bestTime) { bestTime = r.time; bestVal = r.value; found = true; }
             }
-            return (found && bestTime > delOf(key)) ? bestVal : undefined;
+            const live = found && bestTime > delOf(key);
+            return { value: live ? bestVal : undefined, time: live ? bestTime : kt };
         },
     };
 }

package/storage/BulkDatabase2/mergeLock.ts

@@ -21,15 +21,17 @@ function lockKey(collection: string): string {
     return "bulkDatabase2-merge:" + collection;
 }
 
-// Returns true if we now hold the lock (or there's no localStorage to coordinate through, in which
-// case we proceed and rely on the manifest backstop). Returns false if another tab holds a fresh lock.
+// Returns true if we now hold the lock (or there's no localStorage to coordinate through, in which case
+// we proceed — concurrent merges are harmless). Returns false only if a DIFFERENT holder has a fresh
+// lock. Re-stamping our own lock always succeeds, so this doubles as a heartbeat: call it periodically
+// during a long, spaced-out merge to keep the lock alive (and detect if another tab took it over).
 export function tryAcquireMergeLock(collection: string, holderId: string): boolean {
     const ls = getLocalStorage();
     if (!ls) return true;
     const key = lockKey(collection);
     const now = Date.now();
     const existing = ls.getItem(key);
-    if (existing) {
+    if (existing && !existing.startsWith(holderId + ":")) {
         const t = parseInt(existing.slice(existing.lastIndexOf(":") + 1), 10);
         if (Number.isFinite(t) && now - t < LOCK_TTL_MS) return false;
     }

package/storage/BulkDatabase2/streamLog.ts

@@ -24,13 +24,10 @@ function frame(payload: Buffer): Buffer {
     return Buffer.concat([len, payload, len]);
 }
 
-// We deliberately do NOT batch/debounce writes here — each write is framed and appended (and flushed)
-// immediately. If a caller makes many individual writes and that causes lag, the fix is for them to
-// call writeBatch/deleteBatch with the whole set, not for us to silently coalesce. Caller-side
-// batching is strictly faster than anything we could do (it knows the full set up front), and not
-// batching gives the lowest possible latency for callers who genuinely want single writes — which
-// matters because the tab can be closed at any moment, so we want each write on disk as soon as
-// possible rather than sitting in a pending buffer that a close would lose.
+// Framing only — no batching here. BulkDatabase2 coalesces and flushes these framed bytes on a ramping
+// per-collection schedule (see streamAppend in BulkDatabaseBase), because the browser File System Access
+// API rewrites the whole file on every append, so one-append-per-write is quadratic. The first write
+// after a lull still flushes immediately, so a single action then a tab close is saved at once.
 
 // Times are assigned by the caller (BulkDatabase2) so the exact same timestamp lands on disk, in the
 // in-memory overlay, and in the cross-tab broadcast — keeping the global write order consistent.