sliftutils 1.4.2 → 1.4.3

package/index.d.ts

@@ -851,6 +851,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
         getReaderInfo(): Promise<BulkReaderInfo>;
         /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
         compact(): Promise<void>;
+        /**
+         * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
+         * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
+         * "accepted" (in memory + cross-tab), not necessarily "on disk". Call this to force durability — it's
+         * also run automatically on tab hide/close and before every merge.
+         */
+        flush(): Promise<void>;
         /**
          * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
          * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
@@ -878,8 +885,10 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
     export declare const bulkDatabase2Timing: {
         streamSealAgeMs: number;
         mergeCheckIntervalMs: number;
+        mergeSpacingMs: number;
         firstMergeTriggerFiles: number;
         firstMergeTriggerRangeMs: number;
+        writeFlushMaxDelayMs: number;
     };
     export interface ReactiveDeps {
         observe(signal: string): void;
@@ -895,6 +904,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         protected deps: ReactiveDeps;
         private storageFactory;
         constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+        private pendingAppends;
+        private flushTimer;
+        private flushChain;
+        private currentFlushDelay;
+        private lastWriteTime;
         static clearCache(): void;
         storage: {
             (): Promise<FileStorage>;
@@ -919,6 +933,10 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         writeBatch(entries: T[]): Promise<void>;
         delete(key: string): Promise<void>;
         deleteBatch(keys: string[]): Promise<void>;
+        private streamAppend;
+        flush(): Promise<void>;
+        private flushPending;
+        private doFlush;
         update(entry: Partial<T> & {
             key: string;
         }): Promise<void>;
@@ -944,7 +962,9 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private resolveReaders;
         private mergeFileSet;
         private canDeleteStream;
+        private mergeSpacingDelay;
         private testMerge;
+        private findDuplicateGroups;
         private formatInfo;
         private patchColumn;
         getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.2",
+    "version": "1.4.3",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -86,6 +86,13 @@ export interface IBulkDatabase2<T extends {
     getReaderInfo(): Promise<BulkReaderInfo>;
     /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
     compact(): Promise<void>;
+    /**
+     * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
+     * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
+     * "accepted" (in memory + cross-tab), not necessarily "on disk". Call this to force durability — it's
+     * also run automatically on tab hide/close and before every merge.
+     */
+    flush(): Promise<void>;
     /**
      * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
      * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -73,6 +73,14 @@ export interface IBulkDatabase2<T extends { key: string }> {
     /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
     compact(): Promise<void>;
 
+    /**
+     * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
+     * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
+     * "accepted" (in memory + cross-tab), not necessarily "on disk". Call this to force durability — it's
+     * also run automatically on tab hide/close and before every merge.
+     */
+    flush(): Promise<void>;
+
     /**
      * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
      * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -2,8 +2,10 @@ import type { FileStorage } from "../FileFolderAPI";
 export declare const bulkDatabase2Timing: {
     streamSealAgeMs: number;
     mergeCheckIntervalMs: number;
+    mergeSpacingMs: number;
     firstMergeTriggerFiles: number;
     firstMergeTriggerRangeMs: number;
+    writeFlushMaxDelayMs: number;
 };
 export interface ReactiveDeps {
     observe(signal: string): void;
@@ -19,6 +21,11 @@ export declare class BulkDatabaseBase<T extends {
     protected deps: ReactiveDeps;
     private storageFactory;
     constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+    private pendingAppends;
+    private flushTimer;
+    private flushChain;
+    private currentFlushDelay;
+    private lastWriteTime;
     static clearCache(): void;
     storage: {
         (): Promise<FileStorage>;
@@ -43,6 +50,10 @@ export declare class BulkDatabaseBase<T extends {
     writeBatch(entries: T[]): Promise<void>;
     delete(key: string): Promise<void>;
     deleteBatch(keys: string[]): Promise<void>;
+    private streamAppend;
+    flush(): Promise<void>;
+    private flushPending;
+    private doFlush;
     update(entry: Partial<T> & {
         key: string;
     }): Promise<void>;
@@ -68,7 +79,9 @@ export declare class BulkDatabaseBase<T extends {
     private resolveReaders;
     private mergeFileSet;
     private canDeleteStream;
+    private mergeSpacingDelay;
     private testMerge;
+    private findDuplicateGroups;
     private formatInfo;
     private patchColumn;
     getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -8,6 +8,7 @@ import { blockCache, encodeCompressedBlocks, GetRange } from "./blockCache";
 import { STREAM_EXTENSION, StreamEntry, frameRows, frameDeletes, parseStream, streamReaderFromEntries } from "./streamLog";
 import { connect as syncConnect, broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, isSyncSupported, RemoteWrite } from "./syncClient";
 import { tryAcquireMergeLock, releaseMergeLock } from "./mergeLock";
+import { isNode } from "typesafecss";
 import type { FileStorage } from "../FileFolderAPI";
 
 // ───────────────────────────────────────────────────────────────────────────────────────────────
@@ -52,6 +53,13 @@ const FIRST_MERGE_BYTES = TARGET_FILE_BYTES / 2;
 const KEY_GROUP_BYTES = 800 * 1024 * 1024;
 const DUP_THRESHOLD = 0.4;
 
+// The browser File System Access API has no real append — every "append" rewrites the whole file — so
+// streaming one write at a time is quadratic. We coalesce stream writes and flush on a RAMPING delay:
+// the first write after a lull flushes immediately (a single checkbox-then-close is saved at once), and
+// as writes keep coming the delay doubles from this step up to writeFlushMaxDelayMs, batching a burst
+// into one rewrite. (Node has a real append, so there the default delay is 0 = flush every write.)
+const WRITE_FLUSH_FIRST_STEP_MS = 250;
+
 // Time thresholds, mutable so tests can shrink them from hours to milliseconds.
 export const bulkDatabase2Timing = {
     // A writer stops appending to its current stream file once it's this old (starts a fresh one). A
@@ -60,10 +68,20 @@ export const bulkDatabase2Timing = {
     streamSealAgeMs: 10 * 60 * 60 * 1000,
     // Per-instance throttle: a write triggers at most one background testMerge scan per this interval.
     mergeCheckIntervalMs: 30 * 60 * 1000,
+    // A single testMerge can do several merges (one pass-1 + several pass-2 groups). We wait this long
+    // after each one before the next, so a burst of writes doesn't rewrite every index on disk at once —
+    // it spreads the work out to keep peak lag low (important in the browser). The merge lock is kept
+    // alive across the wait. Set to 0 to disable spacing (tests).
+    mergeSpacingMs: 5 * 60 * 1000,
     // The first merge fires when the recent (up to FIRST_MERGE_BYTES) files number more than this...
     firstMergeTriggerFiles: 20,
     // ...or span a wider write-time range than this (data trickling in slowly still gets consolidated).
     firstMergeTriggerRangeMs: 3 * 24 * 60 * 60 * 1000,
+    // Max delay (per collection) before buffered stream writes are flushed to disk; the delay ramps up to
+    // this under sustained writing (see WRITE_FLUSH_FIRST_STEP_MS). 0 = flush every write immediately —
+    // the default in Node, where append is real and cheap; the browser ramps to 15s to avoid rewriting
+    // the whole stream file per write.
+    writeFlushMaxDelayMs: isNode() ? 0 : 15 * 1000,
 };
 
 // Marks a key as deleted in the in-memory overlay.
@@ -177,7 +195,32 @@ export class BulkDatabaseBase<T extends { key: string }> {
         public readonly name: string,
         protected deps: ReactiveDeps,
         private storageFactory: StorageFactory,
-    ) { }
+    ) {
+        // Best-effort: persist buffered stream writes when the tab is hidden/closing. visibilitychange →
+        // hidden fires early enough that the (async) flush usually completes; pagehide is the backstop.
+        // No-op outside a browser window (Node).
+        if (typeof window !== "undefined") {
+            try {
+                window.addEventListener("pagehide", () => void this.flushPending());
+                if (typeof document !== "undefined") {
+                    document.addEventListener("visibilitychange", () => {
+                        if (document.visibilityState === "hidden") void this.flushPending();
+                    });
+                }
+            } catch { /* not in a DOM context */ }
+        }
+    }
+
+    // ---- buffered stream writes (per collection) ----
+    // The browser rewrites the whole stream file on every append, so we coalesce writes and flush on a
+    // ramping delay (see WRITE_FLUSH_FIRST_STEP_MS / writeFlushMaxDelayMs). Each buffered item keeps an
+    // `apply` that re-applies its overlay mutation, so resetReader (which clears the overlay) can restore
+    // writes that aren't on disk yet. Removed from the buffer only once their append succeeds.
+    private pendingAppends: { framed: Buffer; apply: () => void }[] = [];
+    private flushTimer: ReturnType<typeof setTimeout> | undefined;
+    private flushChain: Promise<void> = Promise.resolve();
+    private currentFlushDelay = 0;
+    private lastWriteTime = 0;
 
     // Block range cache is global and immutable-file-safe; clear it to simulate a cold page load
     // (e.g. between an untimed prep step and the timed benchmark).
@@ -331,7 +374,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
     }
 
     // Reset the loaded reader and all derived in-memory caches/overlay. Used only on structural
-    // changes (large direct-bulk write, rollover, compact) after the data has been persisted.
+    // changes (large direct-bulk write, rollover, compact) after the data has been persisted. Writes
+    // still buffered (not yet on disk) are re-applied so the reset doesn't drop them from reads — they
+    // aren't in the reloaded reader until their append lands.
     private resetReader() {
         this.deps.batch(() => {
             this.reader.reset();
@@ -340,6 +385,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.baseFields.clear();
             this.baseFieldsLoading.clear();
             this.overlay.clear();
+            for (const p of this.pendingAppends) p.apply();
             this.deps.invalidate(LOAD_SIGNAL);
             this.deps.invalidate(OVERLAY_SIGNAL);
         });
@@ -367,14 +413,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
             return;
         }
 
-        // Otherwise append to this thread's stream file (one cheap append) and reflect it in the
-        // overlay immediately — no reader reset.
-        const storage = await this.storage();
-        await storage.append(this.getStreamFileName(), framed);
-        this.deps.batch(() => {
-            for (const { time, row } of stamped) this.setOverlayRow(row.key as string, row, time);
-        });
+        // Reflect in the overlay + broadcast to other tabs IMMEDIATELY (in-memory + cross-tab are always
+        // current), but buffer the disk append and flush it on a ramping schedule.
+        const apply = () => { for (const { time, row } of stamped) this.setOverlayRow(row.key as string, row, time); };
+        this.deps.batch(apply);
         for (const { time, row } of stamped) syncBroadcast(this.name, { key: row.key as string, time, value: row });
+        await this.streamAppend(framed, apply);
         void this.maybeMerge();
     }
 
@@ -386,15 +430,61 @@ export class BulkDatabaseBase<T extends { key: string }> {
         if (!keys.length) return;
         void this.syncSetup();
         const stamped = keys.map(key => ({ time: getTimeUnique(), key }));
-        const storage = await this.storage();
-        await storage.append(this.getStreamFileName(), frameDeletes(stamped));
-        this.deps.batch(() => {
-            for (const { time, key } of stamped) this.setOverlayDeleted(key, time);
-        });
+        const apply = () => { for (const { time, key } of stamped) this.setOverlayDeleted(key, time); };
+        this.deps.batch(apply);
         for (const { time, key } of stamped) syncBroadcast(this.name, { key, time, deleted: true });
+        await this.streamAppend(frameDeletes(stamped), apply);
         void this.maybeMerge();
     }
 
+    // Buffers framed stream bytes and flushes on a ramping per-collection schedule (see the field block).
+    // `apply` re-applies this write's overlay mutation if resetReader clears the overlay before it's on
+    // disk. Awaits durability only on an immediate (idle/Node) flush, so a single action — then close — is
+    // saved at once; a burst returns fast and is flushed in the background.
+    private async streamAppend(framed: Buffer, apply: () => void): Promise<void> {
+        this.pendingAppends.push({ framed, apply });
+        const max = bulkDatabase2Timing.writeFlushMaxDelayMs;
+        const now = Date.now();
+        // Immediate when batching is off (Node / real append), or for the first write after a lull.
+        if (max <= 0 || this.currentFlushDelay <= 0 || now - this.lastWriteTime > max) {
+            this.lastWriteTime = now;
+            this.currentFlushDelay = max > 0 ? Math.min(max, WRITE_FLUSH_FIRST_STEP_MS) : 0;
+            await this.flushPending();
+            return;
+        }
+        // Active burst: coalesce into one scheduled flush and ramp the delay toward max.
+        this.lastWriteTime = now;
+        if (this.flushTimer === undefined) {
+            this.flushTimer = setTimeout(() => { this.flushTimer = undefined; void this.flushPending(); }, this.currentFlushDelay);
+        }
+        this.currentFlushDelay = Math.min(max, this.currentFlushDelay * 2);
+    }
+
+    // Flushes all buffered stream writes to disk as one append. Serialized (so two flushes never write the
+    // same file concurrently) and best-effort: a failed append keeps the data buffered for the next try.
+    public async flush(): Promise<void> {
+        await this.flushPending();
+    }
+
+    private async flushPending(): Promise<void> {
+        if (this.flushTimer !== undefined) { clearTimeout(this.flushTimer); this.flushTimer = undefined; }
+        this.flushChain = this.flushChain.then(() => this.doFlush()).catch(e => {
+            console.warn(`${this.name}: stream flush failed, will retry: ${(e as Error).message}`);
+        });
+        return this.flushChain;
+    }
+
+    private async doFlush(): Promise<void> {
+        if (!this.pendingAppends.length) return;
+        const batch = this.pendingAppends.slice();
+        const combined = Buffer.concat(batch.map(p => p.framed));
+        const storage = await this.storage();
+        // Throws on failure -> we don't splice, so the data stays buffered and a later flush retries it.
+        await storage.append(this.getStreamFileName(), combined);
+        // New writes added during the await are after `batch`, so removing the front is exactly the flushed set.
+        this.pendingAppends.splice(0, batch.length);
+    }
+
     public async update(entry: Partial<T> & { key: string }): Promise<void> {
         return this.updateBatch([entry]);
     }
@@ -539,6 +629,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
     public async compact(): Promise<void> {
         if (!tryAcquireMergeLock(this.name, writerId)) return; // someone else is already merging; fine
         try {
+            await this.flushPending(); // get buffered writes on disk so they're folded in
             syncBroadcastSeal(this.name);
             this.streamFileName = undefined;
             const { bulkFiles, streamFiles } = await this.listFiles();
@@ -773,17 +864,44 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return !!info && info.size === readSize;
     }
 
-    // The merge policy. Up to two passes:
-    //  1) Consolidate recent fragmentation: take the newest files up to ~FIRST_MERGE_BYTES and, if they
-    //     number more than firstMergeTriggerFiles or span more than firstMergeTriggerRangeMs, merge them
-    //     into one file. Seals first so recent stream data is complete; in Node (no cross-tab seal) only
-    //     aged streams are folded, so we never re-fold the same un-deletable stream forever.
-    //  2) Key-stratify: sort all keys, walk them in ~KEY_GROUP_BYTES groups, and rewrite the single group
-    //     whose fraction of duplicate (multi-file) keys is highest above DUP_THRESHOLD — merging every
-    //     bulk file overlapping that key range. Over time this sorts the data into key-disjoint files.
-    // Returns whether either pass merged anything.
+    // Waits mergeSpacingMs between merges so a burst of work doesn't rewrite every index at once (keeps
+    // peak lag low for the browser). Heartbeats the merge lock through the wait; returns false if another
+    // tab took the lock over, so the caller stops doing further merges.
+    private async mergeSpacingDelay(): Promise<boolean> {
+        const total = bulkDatabase2Timing.mergeSpacingMs;
+        if (total <= 0) return tryAcquireMergeLock(this.name, writerId);
+        const step = 15 * 1000; // re-stamp the lock well within its TTL
+        let waited = 0;
+        while (waited < total) {
+            await new Promise<void>(r => setTimeout(r, Math.min(step, total - waited)));
+            waited += step;
+            if (!tryAcquireMergeLock(this.name, writerId)) return false; // another tab took over
+        }
+        return true;
+    }
+
+    // The merge policy. Two passes, with a mergeSpacingMs pause after every merge (so a write burst
+    // doesn't rewrite all indexes at once):
+    //  1) Consolidate recent fragmentation (one merge): take the newest files up to ~FIRST_MERGE_BYTES
+    //     and, if they number more than firstMergeTriggerFiles or span more than firstMergeTriggerRangeMs,
+    //     merge them into one file. Seals first so recent stream data is complete; in Node (no cross-tab
+    //     seal) only aged streams are folded, so we never re-fold the same un-deletable stream forever.
+    //  2) Key-stratify (possibly several merges): sort all keys, walk them in ~KEY_GROUP_BYTES groups, and
+    //     rewrite EVERY group whose fraction of duplicate (multi-file) keys exceeds DUP_THRESHOLD —
+    //     highest-duplication first — merging the bulk files overlapping that key range. Groups have
+    //     disjoint key ranges, so one group's merge doesn't change another's duplication; we re-select
+    //     each group's files at merge time (the file set shifts as we go). Over time this sorts the data
+    //     into key-disjoint files. Returns whether any merge happened.
     private async testMerge(): Promise<boolean> {
         let merged = false;
+        await this.flushPending(); // get buffered writes on disk so this pass can fold/consider them
+        // Run a merge, pausing first if we've already merged this pass (so the pause is BETWEEN merges,
+        // never before the first or after the last). Returns false if we lost the lock — stop entirely.
+        const runMerge = async (bulk: BulkFileInfo[], stream: StreamFileInfo[]): Promise<boolean> => {
+            if (merged && !await this.mergeSpacingDelay()) return false;
+            if (await this.mergeFileSet(bulk, stream)) merged = true;
+            return true;
+        };
 
         // ── Pass 1: consolidate recent files. ──
         // Only seal (ask peers + ourselves to abandon current stream files) when cross-tab sync can
@@ -822,62 +940,69 @@ export class BulkDatabaseBase<T extends { key: string }> {
             if (triggered) {
                 const rb = recent.filter(i => i.kind === "bulk").map(i => (i.file as BulkFileInfo));
                 const rs = recent.filter(i => i.kind === "stream").map(i => (i.file as StreamFileInfo));
-                if (await this.mergeFileSet(rb, rs)) merged = true;
+                if (!await runMerge(rb, rs)) return merged;
             }
         }
 
         // ── Pass 2: key-stratify the bulk files to remove duplication. ──
-        {
+        // Compute the qualifying groups' key ranges once (over a post-pass-1 listing); their key ranges
+        // are disjoint, so they stay valid as we merge each in turn.
+        const groups = await this.findDuplicateGroups();
+        for (const g of groups) {
+            // Re-select the files overlapping this group's key range now (earlier merges shifted the set).
             const { bulkFiles } = await this.listFiles();
-            if (bulkFiles.length >= 2) {
-                const infos = await Promise.all(bulkFiles.map(async f => {
-                    try {
-                        const reader = await this.loadFileReader(f.fileName);
-                        const keys = reader.keys;
-                        let min = keys[0], max = keys[0];
-                        for (const k of keys) { if (k < min) min = k; if (k > max) max = k; }
-                        return { file: f, keys, bytes: reader.totalBytes, min, max };
-                    } catch {
-                        return { file: f, keys: [] as string[], bytes: 0, min: undefined as string | undefined, max: undefined as string | undefined };
-                    }
-                }));
-                const usable = infos.filter(i => i.keys.length > 0);
-                const keyCount = new Map<string, number>();
-                let totalSlots = 0, totalBytes = 0;
-                for (const i of usable) {
-                    totalBytes += i.bytes;
-                    for (const k of i.keys) { keyCount.set(k, (keyCount.get(k) || 0) + 1); totalSlots++; }
-                }
-                if (totalSlots > 0) {
-                    const bytesPerSlot = totalBytes / totalSlots;
-                    const sortedKeys = [...keyCount.keys()].sort();
-                    // Walk sorted keys forming ~KEY_GROUP_BYTES groups; remember the group with the highest
-                    // duplicate fraction over the threshold (the most benefit), then merge its files.
-                    let best: { lo: string; hi: string; dup: number } | undefined;
-                    let gStart = 0, gBytes = 0, gSlots = 0, gUnique = 0;
-                    for (let i = 0; i < sortedKeys.length; i++) {
-                        const c = keyCount.get(sortedKeys[i])!;
-                        gBytes += c * bytesPerSlot; gSlots += c; gUnique += 1;
-                        if (gBytes >= KEY_GROUP_BYTES || i === sortedKeys.length - 1) {
-                            const dup = (gSlots - gUnique) / gSlots;
-                            if (dup > DUP_THRESHOLD && (!best || dup > best.dup)) best = { lo: sortedKeys[gStart], hi: sortedKeys[i], dup };
-                            gStart = i + 1; gBytes = 0; gSlots = 0; gUnique = 0;
-                        }
-                    }
-                    if (best) {
-                        const lo = best.lo, hi = best.hi;
-                        const groupFiles = usable
-                            .filter(i => i.min !== undefined && i.max !== undefined && i.min <= hi && i.max >= lo)
-                            .map(i => i.file);
-                        if (groupFiles.length >= 2 && await this.mergeFileSet(groupFiles, [])) merged = true;
-                    }
-                }
-            }
+            const headers = await Promise.all(bulkFiles.map(f => this.readBulkHeader(f.fileName)));
+            const groupFiles = bulkFiles.filter((f, i) => {
+                const h = headers[i];
+                if (!h) return false;
+                // Old files without a key range are treated as spanning all keys (so always overlap).
+                if (h.minKey === undefined || h.maxKey === undefined) return true;
+                return h.minKey <= g.hi && h.maxKey >= g.lo;
+            });
+            if (groupFiles.length >= 2) { if (!await runMerge(groupFiles, [])) return merged; }
         }
 
         return merged;
     }
 
+    // Finds key-range groups worth deduping: sorts all bulk keys, walks them in ~KEY_GROUP_BYTES groups,
+    // and returns the [lo, hi] of every group whose duplicate (multi-file) key fraction exceeds
+    // DUP_THRESHOLD, highest-duplication first (most benefit). Empty when nothing is worth it.
+    private async findDuplicateGroups(): Promise<{ lo: string; hi: string; dup: number }[]> {
+        const { bulkFiles } = await this.listFiles();
+        if (bulkFiles.length < 2) return [];
+        const infos = await Promise.all(bulkFiles.map(async f => {
+            try {
+                const reader = await this.loadFileReader(f.fileName);
+                return { keys: reader.keys, bytes: reader.totalBytes };
+            } catch {
+                return { keys: [] as string[], bytes: 0 };
+            }
+        }));
+        const keyCount = new Map<string, number>();
+        let totalSlots = 0, totalBytes = 0;
+        for (const i of infos) {
+            totalBytes += i.bytes;
+            for (const k of i.keys) { keyCount.set(k, (keyCount.get(k) || 0) + 1); totalSlots++; }
+        }
+        if (totalSlots === 0) return [];
+        const bytesPerSlot = totalBytes / totalSlots;
+        const sortedKeys = [...keyCount.keys()].sort();
+        const groups: { lo: string; hi: string; dup: number }[] = [];
+        let gStart = 0, gBytes = 0, gSlots = 0, gUnique = 0;
+        for (let i = 0; i < sortedKeys.length; i++) {
+            const c = keyCount.get(sortedKeys[i])!;
+            gBytes += c * bytesPerSlot; gSlots += c; gUnique += 1;
+            if (gBytes >= KEY_GROUP_BYTES || i === sortedKeys.length - 1) {
+                const dup = (gSlots - gUnique) / gSlots;
+                if (dup > DUP_THRESHOLD) groups.push({ lo: sortedKeys[gStart], hi: sortedKeys[i], dup });
+                gStart = i + 1; gBytes = 0; gSlots = 0; gUnique = 0;
+            }
+        }
+        groups.sort((a, b) => b.dup - a.dup);
+        return groups;
+    }
+
     private formatInfo(reader: ResolvedReader): string {
         return `(collection has ${blue(formatNumber(reader.rowCount))} rows, ${blue(formatNumber(reader.totalBytes))}B)`;
     }

package/storage/BulkDatabase2/mergeLock.ts

@@ -21,15 +21,17 @@ function lockKey(collection: string): string {
     return "bulkDatabase2-merge:" + collection;
 }
 
-// Returns true if we now hold the lock (or there's no localStorage to coordinate through, in which
-// case we proceed and rely on the manifest backstop). Returns false if another tab holds a fresh lock.
+// Returns true if we now hold the lock (or there's no localStorage to coordinate through, in which case
+// we proceed — concurrent merges are harmless). Returns false only if a DIFFERENT holder has a fresh
+// lock. Re-stamping our own lock always succeeds, so this doubles as a heartbeat: call it periodically
+// during a long, spaced-out merge to keep the lock alive (and detect if another tab took it over).
 export function tryAcquireMergeLock(collection: string, holderId: string): boolean {
     const ls = getLocalStorage();
     if (!ls) return true;
     const key = lockKey(collection);
     const now = Date.now();
     const existing = ls.getItem(key);
-    if (existing) {
+    if (existing && !existing.startsWith(holderId + ":")) {
         const t = parseInt(existing.slice(existing.lastIndexOf(":") + 1), 10);
         if (Number.isFinite(t) && now - t < LOCK_TTL_MS) return false;
     }

package/storage/BulkDatabase2/streamLog.ts

@@ -24,13 +24,10 @@ function frame(payload: Buffer): Buffer {
     return Buffer.concat([len, payload, len]);
 }
 
-// We deliberately do NOT batch/debounce writes here — each write is framed and appended (and flushed)
-// immediately. If a caller makes many individual writes and that causes lag, the fix is for them to
-// call writeBatch/deleteBatch with the whole set, not for us to silently coalesce. Caller-side
-// batching is strictly faster than anything we could do (it knows the full set up front), and not
-// batching gives the lowest possible latency for callers who genuinely want single writes — which
-// matters because the tab can be closed at any moment, so we want each write on disk as soon as
-// possible rather than sitting in a pending buffer that a close would lose.
+// Framing only — no batching here. BulkDatabase2 coalesces and flushes these framed bytes on a ramping
+// per-collection schedule (see streamAppend in BulkDatabaseBase), because the browser File System Access
+// API rewrites the whole file on every append, so one-append-per-write is quadratic. The first write
+// after a lull still flushes immediately, so a single action then a tab close is saved at once.
 
 // Times are assigned by the caller (BulkDatabase2) so the exact same timestamp lands on disk, in the
 // in-memory overlay, and in the cross-tab broadcast — keeping the global write order consistent.