sliftutils 1.3.0 → 1.4.0

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -1,42 +1,71 @@
 import { sort } from "socket-function/src/misc";
 import { getTimeUnique } from "socket-function/src/bits";
-import { BaseBulkDatabaseReader, buildFileBuffer, EMPTY_BUFFER, loadBulkDatabase } from "./BulkDatabaseFormat";
+import { ABSENT, BaseBulkDatabaseReader, buildFileBuffer, BulkHeaderInfo, EMPTY_BUFFER, KEY_COLUMN, loadBulkDatabase, loadBulkHeader, TARGET_FILE_BYTES } from "./BulkDatabaseFormat";
 import { lazy } from "socket-function/src/caching";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, red } from "socket-function/src/formatting/logColors";
 import { blockCache, encodeCompressedBlocks, GetRange } from "./blockCache";
 import { STREAM_EXTENSION, StreamEntry, frameRows, frameDeletes, parseStream, streamReaderFromEntries } from "./streamLog";
-import { connect as syncConnect, broadcast as syncBroadcast, isSyncSupported, RemoteWrite } from "./syncClient";
+import { connect as syncConnect, broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, isSyncSupported, RemoteWrite } from "./syncClient";
+import { tryAcquireMergeLock, releaseMergeLock } from "./mergeLock";
 import type { FileStorage } from "../FileFolderAPI";
 
+// ───────────────────────────────────────────────────────────────────────────────────────────────
+// KNOWN BUGS (accepted, documented):
+//
+//  • Inconsistent directory listing under concurrent merges. A read lists the directory, then loads
+//    each listed file. If a file is missing when we go to read it (a merge deleted it), we re-list and
+//    retry — the deleting merge wrote the replacement first, so the data is never gone, just moved.
+//    But a directory listing is not guaranteed atomic on every filesystem: a listing taken while
+//    another writer is mid-swap can in principle return a set of files that never simultaneously
+//    existed (e.g. the replacement but not a sibling it depends on). That yields a momentarily
+//    INCONSISTENT view — some keys may read stale or missing. It does NOT lose data on disk: a reload
+//    (refresh the page) re-lists and resolves correctly. We accept this rather than reintroduce a
+//    manifest; it's rare and OS/filesystem-dependent.
+// ───────────────────────────────────────────────────────────────────────────────────────────────
+
 // BulkDatabase2's compressed-block format is not compatible with BulkDatabase, so it uses its own
 // folder rather than sharing bulkDatabases/.
 const BULK_ROOT_FOLDER = "bulkDatabases2";
 const FILE_EXTENSION = ".bulk";
-// Once this many bulk files pile up we run a merge pass to consolidate small ones (bounded by the
-// byte caps below), so reads don't fan out across an unbounded number of files.
-const MERGE_FILE_COUNT = 8;
-
-// Memory ceiling for a single merge. Files are merged in contiguous (newest-first) runs whose combined
-// LOGICAL (uncompressed) size stays under MERGE_MAX_BYTES, so a merge never reads more than this into
-// memory at once. A file at or above MERGE_MIN_BYTES is "sealed": big enough already, never read back
-// in to be merged again. Sizes are measured uncompressed (from each file's index) because that — not
-// the smaller on-disk compressed size — is what actually lands in memory during a merge.
-const MERGE_MIN_BYTES = 400 * 1024 * 1024;
-const MERGE_MAX_BYTES = 800 * 1024 * 1024;
-
-// Tier-0 streaming rolls over into a columnar bulk file once it gets big enough — by row count,
-// byte size, or file count (many threads each stream to their own file). A single writeBatch that
-// already exceeds the row/byte limits skips streaming and writes a bulk file directly.
+// A single writeBatch that already exceeds these limits skips the tier-0 stream and folds straight
+// into a bulk file (streaming thousands of rows one frame at a time would be pointless).
 const ROLLOVER_ROWS = 5000;
 const ROLLOVER_BYTES = 5 * 1024 * 1024;
-const ROLLOVER_FILES = 100;
 
-// An unreadable file might be a write that is still in progress (another thread), so we can't delete
-// it on sight. Once it has been unreadable for longer than this (by its filename timestamp), no
-// writer is plausibly still working on it, so we delete it. Until then we just warn.
+// An unreadable (corrupt/torn, not merely missing) file might be a write still in progress, so we
+// can't delete it on sight. Once it's been unreadable for longer than this (by its name timestamp),
+// no writer is plausibly still working on it, so we delete it. Until then we just warn.
 const STALE_DELETE_MS = 24 * 60 * 60 * 1000;
 
+// A read lists the directory then loads each file; if a file vanished (a merge deleted it) we re-list
+// and retry, since the merge wrote the replacement first. Bounded so a pathological merge storm can't
+// loop forever — after this many tries we load whatever's currently there (the documented inconsistent
+// -view bug), which a later reload resolves.
+const MAX_READ_ATTEMPTS = 8;
+
+// The first ("consolidate recent") merge accumulates the newest files up to this many bytes into one
+// file (half the target, so it has room to grow before it needs splitting). The key-stratified second
+// merge groups keys into runs of this many bytes and only rewrites a group whose fraction of duplicate
+// (multi-file) keys exceeds DUP_THRESHOLD — i.e. only when deduping actually buys enough.
+const FIRST_MERGE_BYTES = TARGET_FILE_BYTES / 2;
+const KEY_GROUP_BYTES = 800 * 1024 * 1024;
+const DUP_THRESHOLD = 0.4;
+
+// Time thresholds, mutable so tests can shrink them from hours to milliseconds.
+export const bulkDatabase2Timing = {
+    // A writer stops appending to its current stream file once it's this old (starts a fresh one). A
+    // stream file older than this is therefore safe for a merge to delete: its writer has provably moved
+    // on to a new file and will never append to it again.
+    streamSealAgeMs: 10 * 60 * 60 * 1000,
+    // Per-instance throttle: a write triggers at most one background testMerge scan per this interval.
+    mergeCheckIntervalMs: 30 * 60 * 1000,
+    // The first merge fires when the recent (up to FIRST_MERGE_BYTES) files number more than this...
+    firstMergeTriggerFiles: 20,
+    // ...or span a wider write-time range than this (data trickling in slowly still gets consolidated).
+    firstMergeTriggerRangeMs: 3 * 24 * 60 * 60 * 1000,
+};
+
 // Marks a key as deleted in the in-memory overlay.
 const DELETED = Symbol("deleted");
 // Each overlay entry carries the write's unique timestamp so cross-tab writes can be ordered: a
@@ -86,6 +115,12 @@ const LOAD_SIGNAL = NULL + "load";
 const OVERLAY_SIGNAL = NULL + "overlay";
 
 let fileNameCounter = 0;
+// Random per-process id baked into file names so two processes (tabs) writing the same collection
+// never collide on a name when they pick the same timestamp/counter in the same millisecond.
+const writerId = Math.random().toString(36).slice(2, 10);
+function nextCounter(): number {
+    return ++fileNameCounter;
+}
 
 type BulkFileInfo = { fileName: string; level: number; timestamp: number };
 
@@ -100,10 +135,9 @@ function nextFileTime(): number {
 
 // Files are ordered purely by timestamp (newest-first). A merged file is given the newest timestamp
 // of the run it replaced, so it occupies exactly that run's slot. The leading "0" is a vestigial
-// field kept so the name stays in the historical level_timestamp_counter shape parseFileName expects.
+// field kept so the name stays in the historical level_timestamp_..._counter shape parseFileName reads.
 function newFileName(timestamp: number): string {
-    fileNameCounter++;
-    return `0_${timestamp}_${fileNameCounter}${FILE_EXTENSION}`;
+    return `0_${timestamp}_${writerId}_${nextCounter()}${FILE_EXTENSION}`;
 }
 
 type StreamFileInfo = { fileName: string; timestamp: number };
@@ -121,13 +155,21 @@ function parseStreamFileName(fileName: string): StreamFileInfo | undefined {
 function parseFileName(fileName: string): BulkFileInfo | undefined {
     if (!fileName.endsWith(FILE_EXTENSION)) return undefined;
     const parts = fileName.slice(0, -FILE_EXTENSION.length).split("_");
-    if (parts.length !== 3) return undefined;
+    // Accept both the old 3-part (level_timestamp_counter) and new 4-part
+    // (level_timestamp_writerId_counter) shapes; level + timestamp are always the first two fields.
+    if (parts.length < 3) return undefined;
     const level = parseInt(parts[0], 10);
     const timestamp = parseInt(parts[1], 10);
     if (!Number.isFinite(level) || !Number.isFinite(timestamp)) return undefined;
     return { fileName, level, timestamp };
 }
 
+// A file we listed is gone now (a concurrent merge deleted it after writing its replacement). Distinct
+// from a corrupt/torn file: missing => the data moved, so re-list and retry; corrupt => skip the file.
+class MissingFileError extends Error { }
+// Thrown out of a read build when a listed file went missing mid-load, so the build re-lists and retries.
+class FilesChangedError extends Error { }
+
 // All of BulkDatabase2's behavior, with no dependency on mobx or on a particular storage backend.
 // Reactivity is delegated to the injected ReactiveDeps and storage to the injected StorageFactory.
 export class BulkDatabaseBase<T extends { key: string }> {
@@ -160,42 +202,82 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // This instance's tier-0 stream file. Each instance (≈ each thread/tab) streams to its own file
     // so concurrent writers never touch the same file.
     private streamFileName: string | undefined;
-    private streamRowsWritten = 0;
+    // Seeded to construction time (not 0) so a fresh instance doesn't immediately seal+merge on its very
+    // first write — the first background merge check waits a full interval after construction.
+    private lastMergeCheck = Date.now();
     private getStreamFileName(): string {
+        // Seal (stop appending to) our current file once it's old enough, so no file is ever appended
+        // to past the seal age — that's what lets a consolidation safely fold it once it's aged out.
+        if (this.streamFileName) {
+            const info = parseStreamFileName(this.streamFileName);
+            if (info && Date.now() - info.timestamp >= bulkDatabase2Timing.streamSealAgeMs) this.streamFileName = undefined;
+        }
         if (!this.streamFileName) {
             this.streamFileName = `stream_${Date.now()}_${Math.random().toString(36).slice(2, 10)}${STREAM_EXTENSION}`;
         }
         return this.streamFileName;
     }
 
-    // Sets an overlay entry and invalidates both that key's signal and the overlay-wide signal.
-    private setOverlay(key: string, entry: OverlayEntry) {
-        this.overlay.set(key, entry);
+    private invalidateOverlay(key: string) {
         this.deps.invalidate(key);
         this.deps.invalidate(OVERLAY_SIGNAL);
     }
 
-    private reader = lazy(async (): Promise<BaseBulkDatabaseReader> => {
+    // Merges a (possibly partial) row onto the key's current overlay value, so a partial write/update
+    // only changes the columns it includes — the rest fall through to disk on read. A prior delete is
+    // reset (the key is being re-created).
+    private setOverlayRow(key: string, row: Record<string, unknown>, time: number) {
+        const existing = this.overlay.get(key);
+        const value = existing && existing.value !== DELETED ? { ...existing.value, ...row } : { ...row };
+        this.overlay.set(key, { time, value });
+        this.invalidateOverlay(key);
+    }
+
+    private setOverlayDeleted(key: string, time: number) {
+        this.overlay.set(key, { time, value: DELETED });
+        this.invalidateOverlay(key);
+    }
+
+    private reader = lazy(async (): Promise<ResolvedReader> => {
+        // A merge can delete a file between our directory listing and our read of it. The merge wrote the
+        // replacement first, so the data isn't gone — it just moved to a file our stale listing didn't
+        // include. So on a missing file we re-list and rebuild. Bounded; the last attempt tolerates a
+        // missing file (loads whatever is there — the documented inconsistent-view bug, fixed by reload).
         let start = Date.now();
-        const [bulkFiles, streamFiles] = await Promise.all([this.listFiles(), this.listStreamFiles()]);
-        // Load everything in parallel: each bulk file's columnar reader, plus all streamed entries.
-        // A corrupt/truncated bulk file is skipped with a warning rather than breaking the load or
-        // returning bad values: the write protocol always writes a new file before removing the old
-        // ones it supersedes, so a partially-written file's data still exists in another file.
+        for (let attempt = 0; ; attempt++) {
+            try {
+                return await this.buildReader(start, attempt >= MAX_READ_ATTEMPTS);
+            } catch (e) {
+                if (e instanceof FilesChangedError && attempt < MAX_READ_ATTEMPTS) continue;
+                throw e;
+            }
+        }
+    });
+
+    // One read build over a directory listing. Loads every bulk file's columnar reader plus all streamed
+    // entries, then joins them by write-time. A corrupt/torn bulk file is skipped with a warning (its
+    // data lives in another file). A *missing* file (deleted by a concurrent merge) throws
+    // FilesChangedError so the caller re-lists — unless tolerateMissing, when we proceed without it.
+    private async buildReader(start: number, tolerateMissing: boolean): Promise<ResolvedReader> {
+        const { bulkFiles, streamFiles } = await this.listFiles();
+        let filesChanged = false;
         const [bulkReadersRaw, streamData] = await Promise.all([
             Promise.all(bulkFiles.map(async f => {
                 try {
                     return await this.loadFileReader(f.fileName);
                 } catch (e) {
+                    if (e instanceof MissingFileError) { filesChanged = true; return undefined; }
                     await this.handleUnreadableFile(f, (e as Error).message);
                     return undefined;
                 }
             })),
             this.loadStreamEntries(streamFiles),
         ]);
+        if (streamData.missing) filesChanged = true;
+        if (filesChanged && !tolerateMissing) throw new FilesChangedError();
+
         const bulkReaders = bulkReadersRaw.filter((r): r is BaseBulkDatabaseReader => !!r);
-        // Streamed entries are the newest writes, so their reader goes first (the join is newest-wins
-        // and lets the stream's deletes tombstone keys in the older bulk readers).
+        // The join resolves purely by write-time, so reader order doesn't matter.
         const readers: BaseBulkDatabaseReader[] = [];
         const ordered = this.orderStreamEntries(streamData.entries);
         if (ordered.length) {
@@ -206,15 +288,14 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.streamTimes = new Map();
         }
         readers.push(...bulkReaders);
-        const joined = joinBulkDatabases(readers);
+        const joined = await joinBulkDatabases(readers);
 
         let time = Date.now() - start;
         if (time > 50) {
-            let totalKeysSize = readers.map(x => x.columns.find(c => c.column === "key")?.byteSize || 0).reduce((a, b) => a + b, 0);
-            console.log(`${blue(`${this.name} loaded`)} in ${red(formatTime(time))} (${blue(formatNumber(joined.rowCount))} rows, ${bulkFiles.length} bulk + ${streamFiles.length} stream files, ${blue(formatNumber(totalKeysSize))} keys size)`);
+            console.log(`${blue(`${this.name} loaded`)} in ${red(formatTime(time))} (${blue(formatNumber(joined.rowCount))} rows, ${bulkFiles.length} bulk + ${streamFiles.length} stream files)`);
         }
         return joined;
-    });
+    }
 
     // Connects to the cross-tab BroadcastChannel (browser only) so writes in other tabs of this
     // collection update our overlay. Runs once; no-op in Node / where BroadcastChannel is unavailable.
@@ -224,7 +305,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
     private syncSetup = lazy(async () => {
         if (!isSyncSupported()) return;
         await this.reader();
-        let recent = await syncConnect(this.name, write => this.applyRemote(write));
+        // onSeal: a peer is about to fold recent data; drop our current stream file so we stop appending
+        // to it (our next write starts a fresh one), letting the merge fold it whole.
+        let recent = await syncConnect(this.name, write => this.applyRemote(write), () => { this.streamFileName = undefined; });
         for (let write of recent) this.applyRemote(write);
     });
 
@@ -242,8 +325,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
     private applyRemote(write: RemoteWrite) {
         if (write.time <= this.localTime(write.key)) return;
         this.deps.batch(() => {
-            if (write.deleted) this.setOverlay(write.key, { time: write.time, value: DELETED });
-            else this.setOverlay(write.key, { time: write.time, value: write.value as Record<string, unknown> });
+            if (write.deleted) this.setOverlayDeleted(write.key, write.time);
+            else this.setOverlayRow(write.key, write.value as Record<string, unknown>, write.time);
         });
     }
 
@@ -277,10 +360,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const stamped = rows.map(row => ({ time: getTimeUnique(), row }));
         const framed = frameRows(stamped);
 
-        // A batch that already exceeds the rollover limits skips tier-0 and writes a bulk file directly.
+        // A batch that already exceeds the limits skips the tier-0 stream and writes a bulk file directly
+        // (streaming thousands of rows one frame at a time would be pointless).
         if (entries.length >= ROLLOVER_ROWS || framed.length >= ROLLOVER_BYTES) {
             await this.writeBulkFile(rows);
-            this.resetReader();
             return;
         }
 
@@ -288,12 +371,11 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // overlay immediately — no reader reset.
         const storage = await this.storage();
         await storage.append(this.getStreamFileName(), framed);
-        this.streamRowsWritten += entries.length;
         this.deps.batch(() => {
-            for (const { time, row } of stamped) this.setOverlay(row.key as string, { time, value: row });
+            for (const { time, row } of stamped) this.setOverlayRow(row.key as string, row, time);
         });
         for (const { time, row } of stamped) syncBroadcast(this.name, { key: row.key as string, time, value: row });
-        await this.maybeRolloverStream();
+        void this.maybeMerge();
     }
 
     public async delete(key: string): Promise<void> {
@@ -306,46 +388,101 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const stamped = keys.map(key => ({ time: getTimeUnique(), key }));
         const storage = await this.storage();
         await storage.append(this.getStreamFileName(), frameDeletes(stamped));
-        this.streamRowsWritten += keys.length;
         this.deps.batch(() => {
-            for (const { time, key } of stamped) this.setOverlay(key, { time, value: DELETED });
+            for (const { time, key } of stamped) this.setOverlayDeleted(key, time);
         });
         for (const { time, key } of stamped) syncBroadcast(this.name, { key, time, deleted: true });
-        await this.maybeRolloverStream();
+        void this.maybeMerge();
     }
 
-    // Writes the rows as one or more columnar bulk files (buildFileBuffer splits a too-large batch by
-    // row range so no single file approaches the Buffer size limit), all sharing one timestamp since
-    // they're one write with disjoint keys. Then, if enough files have accumulated, runs bounded merge
-    // passes until nothing more can be consolidated.
-    private async writeBulkFile(rows: Record<string, unknown>[]): Promise<void> {
-        const storage = await this.storage();
-        const timestamp = nextFileTime();
-        for (const buffer of buildFileBuffer(rows)) {
-            await storage.set(newFileName(timestamp), encodeCompressedBlocks(buffer));
-        }
-        if ((await this.listFiles()).length >= MERGE_FILE_COUNT) {
-            while (await this.mergeFiles() > 0) { /* keep merging until no run can be consolidated */ }
+    public async update(entry: Partial<T> & { key: string }): Promise<void> {
+        return this.updateBatch([entry]);
+    }
+
+    // Like writeBatch, but each entry is a partial row — only the fields to change, plus the required
+    // key. Partial fields merge onto the existing row (unset columns fall through to the current value);
+    // an entry whose key isn't in the collection is skipped with a warning, since update never creates keys.
+    public async updateBatch(entries: (Partial<T> & { key: string })[]): Promise<void> {
+        if (!entries.length) return;
+        void this.syncSetup();
+        const reader = await this.reader();
+        const diskKeys = new Set(reader.keys);
+        const present: T[] = [];
+        for (const entry of entries) {
+            const overlayEntry = this.overlay.get(entry.key);
+            const exists = overlayEntry ? overlayEntry.value !== DELETED : diskKeys.has(entry.key);
+            if (!exists) {
+                console.warn(`${this.name}.update: key ${JSON.stringify(entry.key)} is not in the collection, ignoring`);
+                continue;
+            }
+            present.push(entry as unknown as T);
         }
+        if (present.length) await this.writeBatch(present);
     }
 
-    private async listStreamFiles(): Promise<StreamFileInfo[]> {
+    // Lists every bulk + stream file currently on disk (no manifest — every file is part of the
+    // database). Bulk newest-first, streams oldest-first. Duplicate data (from a crashed/raced merge)
+    // is harmless: reads resolve by write-time and a later merge with enough duplication removes it.
+    private async listFiles(): Promise<{ bulkFiles: BulkFileInfo[]; streamFiles: StreamFileInfo[] }> {
         const storage = await this.storage();
         const names = await storage.getKeys();
-        const files = names.flatMap(n => {
-            const parsed = parseStreamFileName(n);
-            return parsed && [parsed] || [];
+        const bulkFiles: BulkFileInfo[] = [];
+        const streamFiles: StreamFileInfo[] = [];
+        for (const n of names) {
+            if (n.endsWith(FILE_EXTENSION)) { const p = parseFileName(n); if (p) bulkFiles.push(p); }
+            else if (n.endsWith(STREAM_EXTENSION)) { const p = parseStreamFileName(n); if (p) streamFiles.push(p); }
+        }
+        // Newest-first by timestamp; ties broken by file name for determinism.
+        bulkFiles.sort((a, b) => {
+            if (a.timestamp !== b.timestamp) return b.timestamp - a.timestamp;
+            return a.fileName < b.fileName && 1 || a.fileName > b.fileName && -1 || 0;
         });
-        sort(files, f => f.timestamp);
-        return files;
+        sort(streamFiles, f => f.timestamp);
+        return { bulkFiles, streamFiles };
+    }
+
+    // Writes `rows` directly as bulk file(s), stamped with the current time as their write-time (the rows
+    // are being written now). Used by the large-batch write path: no manifest, just new files on disk.
+    // The rows carry time=now, so the join orders them correctly against any older stream entry for the
+    // same key (newer time wins) — no clobber. A later testMerge consolidates them.
+    private async writeBulkFile(rows: Record<string, unknown>[]): Promise<void> {
+        const storage = await this.storage();
+        const timestamp = nextFileTime();
+        const now = Date.now();
+        const times = rows.map(() => now);
+        for (const buffer of buildFileBuffer(rows, times)) {
+            const name = newFileName(timestamp);
+            await storage.set(name, encodeCompressedBlocks(buffer));
+        }
+        this.resetReader();
+        void this.maybeMerge();
     }
 
     // Reads and parses every stream file in parallel. Returns per-write entries (each carrying its
-    // unique timestamp + originating file) so callers can order writes globally across files.
-    private async loadStreamEntries(streamFiles: StreamFileInfo[]): Promise<{ entries: { time: number; fileName: string; entry: StreamEntry }[]; totalBytes: number }> {
-        if (!streamFiles.length) return { entries: [], totalBytes: 0 };
+    // unique timestamp + originating file) so callers can order writes globally across files, the
+    // prefix size we read per file (so a merge can verify nothing was appended before deleting it), and
+    // whether any listed file was missing (so a read can re-list and retry — a merge deleted it).
+    private async loadStreamEntries(streamFiles: StreamFileInfo[]): Promise<{ entries: { time: number; fileName: string; entry: StreamEntry }[]; totalBytes: number; missing: boolean; sizes: Map<string, number> }> {
+        const sizes = new Map<string, number>();
+        if (!streamFiles.length) return { entries: [], totalBytes: 0, missing: false, sizes };
         const storage = await this.storage();
-        const buffers = await Promise.all(streamFiles.map(f => storage.get(f.fileName)));
+        let missing = false;
+        // Read a bounded prefix [0, size) rather than the whole file: a foreign writer may be appending
+        // concurrently, and storage.get() errors when the file grows past the size it stat'd. Reading a
+        // prefix is tolerant — parseStream stops at the last complete frame, the file stays valid, and a
+        // later read picks up the rest. A file removed out from under us (a merge) sets `missing`.
+        const buffers = await Promise.all(streamFiles.map(async f => {
+            try {
+                const info = await storage.getInfo(f.fileName);
+                if (!info) { missing = true; return undefined; }
+                sizes.set(f.fileName, info.size);
+                if (info.size === 0) return undefined;
+                return await storage.getRange(f.fileName, { start: 0, end: info.size });
+            } catch {
+                missing = true;
+                return undefined;
+            }
+        }));
         const entries: { time: number; fileName: string; entry: StreamEntry }[] = [];
         let totalBytes = 0;
         for (let i = 0; i < streamFiles.length; i++) {
@@ -360,7 +497,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 entries.push({ time: entry.time, fileName: streamFiles[i].fileName, entry });
             }
         }
-        return { entries, totalBytes };
+        return { entries, totalBytes, missing, sizes };
     }
 
     // Global mutation order across per-thread files: by unique timestamp, ties broken by file name.
@@ -372,88 +509,77 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return entries.map(e => e.entry);
     }
 
-    private async maybeRolloverStream(): Promise<void> {
-        const streamFiles = await this.listStreamFiles();
-        const storage = await this.storage();
-        let totalBytes = 0;
-        for (const f of streamFiles) {
-            const info = await storage.getInfo(f.fileName);
-            totalBytes += info?.size || 0;
-        }
-        if (streamFiles.length > ROLLOVER_FILES || totalBytes > ROLLOVER_BYTES || this.streamRowsWritten > ROLLOVER_ROWS) {
-            await this.rolloverStream(streamFiles);
-        }
-    }
-
-    // Combine all tier-0 stream files into a single columnar bulk file (newest-wins per key, deletes
-    // applied), delete the consumed stream files, and re-persist surviving tombstones to a fresh
-    // stream file so deletes of keys that live in older bulk files are not lost.
-    private async rolloverStream(streamFiles: StreamFileInfo[]): Promise<void> {
-        const { entries } = await this.loadStreamEntries(streamFiles);
-        const ordered = this.orderStreamEntries(entries);
-        const byKey = new Map<string, Record<string, unknown>>();
-        const deleted = new Map<string, number>();
-        for (const e of ordered) {
-            if (e.deletedKey !== undefined) {
-                byKey.delete(e.deletedKey);
-                deleted.set(e.deletedKey, e.time);
-            } else if (e.row) {
-                let key = e.row.key as string;
-                byKey.set(key, e.row);
-                deleted.delete(key);
-            }
+    // Throttled, fire-and-forget after writes: run a background merge check at most once per interval.
+    private async maybeMerge(): Promise<void> {
+        const now = Date.now();
+        if (now - this.lastMergeCheck < bulkDatabase2Timing.mergeCheckIntervalMs) return;
+        this.lastMergeCheck = now;
+        try {
+            await this.tryMergeNow();
+        } catch (e) {
+            console.warn(`${this.name}: background merge failed: ${(e as Error).message}`);
+        }
+    }
+
+    // Runs one merge pass now (the same one maybeMerge runs on a timer). Returns whether it merged
+    // anything, and whether it bailed because another tab/process holds the merge lock — so a caller
+    // (e.g. a 30-minute scheduler) can tell "nothing to do" from "someone else is doing it".
+    public async tryMergeNow(): Promise<{ merged: boolean; lockFailed: boolean }> {
+        if (!tryAcquireMergeLock(this.name, writerId)) return { merged: false, lockFailed: true };
+        try {
+            return { merged: await this.testMerge(), lockFailed: false };
+        } finally {
+            releaseMergeLock(this.name, writerId);
         }
-        if (byKey.size) await this.writeBulkFile([...byKey.values()]);
-        const storage = await this.storage();
-        // Persist surviving tombstones (keeping their original timestamps) to a FRESH stream file
-        // before removing the consumed files, so a crash in between can't drop the deletes. The window
-        // is at worst redundant (deletes present in both old and new files), never missing.
-        this.streamFileName = undefined;
-        this.streamRowsWritten = 0;
-        if (deleted.size) {
-            await storage.append(this.getStreamFileName(), frameDeletes([...deleted].map(([key, time]) => ({ time, key }))));
-        }
-        for (const f of streamFiles) await storage.remove(f.fileName);
-        this.resetReader();
     }
 
-    // Consolidate as much as the caps allow: repeatedly merge contiguous non-sealed runs until nothing
-    // more can be combined. Unlike a naive "merge everything into one file", this respects MERGE_MAX_BYTES
-    // so it never loads the whole collection into memory — a multi-GB collection settles into several
-    // capped files (sealed files are left as-is) rather than one giant one.
+    // Full compaction: fold + dedup everything into key-sorted, ~256MB files. Reads the whole collection
+    // into memory (the accepted soft bound), so it's an explicit, occasional call. Deletes consumed bulk
+    // files and any stream file it's safe to (aged, or sealed-and-stable).
     public async compact(): Promise<void> {
-        let merged = false;
-        while (await this.mergeFiles() > 0) merged = true;
-        if (merged) this.resetReader();
+        if (!tryAcquireMergeLock(this.name, writerId)) return; // someone else is already merging; fine
+        try {
+            syncBroadcastSeal(this.name);
+            this.streamFileName = undefined;
+            const { bulkFiles, streamFiles } = await this.listFiles();
+            if (bulkFiles.length + streamFiles.length >= 1) await this.mergeFileSet(bulkFiles, streamFiles);
+        } finally {
+            releaseMergeLock(this.name, writerId);
+        }
     }
 
-    private async listFiles(): Promise<BulkFileInfo[]> {
-        const storage = await this.storage();
-        const names = await storage.getKeys();
-        const files = names.flatMap(n => {
-            const parsed = parseFileName(n);
-            return parsed && [parsed] || [];
+    // The unified merge entry point: rewrite everything overlapping [timeLo, timeHi] into fresh
+    // key-sorted ~256MB bulk file(s). Selects bulk files by their header time range and stream files by
+    // their (creation .. seal-age) window. If the range reaches the present, first asks peers to seal so
+    // recent stream data is complete. Callers: testMerge (recent / key-group ranges); external callers
+    // can pass any range — older data just produces older files.
+    public async merge(timeLo: number, timeHi: number): Promise<void> {
+        if (timeHi >= Date.now()) { syncBroadcastSeal(this.name); this.streamFileName = undefined; }
+        const { bulkFiles, streamFiles } = await this.listFiles();
+        const headers = await Promise.all(bulkFiles.map(f => this.readBulkHeader(f.fileName)));
+        const selBulk = bulkFiles.filter((f, i) => {
+            const h = headers[i];
+            if (!h) return false;
+            // Old files (no recorded time range) only belong to a merge that reaches back to the start.
+            if (!h.maxTime && !h.minTime) return timeLo <= 0;
+            return h.minTime <= timeHi && h.maxTime >= timeLo;
         });
-        // Newest-first by timestamp; ties broken by file name (descending) for a deterministic order.
-        // A merged file inherits the newest timestamp of the run it replaced, so it lands exactly where
-        // that run was — keeping newest-wins correct without any level bookkeeping.
-        files.sort((a, b) => {
-            if (a.timestamp !== b.timestamp) return b.timestamp - a.timestamp;
-            return a.fileName < b.fileName && 1 || a.fileName > b.fileName && -1 || 0;
-        });
-        return files;
+        const selStream = streamFiles.filter(f =>
+            f.timestamp <= timeHi && f.timestamp + bulkDatabase2Timing.streamSealAgeMs >= timeLo);
+        if (selBulk.length + selStream.length < 2) return;
+        await this.mergeFileSet(selBulk, selStream);
     }
 
-    private async makeRawGetRange(fileName: string): Promise<{ rawGetRange: GetRange; size: number } | undefined> {
+    // Throws MissingFileError (not a generic error) when the file is gone, so callers can distinguish a
+    // file a merge deleted (re-list and retry / skip) from a corrupt one (handle as unreadable).
+    private async makeRawGetRange(fileName: string): Promise<{ rawGetRange: GetRange; size: number }> {
         const storage = await this.storage();
         const info = await storage.getInfo(fileName);
-        if (!info) return undefined;
+        if (!info) throw new MissingFileError(`bulk file ${fileName} is missing`);
         const rawGetRange: GetRange = async (start, end) => {
             if (end <= start) return EMPTY_BUFFER;
             const buf = await storage.getRange(fileName, { start, end });
-            if (!buf) {
-                throw new Error(`Expected range [${start}, ${end}) of ${fileName}, file was missing`);
-            }
+            if (!buf) throw new MissingFileError(`range [${start}, ${end}) of ${fileName} is missing`);
             return buf;
         };
         return { rawGetRange, size: info.size };
@@ -461,9 +587,6 @@ export class BulkDatabaseBase<T extends { key: string }> {
 
     private async loadFileReader(fileName: string): Promise<BaseBulkDatabaseReader> {
         const raw = await this.makeRawGetRange(fileName);
-        if (!raw) {
-            throw new Error(`Expected bulk file to exist, was missing: ${fileName}`);
-        }
         const fileId = nullJoin(this.name, fileName);
         // Files are immutable and stored as compressed blocks; replace getRange with a block-cached,
         // decompressing version (same interface) and read the logical (uncompressed) size from its
@@ -472,13 +595,25 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return loadBulkDatabase({ totalBytes: opened.uncompressedSize, getRange: opened.getRange });
     }
 
+    // Reads only a bulk file's header (row count, time range, key range) — no column data — for merge
+    // planning. Returns undefined for a missing/corrupt file so the planner just leaves it out.
+    private async readBulkHeader(fileName: string): Promise<BulkHeaderInfo | undefined> {
+        try {
+            const raw = await this.makeRawGetRange(fileName);
+            const fileId = nullJoin(this.name, fileName);
+            const opened = await blockCache.open(fileId, raw.size, raw.rawGetRange);
+            return await loadBulkHeader(opened.getRange, opened.uncompressedSize);
+        } catch {
+            return undefined;
+        }
+    }
+
     // Logical (uncompressed) size of a bulk file, read from its (cached) index without loading data.
     // Used by the merge planner to bound how much it reads at once. Returns undefined for a file that's
     // missing or unreadable so the planner simply leaves it out of any merge.
     private async fileLogicalSize(fileName: string): Promise<number | undefined> {
         try {
             const raw = await this.makeRawGetRange(fileName);
-            if (!raw) return undefined;
             const fileId = nullJoin(this.name, fileName);
             const opened = await blockCache.open(fileId, raw.size, raw.rawGetRange);
             return opened.uncompressedSize;
@@ -507,92 +642,256 @@ export class BulkDatabaseBase<T extends { key: string }> {
         console.warn(`${this.name}: skipping unreadable bulk file ${file.fileName} (recent — may be an in-progress write): ${message}`);
     }
 
-    // Merges exactly the files it's given (already newest-first), newest-wins per key, writing the
-    // result with `timestamp` so the new file takes the slot of the run it replaced, then deletes the
-    // consumed files. The caller is responsible for keeping the input under MERGE_MAX_BYTES — this
-    // function blindly reads it all into memory.
-    private async mergeFilesBase(files: BulkFileInfo[], timestamp: number): Promise<void> {
-        const storage = await this.storage();
-        const readers = await Promise.all(files.map(f => this.loadFileReader(f.fileName)));
-
-        const seen = new Set<string>();
-        const mergedRows: Record<string, unknown>[] = [];
-        for (const reader of readers) {
-            const colData: Record<string, unknown[]> = {};
+    // Resolves a set of readers (stream + bulk) by ACTUAL write-time into merged rows + per-row times,
+    // plus the surviving tombstones (keys whose newest event is a delete). For each key/column, the
+    // value with the newest write-time across readers wins (non-ABSENT); the row's time is the newest of
+    // those. A key is deleted iff its newest delete is newer than its newest set. This is the same
+    // time-resolution reads use, captured so a merge can write the result back as bulk + a carry stream.
+    private async resolveReaders(readers: BaseBulkDatabaseReader[]): Promise<{ rows: Record<string, unknown>[]; times: number[]; deletes: Map<string, number> }> {
+        const loaded = await Promise.all(readers.map(async reader => {
+            const cols = new Map<string, Map<string, { value: unknown; time: number }>>();
             for (const col of reader.columns) {
-                colData[col.column] = (await reader.getColumn(col.column)).map(r => r.value);
+                if (col.column === KEY_COLUMN) continue;
+                const entries = await reader.getColumn(col.column);
+                cols.set(col.column, new Map(entries.map(e => [e.key, { value: e.value, time: e.time }])));
+            }
+            return { keyTimes: reader.keyTimes, deleteTimes: reader.deleteTimes, cols };
+        }));
+
+        const deleteTime = new Map<string, number>();
+        for (const l of loaded) {
+            if (!l.deleteTimes) continue;
+            for (const [k, t] of l.deleteTimes) deleteTime.set(k, Math.max(deleteTime.get(k) ?? -Infinity, t));
+        }
+        const keyTime = new Map<string, number>();
+        for (const l of loaded) {
+            for (const [k, t] of l.keyTimes) keyTime.set(k, Math.max(keyTime.get(k) ?? -Infinity, t));
+        }
+        const allCols = new Set<string>();
+        for (const l of loaded) for (const c of l.cols.keys()) allCols.add(c);
+
+        const rows: Record<string, unknown>[] = [];
+        const times: number[] = [];
+        const deletes = new Map<string, number>();
+        const allKeys = new Set<string>([...keyTime.keys(), ...deleteTime.keys()]);
+        for (const key of allKeys) {
+            const setT = keyTime.get(key) ?? -Infinity;
+            const delT = deleteTime.get(key) ?? -Infinity;
+            if (setT <= delT) {
+                // The newest event for this key is a delete — carry the tombstone forward so it keeps
+                // suppressing any older set living in a file outside this merge.
+                if (delT > -Infinity) deletes.set(key, delT);
+                continue;
             }
-            for (let i = 0; i < reader.keys.length; i++) {
-                const key = reader.keys[i];
-                if (seen.has(key)) continue;
-                seen.add(key);
-                const row: Record<string, unknown> = {};
-                for (const col of reader.columns) {
-                    row[col.column] = colData[col.column][i];
+            const row: Record<string, unknown> = { [KEY_COLUMN]: key };
+            let rowTime = setT;
+            for (const col of allCols) {
+                let bestTime = -Infinity;
+                let bestVal: unknown;
+                let found = false;
+                for (const l of loaded) {
+                    const cell = l.cols.get(col)?.get(key);
+                    if (!cell || cell.value === ABSENT) continue;
+                    if (cell.time > bestTime) { bestTime = cell.time; bestVal = cell.value; found = true; }
                 }
-                mergedRows.push(row);
+                if (found) { row[col] = bestVal; if (bestTime > rowTime) rowTime = bestTime; }
             }
+            rows.push(row);
+            times.push(rowTime === -Infinity ? 0 : rowTime);
         }
+        return { rows, times, deletes };
+    }
+
+    // The one merge primitive. Reads the given bulk + stream files (skipping any that vanished or won't
+    // parse — their data lives elsewhere), resolves them by write-time, writes the result back as fresh
+    // key-sorted ~256MB bulk file(s) plus a carry stream for surviving tombstones, THEN deletes the
+    // inputs it consumed. Output is always written before any delete, so a crash leaves duplicates (next
+    // merge removes them), never a gap. A bulk file is deleted only if we actually read it; a stream file
+    // only if it's aged out (its writer has switched files) or — when cross-tab sync sealed it — its size
+    // didn't change while we read it. Returns whether it produced anything.
+    private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[]): Promise<boolean> {
+        const storage = await this.storage();
+        const timestamp = nextFileTime();
+        const now = Date.now();
+
+        const consumedBulk: BulkFileInfo[] = [];
+        const bulkReaders: BaseBulkDatabaseReader[] = [];
+        await Promise.all(bulkFiles.map(async f => {
+            try {
+                const r = await this.loadFileReader(f.fileName);
+                bulkReaders.push(r);
+                consumedBulk.push(f); // only files we actually read are safe to delete afterwards
+            } catch { /* missing or corrupt — skip; its data lives in another file */ }
+        }));
 
-        // The input is under the cap, so buildFileBuffer almost always returns a single buffer; the loop
-        // is only here to stay correct if a merge's deduped output still happens to exceed the split size.
-        for (const buffer of buildFileBuffer(mergedRows)) {
-            await storage.set(newFileName(timestamp), encodeCompressedBlocks(buffer));
+        const streamData = await this.loadStreamEntries(streamFiles);
+        const ordered = this.orderStreamEntries(streamData.entries);
+        const streamReader = ordered.length ? streamReaderFromEntries(ordered, 0).reader : undefined;
+
+        const readers = streamReader ? [streamReader, ...bulkReaders] : bulkReaders;
+        if (!readers.length) return false;
+
+        const { rows, times, deletes } = await this.resolveReaders(readers);
+
+        // Write all outputs BEFORE deleting any input, so a throw mid-write just leaves duplicates.
+        const newNames: string[] = [];
+        if (rows.length) {
+            for (const buffer of buildFileBuffer(rows, times)) {
+                const name = newFileName(timestamp);
+                await storage.set(name, encodeCompressedBlocks(buffer));
+                newNames.push(name);
+            }
         }
-        for (const f of files) {
-            await storage.remove(f.fileName);
+        if (deletes.size) {
+            const carryName = `stream_${Date.now()}_${Math.random().toString(36).slice(2, 10)}${STREAM_EXTENSION}`;
+            await storage.set(carryName, frameDeletes([...deletes].map(([key, time]) => ({ time, key }))));
         }
+
+        const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };
+        for (const f of consumedBulk) await remove(f.fileName);
+        for (const f of streamFiles) {
+            if (await this.canDeleteStream(f, now, streamData.sizes)) await remove(f.fileName);
+        }
+
+        this.resetReader();
+        return newNames.length > 0 || deletes.size > 0;
+    }
+
+    // A stream file is safe to delete iff no writer will ever append to it again: it's aged past the seal
+    // age (its writer has provably started a fresh file), OR cross-tab sync is active (so the seal we
+    // broadcast reached peers) and its size hasn't changed since we read it (nothing was appended during
+    // the merge). When neither holds we leave it: its data is now duplicated into bulk (resolved by time)
+    // and a later merge deletes it once aged. (Recreate-on-append means even a wrong delete wouldn't lose
+    // data, but the aged check also rules out the rare sparse-offset append race.)
+    private async canDeleteStream(f: StreamFileInfo, now: number, sizes: Map<string, number>): Promise<boolean> {
+        if (now - f.timestamp >= bulkDatabase2Timing.streamSealAgeMs) return true;
+        if (!isSyncSupported()) return false;
+        const readSize = sizes.get(f.fileName);
+        if (readSize === undefined) return false;
+        let info;
+        try { info = await (await this.storage()).getInfo(f.fileName); } catch { return false; }
+        return !!info && info.size === readSize;
     }
 
-    // The cap-aware merge planner. Walks the files newest-first and merges contiguous runs of
-    // non-sealed files, each run capped at MERGE_MAX_BYTES of LOGICAL size so a single merge never
-    // loads more than that into memory. A file at/over MERGE_MIN_BYTES is sealed (left untouched) and
-    // breaks the run, as does an unreadable file. Because each run is contiguous in the newest-first
-    // order and its merged file keeps the run's newest timestamp, newest-wins ordering is preserved
-    // with no inversions. Returns the number of runs merged (0 means nothing left to consolidate).
-    private async mergeFiles(): Promise<number> {
-        const files = await this.listFiles();
-        const sizes = await Promise.all(files.map(f => this.fileLogicalSize(f.fileName)));
+    // The merge policy. Up to two passes:
+    //  1) Consolidate recent fragmentation: take the newest files up to ~FIRST_MERGE_BYTES and, if they
+    //     number more than firstMergeTriggerFiles or span more than firstMergeTriggerRangeMs, merge them
+    //     into one file. Seals first so recent stream data is complete; in Node (no cross-tab seal) only
+    //     aged streams are folded, so we never re-fold the same un-deletable stream forever.
+    //  2) Key-stratify: sort all keys, walk them in ~KEY_GROUP_BYTES groups, and rewrite the single group
+    //     whose fraction of duplicate (multi-file) keys is highest above DUP_THRESHOLD — merging every
+    //     bulk file overlapping that key range. Over time this sorts the data into key-disjoint files.
+    // Returns whether either pass merged anything.
+    private async testMerge(): Promise<boolean> {
+        let merged = false;
 
-        const batches: BulkFileInfo[][] = [];
-        let batch: BulkFileInfo[] = [];
-        let batchBytes = 0;
-        const flush = () => {
-            // A run of one file is pointless to "merge" — only consolidate when there are at least two.
-            if (batch.length >= 2) batches.push(batch);
-            batch = [];
-            batchBytes = 0;
-        };
-        for (let i = 0; i < files.length; i++) {
-            const size = sizes[i];
-            if (size === undefined || size >= MERGE_MIN_BYTES) {
-                flush();
-                continue;
+        // ── Pass 1: consolidate recent files. ──
+        // Only seal (ask peers + ourselves to abandon current stream files) when cross-tab sync can
+        // actually fold recent streams; in Node it would just churn — fragmenting streams every pass for
+        // no benefit, since canDeleteStream there only deletes aged files anyway.
+        const foldRecentStreams = isSyncSupported(); // see canDeleteStream: else we'd re-fold forever
+        if (foldRecentStreams) {
+            syncBroadcastSeal(this.name);
+            this.streamFileName = undefined; // seal our own current stream so its recent data is complete
+        }
+        {
+            const { bulkFiles, streamFiles } = await this.listFiles();
+            const bulkMeta = await Promise.all(bulkFiles.map(async f => {
+                const [size, header] = await Promise.all([this.fileLogicalSize(f.fileName), this.readBulkHeader(f.fileName)]);
+                return { kind: "bulk" as const, file: f, bytes: size ?? 0, time: header?.maxTime || f.timestamp };
+            }));
+            const streamMeta: { kind: "stream"; file: StreamFileInfo; bytes: number; time: number }[] = [];
+            for (const f of streamFiles) {
+                const aged = Date.now() - f.timestamp >= bulkDatabase2Timing.streamSealAgeMs;
+                if (!foldRecentStreams && !aged) continue;
+                let bytes = 0;
+                try { const info = await (await this.storage()).getInfo(f.fileName); bytes = info?.size ?? 0; } catch { bytes = 0; }
+                streamMeta.push({ kind: "stream", file: f, bytes, time: f.timestamp });
+            }
+            const items = [...bulkMeta, ...streamMeta].sort((a, b) => b.time - a.time);
+            const recent: typeof items = [];
+            let recentBytes = 0;
+            for (const it of items) {
+                recent.push(it);
+                recentBytes += it.bytes;
+                if (recentBytes >= FIRST_MERGE_BYTES) break;
+            }
+            const span = recent.length ? recent[0].time - recent[recent.length - 1].time : 0;
+            const triggered = recent.length >= 2
+                && (recent.length > bulkDatabase2Timing.firstMergeTriggerFiles || span > bulkDatabase2Timing.firstMergeTriggerRangeMs);
+            if (triggered) {
+                const rb = recent.filter(i => i.kind === "bulk").map(i => (i.file as BulkFileInfo));
+                const rs = recent.filter(i => i.kind === "stream").map(i => (i.file as StreamFileInfo));
+                if (await this.mergeFileSet(rb, rs)) merged = true;
             }
-            if (batch.length && batchBytes + size > MERGE_MAX_BYTES) flush();
-            batch.push(files[i]);
-            batchBytes += size;
         }
-        flush();
 
-        // batch[0] is the newest file in each (newest-first) run, so its timestamp is the run's slot.
-        for (const runFiles of batches) {
-            await this.mergeFilesBase(runFiles, runFiles[0].timestamp);
+        // ── Pass 2: key-stratify the bulk files to remove duplication. ──
+        {
+            const { bulkFiles } = await this.listFiles();
+            if (bulkFiles.length >= 2) {
+                const infos = await Promise.all(bulkFiles.map(async f => {
+                    try {
+                        const reader = await this.loadFileReader(f.fileName);
+                        const keys = reader.keys;
+                        let min = keys[0], max = keys[0];
+                        for (const k of keys) { if (k < min) min = k; if (k > max) max = k; }
+                        return { file: f, keys, bytes: reader.totalBytes, min, max };
+                    } catch {
+                        return { file: f, keys: [] as string[], bytes: 0, min: undefined as string | undefined, max: undefined as string | undefined };
+                    }
+                }));
+                const usable = infos.filter(i => i.keys.length > 0);
+                const keyCount = new Map<string, number>();
+                let totalSlots = 0, totalBytes = 0;
+                for (const i of usable) {
+                    totalBytes += i.bytes;
+                    for (const k of i.keys) { keyCount.set(k, (keyCount.get(k) || 0) + 1); totalSlots++; }
+                }
+                if (totalSlots > 0) {
+                    const bytesPerSlot = totalBytes / totalSlots;
+                    const sortedKeys = [...keyCount.keys()].sort();
+                    // Walk sorted keys forming ~KEY_GROUP_BYTES groups; remember the group with the highest
+                    // duplicate fraction over the threshold (the most benefit), then merge its files.
+                    let best: { lo: string; hi: string; dup: number } | undefined;
+                    let gStart = 0, gBytes = 0, gSlots = 0, gUnique = 0;
+                    for (let i = 0; i < sortedKeys.length; i++) {
+                        const c = keyCount.get(sortedKeys[i])!;
+                        gBytes += c * bytesPerSlot; gSlots += c; gUnique += 1;
+                        if (gBytes >= KEY_GROUP_BYTES || i === sortedKeys.length - 1) {
+                            const dup = (gSlots - gUnique) / gSlots;
+                            if (dup > DUP_THRESHOLD && (!best || dup > best.dup)) best = { lo: sortedKeys[gStart], hi: sortedKeys[i], dup };
+                            gStart = i + 1; gBytes = 0; gSlots = 0; gUnique = 0;
+                        }
+                    }
+                    if (best) {
+                        const lo = best.lo, hi = best.hi;
+                        const groupFiles = usable
+                            .filter(i => i.min !== undefined && i.max !== undefined && i.min <= hi && i.max >= lo)
+                            .map(i => i.file);
+                        if (groupFiles.length >= 2 && await this.mergeFileSet(groupFiles, [])) merged = true;
+                    }
+                }
+            }
         }
-        return batches.length;
+
+        return merged;
     }
 
-    private formatInfo(reader: BaseBulkDatabaseReader): string {
+    private formatInfo(reader: ResolvedReader): string {
         return `(collection has ${blue(formatNumber(reader.rowCount))} rows, ${blue(formatNumber(reader.totalBytes))}B)`;
     }
 
-    // Applies the overlay (pending writes/deletes) on top of a base column. No-op when empty.
+    // Applies the overlay (pending writes/deletes) on top of a base column. No-op when empty. An
+    // overlay entry that doesn't include this column leaves the base (disk) value in place — a partial
+    // write/update only overrides the columns it set; everything else falls through.
     private patchColumn(base: { key: string; value: unknown }[], column: string): { key: string; value: unknown }[] {
         if (this.overlay.size === 0) return base;
         const map = new Map(base.map(e => [e.key, e.value]));
         for (const [key, entry] of this.overlay) {
-            if (entry.value === DELETED) map.delete(key);
-            else map.set(key, entry.value[column]);
+            if (entry.value === DELETED) { map.delete(key); continue; }
+            if (column in entry.value) map.set(key, entry.value[column]);
+            else if (!map.has(key)) map.set(key, undefined);
         }
         return [...map].map(([key, value]) => ({ key, value }));
     }
@@ -604,7 +903,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const entry = this.overlay.get(key);
         if (entry !== undefined) {
             if (entry.value === DELETED) return undefined;
-            return entry.value[String(column)] as T[Column];
+            if (String(column) in entry.value) return entry.value[String(column)] as T[Column];
+            // column not set in the overlay entry — fall through to disk
         }
         let time = Date.now();
         let reader = await this.reader();
@@ -689,7 +989,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
         let entry = this.overlay.get(key);
         if (entry !== undefined) {
             if (entry.value === DELETED) return undefined;
-            return entry.value[col] as T[Column];
+            if (col in entry.value) return entry.value[col] as T[Column];
+            // column not set in the overlay entry — fall through to the base field cache
         }
         let cacheKey = nullJoin(col, key);
         if (!this.baseFields.has(cacheKey)) {
@@ -730,24 +1031,38 @@ export class BulkDatabaseBase<T extends { key: string }> {
     }
 }
 
-// Lowest indexes are read first (newest-wins). A reader's deletedKeys tombstone a key in all older
-// readers; the newest reader that has a key live wins.
-function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): BaseBulkDatabaseReader {
-    const keySets = databases.map(db => new Set(db.keys));
-    const deleted = new Set<string>();
+// The merged, time-resolved view over all readers. getColumn/getSingleField return plain resolved
+// values (not {value,time}); the base layers the overlay on top of these.
+type ResolvedReader = {
+    rowCount: number;
+    totalBytes: number;
+    keys: string[];
+    columns: { column: string; byteSize: number }[];
+    getColumn: (column: string) => Promise<{ key: string; value: unknown }[]>;
+    getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+};
+
+// Resolve every read by ACTUAL write-time across all readers (stream + bulk), per key and per column:
+//  - a column resolves to the value with the newest write-time among readers that set it (non-ABSENT);
+//    a reader that never set the column for that key falls through to an older reader.
+//  - a key is live iff its newest write is newer than its newest delete; per column, the value is
+//    suppressed if a delete is newer than that column's newest set.
+// No reliance on file order or partitioning — time is the only thing that decides.
+async function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): Promise<ResolvedReader> {
+    const deleteTime = new Map<string, number>();
     for (const db of databases) {
-        if (db.deletedKeys) for (const key of db.deletedKeys) deleted.add(key);
+        if (!db.deleteTimes) continue;
+        for (const [key, t] of db.deleteTimes) deleteTime.set(key, Math.max(deleteTime.get(key) ?? -Infinity, t));
     }
-
-    const keys: string[] = [];
-    const keySeen = new Set<string>();
+    const keyTime = new Map<string, number>();
     for (const db of databases) {
-        for (const key of db.keys) {
-            if (keySeen.has(key) || deleted.has(key)) continue;
-            keySeen.add(key);
-            keys.push(key);
-        }
+        for (const [key, t] of db.keyTimes) keyTime.set(key, Math.max(keyTime.get(key) ?? -Infinity, t));
     }
+    const delOf = (key: string) => deleteTime.get(key) ?? -Infinity;
+    // Live keys: newest write strictly newer than newest delete.
+    const keys: string[] = [];
+    for (const [key, t] of keyTime) if (t > delOf(key)) keys.push(key);
+
     const columns: { column: string; byteSize: number }[] = [];
     const columnByName = new Map<string, { column: string; byteSize: number }>();
     for (const db of databases) {
@@ -761,37 +1076,41 @@ function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): BaseBulkDatabas
             existing.byteSize += col.byteSize;
         }
     }
+
     return {
         totalBytes: databases.reduce((acc, db) => acc + db.totalBytes, 0),
         rowCount: keys.length,
         keys,
         columns,
         async getColumn(column) {
-            const result: { key: string; value: unknown }[] = [];
-            const taken = new Set<string>();
-            for (const db of databases) {
-                // NOTE: This is annoying logic that's needed so that if the column is removed and you write to it, it will clobber the old value. Otherwise, if we just start making something undefined, it might not clobber the old value because the column wouldn't exist. Ugh...
-                let values: unknown[] | undefined;
-                if (db.columns.some(c => c.column === column)) {
-                    values = (await db.getColumn(column)).map(r => r.value);
+            const perReader = await Promise.all(databases.map(async db => {
+                if (!db.columns.some(c => c.column === column)) return undefined;
+                const entries = await db.getColumn(column);
+                return new Map(entries.map(e => [e.key, { value: e.value, time: e.time }]));
+            }));
+            return keys.map(key => {
+                let bestTime = -Infinity;
+                let bestVal: unknown;
+                let found = false;
+                for (const m of perReader) {
+                    const cell = m && m.get(key);
+                    if (!cell || cell.value === ABSENT) continue;
+                    if (cell.time > bestTime) { bestTime = cell.time; bestVal = cell.value; found = true; }
                 }
-                for (let i = 0; i < db.keys.length; i++) {
-                    const key = db.keys[i];
-                    if (taken.has(key) || deleted.has(key)) continue;
-                    taken.add(key);
-                    result.push({ key, value: values && values[i] });
-                }
-            }
-            return result;
+                return { key, value: (found && bestTime > delOf(key)) ? bestVal : undefined };
+            });
         },
         async getSingleField(key, column) {
-            if (deleted.has(key)) return undefined;
-            for (let i = 0; i < databases.length; i++) {
-                if (!keySets[i].has(key)) continue;
-                if (!databases[i].columns.some(c => c.column === column)) return undefined;
-                return await databases[i].getSingleField(key, column);
+            let bestTime = -Infinity;
+            let bestVal: unknown;
+            let found = false;
+            for (const db of databases) {
+                if (!db.columns.some(c => c.column === column)) continue;
+                const r = await db.getSingleField(key, column);
+                if (r === ABSENT) continue;
+                if (r.time > bestTime) { bestTime = r.time; bestVal = r.value; found = true; }
             }
-            return undefined;
+            return (found && bestTime > delOf(key)) ? bestVal : undefined;
         },
     };
 }