sliftutils 1.3.0 → 1.3.1

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -1,12 +1,14 @@
 import { sort } from "socket-function/src/misc";
 import { getTimeUnique } from "socket-function/src/bits";
-import { BaseBulkDatabaseReader, buildFileBuffer, EMPTY_BUFFER, loadBulkDatabase } from "./BulkDatabaseFormat";
+import { ABSENT, BaseBulkDatabaseReader, buildFileBuffer, EMPTY_BUFFER, KEY_COLUMN, loadBulkDatabase } from "./BulkDatabaseFormat";
 import { lazy } from "socket-function/src/caching";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, red } from "socket-function/src/formatting/logColors";
 import { blockCache, encodeCompressedBlocks, GetRange } from "./blockCache";
 import { STREAM_EXTENSION, StreamEntry, frameRows, frameDeletes, parseStream, streamReaderFromEntries } from "./streamLog";
 import { connect as syncConnect, broadcast as syncBroadcast, isSyncSupported, RemoteWrite } from "./syncClient";
+import { Manifest, chooseManifest, isManifestName, manifestFileName, parseManifestStartTime } from "./manifest";
+import { tryAcquireMergeLock, releaseMergeLock } from "./mergeLock";
 import type { FileStorage } from "../FileFolderAPI";
 
 // BulkDatabase2's compressed-block format is not compatible with BulkDatabase, so it uses its own
@@ -25,18 +27,38 @@ const MERGE_FILE_COUNT = 8;
 const MERGE_MIN_BYTES = 400 * 1024 * 1024;
 const MERGE_MAX_BYTES = 800 * 1024 * 1024;
 
-// Tier-0 streaming rolls over into a columnar bulk file once it gets big enough — by row count,
-// byte size, or file count (many threads each stream to their own file). A single writeBatch that
-// already exceeds the row/byte limits skips streaming and writes a bulk file directly.
+// A single writeBatch that already exceeds these limits skips the tier-0 stream and folds straight
+// into a bulk file (streaming thousands of rows one frame at a time would be pointless).
 const ROLLOVER_ROWS = 5000;
 const ROLLOVER_BYTES = 5 * 1024 * 1024;
-const ROLLOVER_FILES = 100;
 
 // An unreadable file might be a write that is still in progress (another thread), so we can't delete
 // it on sight. Once it has been unreadable for longer than this (by its filename timestamp), no
 // writer is plausibly still working on it, so we delete it. Until then we just warn.
 const STALE_DELETE_MS = 24 * 60 * 60 * 1000;
 
+// All the time thresholds, mutable so tests can shrink them from hours to milliseconds. The ordering
+// invariant relies on: streamSealAgeMs < foldDataAgeMs (a file is sealed well before it's foldable),
+// and foldTriggerAgeMs >= foldDataAgeMs.
+export const bulkDatabase2Timing = {
+    // A writer stops appending to its current stream file once it's this old (starts a fresh one), so no
+    // file is ever appended to past this age.
+    streamSealAgeMs: 10 * 60 * 60 * 1000,
+    // A fold reads every stream file CREATED longer ago than this (all sealed, with margin), and moves
+    // only the entries whose WRITE-TIME is older than this into bulk; newer entries are re-streamed.
+    // This single cutoff is what keeps every bulk write strictly older than every stream write.
+    foldDataAgeMs: 12 * 60 * 60 * 1000,
+    // We don't bother folding until some stream file is older than this (fold in big infrequent batches).
+    foldTriggerAgeMs: 36 * 60 * 60 * 1000,
+    // Per-instance throttle on how often we scan to see whether a fold is due.
+    foldCheckIntervalMs: 5 * 1000,
+    // How long a superseded/orphaned/old-manifest file must sit before cleanup deletes it (by its name
+    // timestamp) — long enough that any reader still on an older manifest has finished.
+    cleanupAgeMs: 60 * 1000,
+    // Per-instance throttle on cleanup scans.
+    cleanupIntervalMs: 10 * 1000,
+};
+
 // Marks a key as deleted in the in-memory overlay.
 const DELETED = Symbol("deleted");
 // Each overlay entry carries the write's unique timestamp so cross-tab writes can be ordered: a
@@ -86,6 +108,12 @@ const LOAD_SIGNAL = NULL + "load";
 const OVERLAY_SIGNAL = NULL + "overlay";
 
 let fileNameCounter = 0;
+// Random per-process id baked into file names so two processes (tabs) writing the same collection
+// never collide on a name when they pick the same timestamp/counter in the same millisecond.
+const writerId = Math.random().toString(36).slice(2, 10);
+function nextCounter(): number {
+    return ++fileNameCounter;
+}
 
 type BulkFileInfo = { fileName: string; level: number; timestamp: number };
 
@@ -100,10 +128,9 @@ function nextFileTime(): number {
 
 // Files are ordered purely by timestamp (newest-first). A merged file is given the newest timestamp
 // of the run it replaced, so it occupies exactly that run's slot. The leading "0" is a vestigial
-// field kept so the name stays in the historical level_timestamp_counter shape parseFileName expects.
+// field kept so the name stays in the historical level_timestamp_..._counter shape parseFileName reads.
 function newFileName(timestamp: number): string {
-    fileNameCounter++;
-    return `0_${timestamp}_${fileNameCounter}${FILE_EXTENSION}`;
+    return `0_${timestamp}_${writerId}_${nextCounter()}${FILE_EXTENSION}`;
 }
 
 type StreamFileInfo = { fileName: string; timestamp: number };
@@ -121,7 +148,9 @@ function parseStreamFileName(fileName: string): StreamFileInfo | undefined {
 function parseFileName(fileName: string): BulkFileInfo | undefined {
     if (!fileName.endsWith(FILE_EXTENSION)) return undefined;
     const parts = fileName.slice(0, -FILE_EXTENSION.length).split("_");
-    if (parts.length !== 3) return undefined;
+    // Accept both the old 3-part (level_timestamp_counter) and new 4-part
+    // (level_timestamp_writerId_counter) shapes; level + timestamp are always the first two fields.
+    if (parts.length < 3) return undefined;
     const level = parseInt(parts[0], 10);
     const timestamp = parseInt(parts[1], 10);
     if (!Number.isFinite(level) || !Number.isFinite(timestamp)) return undefined;
@@ -160,24 +189,44 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // This instance's tier-0 stream file. Each instance (≈ each thread/tab) streams to its own file
     // so concurrent writers never touch the same file.
     private streamFileName: string | undefined;
-    private streamRowsWritten = 0;
+    private lastCleanup = 0;
+    private lastFoldCheck = 0;
     private getStreamFileName(): string {
+        // Seal (stop appending to) our current file once it's old enough, so no file is ever appended
+        // to past the seal age — that's what lets a consolidation safely fold it once it's aged out.
+        if (this.streamFileName) {
+            const info = parseStreamFileName(this.streamFileName);
+            if (info && Date.now() - info.timestamp >= bulkDatabase2Timing.streamSealAgeMs) this.streamFileName = undefined;
+        }
         if (!this.streamFileName) {
             this.streamFileName = `stream_${Date.now()}_${Math.random().toString(36).slice(2, 10)}${STREAM_EXTENSION}`;
         }
         return this.streamFileName;
     }
 
-    // Sets an overlay entry and invalidates both that key's signal and the overlay-wide signal.
-    private setOverlay(key: string, entry: OverlayEntry) {
-        this.overlay.set(key, entry);
+    private invalidateOverlay(key: string) {
         this.deps.invalidate(key);
         this.deps.invalidate(OVERLAY_SIGNAL);
     }
 
-    private reader = lazy(async (): Promise<BaseBulkDatabaseReader> => {
+    // Merges a (possibly partial) row onto the key's current overlay value, so a partial write/update
+    // only changes the columns it includes — the rest fall through to disk on read. A prior delete is
+    // reset (the key is being re-created).
+    private setOverlayRow(key: string, row: Record<string, unknown>, time: number) {
+        const existing = this.overlay.get(key);
+        const value = existing && existing.value !== DELETED ? { ...existing.value, ...row } : { ...row };
+        this.overlay.set(key, { time, value });
+        this.invalidateOverlay(key);
+    }
+
+    private setOverlayDeleted(key: string, time: number) {
+        this.overlay.set(key, { time, value: DELETED });
+        this.invalidateOverlay(key);
+    }
+
+    private reader = lazy(async (): Promise<ResolvedReader> => {
         let start = Date.now();
-        const [bulkFiles, streamFiles] = await Promise.all([this.listFiles(), this.listStreamFiles()]);
+        const { bulkFiles, streamFiles } = await this.getValidFiles();
         // Load everything in parallel: each bulk file's columnar reader, plus all streamed entries.
         // A corrupt/truncated bulk file is skipped with a warning rather than breaking the load or
         // returning bad values: the write protocol always writes a new file before removing the old
@@ -194,8 +243,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.loadStreamEntries(streamFiles),
         ]);
         const bulkReaders = bulkReadersRaw.filter((r): r is BaseBulkDatabaseReader => !!r);
-        // Streamed entries are the newest writes, so their reader goes first (the join is newest-wins
-        // and lets the stream's deletes tombstone keys in the older bulk readers).
+        // The join resolves purely by write-time, so reader order doesn't matter.
         const readers: BaseBulkDatabaseReader[] = [];
         const ordered = this.orderStreamEntries(streamData.entries);
         if (ordered.length) {
@@ -206,13 +254,13 @@ export class BulkDatabaseBase<T extends { key: string }> {
             this.streamTimes = new Map();
         }
         readers.push(...bulkReaders);
-        const joined = joinBulkDatabases(readers);
+        const joined = await joinBulkDatabases(readers);
 
         let time = Date.now() - start;
         if (time > 50) {
-            let totalKeysSize = readers.map(x => x.columns.find(c => c.column === "key")?.byteSize || 0).reduce((a, b) => a + b, 0);
-            console.log(`${blue(`${this.name} loaded`)} in ${red(formatTime(time))} (${blue(formatNumber(joined.rowCount))} rows, ${bulkFiles.length} bulk + ${streamFiles.length} stream files, ${blue(formatNumber(totalKeysSize))} keys size)`);
+            console.log(`${blue(`${this.name} loaded`)} in ${red(formatTime(time))} (${blue(formatNumber(joined.rowCount))} rows, ${bulkFiles.length} bulk + ${streamFiles.length} stream files)`);
         }
+        void this.cleanup(); // opportunistic, throttled, fire-and-forget — reads help GC orphans too
         return joined;
     });
 
@@ -242,8 +290,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
     private applyRemote(write: RemoteWrite) {
         if (write.time <= this.localTime(write.key)) return;
         this.deps.batch(() => {
-            if (write.deleted) this.setOverlay(write.key, { time: write.time, value: DELETED });
-            else this.setOverlay(write.key, { time: write.time, value: write.value as Record<string, unknown> });
+            if (write.deleted) this.setOverlayDeleted(write.key, write.time);
+            else this.setOverlayRow(write.key, write.value as Record<string, unknown>, write.time);
         });
     }
 
@@ -277,10 +325,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const stamped = rows.map(row => ({ time: getTimeUnique(), row }));
         const framed = frameRows(stamped);
 
-        // A batch that already exceeds the rollover limits skips tier-0 and writes a bulk file directly.
+        // A batch that already exceeds the limits skips the tier-0 stream and writes a bulk file directly
+        // (streaming thousands of rows one frame at a time would be pointless).
         if (entries.length >= ROLLOVER_ROWS || framed.length >= ROLLOVER_BYTES) {
             await this.writeBulkFile(rows);
-            this.resetReader();
             return;
         }
 
@@ -288,9 +336,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // overlay immediately — no reader reset.
         const storage = await this.storage();
         await storage.append(this.getStreamFileName(), framed);
-        this.streamRowsWritten += entries.length;
         this.deps.batch(() => {
-            for (const { time, row } of stamped) this.setOverlay(row.key as string, { time, value: row });
+            for (const { time, row } of stamped) this.setOverlayRow(row.key as string, row, time);
         });
         for (const { time, row } of stamped) syncBroadcast(this.name, { key: row.key as string, time, value: row });
         await this.maybeRolloverStream();
@@ -306,38 +353,187 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const stamped = keys.map(key => ({ time: getTimeUnique(), key }));
         const storage = await this.storage();
         await storage.append(this.getStreamFileName(), frameDeletes(stamped));
-        this.streamRowsWritten += keys.length;
         this.deps.batch(() => {
-            for (const { time, key } of stamped) this.setOverlay(key, { time, value: DELETED });
+            for (const { time, key } of stamped) this.setOverlayDeleted(key, time);
         });
         for (const { time, key } of stamped) syncBroadcast(this.name, { key, time, deleted: true });
         await this.maybeRolloverStream();
     }
 
-    // Writes the rows as one or more columnar bulk files (buildFileBuffer splits a too-large batch by
-    // row range so no single file approaches the Buffer size limit), all sharing one timestamp since
-    // they're one write with disjoint keys. Then, if enough files have accumulated, runs bounded merge
-    // passes until nothing more can be consolidated.
+    public async update(entry: Partial<T> & { key: string }): Promise<void> {
+        return this.updateBatch([entry]);
+    }
+
+    // Like writeBatch, but each entry is a partial row — only the fields to change, plus the required
+    // key. Partial fields merge onto the existing row (unset columns fall through to the current value);
+    // an entry whose key isn't in the collection is skipped with a warning, since update never creates keys.
+    public async updateBatch(entries: (Partial<T> & { key: string })[]): Promise<void> {
+        if (!entries.length) return;
+        void this.syncSetup();
+        const reader = await this.reader();
+        const diskKeys = new Set(reader.keys);
+        const present: T[] = [];
+        for (const entry of entries) {
+            const overlayEntry = this.overlay.get(entry.key);
+            const exists = overlayEntry ? overlayEntry.value !== DELETED : diskKeys.has(entry.key);
+            if (!exists) {
+                console.warn(`${this.name}.update: key ${JSON.stringify(entry.key)} is not in the collection, ignoring`);
+                continue;
+            }
+            present.push(entry as unknown as T);
+        }
+        if (present.length) await this.writeBatch(present);
+    }
+
+    // Resolves the authoritative on-disk state via manifests (see manifest.ts): valid bulk files
+    // (newest-first) + valid stream files, plus the chosen manifest and the raw name lists the
+    // commit/cleanup paths need. No manifest at all => every bulk file is valid (back-compat). Stream
+    // files are valid unless the chosen manifest lists them as already folded into a bulk file.
+    private async getValidFiles(): Promise<{
+        bulkFiles: BulkFileInfo[];
+        streamFiles: StreamFileInfo[];
+        manifest: Manifest | undefined;
+        manifestName: string | undefined;
+        allBulkNames: string[];
+        allStreamNames: string[];
+        manifestNames: string[];
+    }> {
+        const storage = await this.storage();
+        const names = await storage.getKeys();
+        const manifestNames: string[] = [];
+        const allBulkNames: string[] = [];
+        const allStreamNames: string[] = [];
+        for (const n of names) {
+            if (isManifestName(n)) manifestNames.push(n);
+            else if (n.endsWith(FILE_EXTENSION)) allBulkNames.push(n);
+            else if (n.endsWith(STREAM_EXTENSION)) allStreamNames.push(n);
+        }
+        const parsed = (await Promise.all(manifestNames.map(async name => {
+            try {
+                const buf = await storage.get(name);
+                if (!buf) return undefined;
+                return { name, manifest: JSON.parse(buf.toString("utf8")) as Manifest };
+            } catch {
+                return undefined; // torn/corrupt/half-written manifest — ignore it
+            }
+        }))).filter((m): m is { name: string; manifest: Manifest } => !!m);
+        const chosen = chooseManifest(parsed);
+
+        let validBulkNames: string[];
+        if (!chosen) {
+            validBulkNames = allBulkNames;
+        } else {
+            const valid = new Set(chosen.manifest.validBulkFiles);
+            validBulkNames = allBulkNames.filter(n => valid.has(n));
+        }
+        const ignored = new Set(chosen?.manifest.ignoredStreamFiles || []);
+        const validStreamNames = allStreamNames.filter(n => !ignored.has(n));
+
+        const bulkFiles = validBulkNames.flatMap(n => { const p = parseFileName(n); return p ? [p] : []; });
+        // Newest-first by timestamp; ties broken by file name for determinism.
+        bulkFiles.sort((a, b) => {
+            if (a.timestamp !== b.timestamp) return b.timestamp - a.timestamp;
+            return a.fileName < b.fileName && 1 || a.fileName > b.fileName && -1 || 0;
+        });
+        const streamFiles = validStreamNames.flatMap(n => { const p = parseStreamFileName(n); return p ? [p] : []; });
+        sort(streamFiles, f => f.timestamp);
+
+        return { bulkFiles, streamFiles, manifest: chosen?.manifest, manifestName: chosen?.name, allBulkNames, allStreamNames, manifestNames };
+    }
+
+    // Writes a brand-new manifest (never clobbering an existing one) capturing the full valid state.
+    private async commitManifest(startTime: number, readFiles: string[], validBulkFiles: string[], ignoredStreamFiles: string[]): Promise<void> {
+        const storage = await this.storage();
+        const manifest: Manifest = { startTime, validBulkFiles, ignoredStreamFiles, readFiles };
+        await storage.set(manifestFileName(startTime, writerId, nextCounter()), Buffer.from(JSON.stringify(manifest), "utf8"));
+    }
+
+    // Writes `rows` directly as bulk file(s), stamped with the current time as their data range (the rows
+    // are being written now). Used by the large-batch write path. Commits a new manifest adding them to
+    // the valid set. The rows carry time=now, so the join orders them correctly against any older stream
+    // entry for the same key (newer time wins) — no clobber.
     private async writeBulkFile(rows: Record<string, unknown>[]): Promise<void> {
         const storage = await this.storage();
-        const timestamp = nextFileTime();
-        for (const buffer of buildFileBuffer(rows)) {
-            await storage.set(newFileName(timestamp), encodeCompressedBlocks(buffer));
+        const startTime = nextFileTime();
+        const view = await this.getValidFiles();
+        const now = Date.now();
+        const times = rows.map(() => now);
+        const newBulkNames: string[] = [];
+        for (const buffer of buildFileBuffer(rows, times)) {
+            const name = newFileName(startTime);
+            await storage.set(name, encodeCompressedBlocks(buffer));
+            newBulkNames.push(name);
         }
-        if ((await this.listFiles()).length >= MERGE_FILE_COUNT) {
-            while (await this.mergeFiles() > 0) { /* keep merging until no run can be consolidated */ }
+        const validBulkFiles = view.bulkFiles.map(f => f.fileName).concat(newBulkNames);
+        const readFiles = [...view.allBulkNames, ...view.allStreamNames, ...view.manifestNames];
+        await this.commitManifest(startTime, readFiles, validBulkFiles, view.manifest?.ignoredStreamFiles || []);
+        this.resetReader();
+        if (validBulkFiles.length >= MERGE_FILE_COUNT) {
+            while (await this.mergeFiles() > 0) { /* consolidate accumulated bulk files */ }
         }
+        await this.cleanup();
     }
 
-    private async listStreamFiles(): Promise<StreamFileInfo[]> {
+    // Fold. Reads every stream file CREATED before the cutoff (all sealed, since seal age < fold age),
+    // resolves them into one bulk file carrying each key's latest write-time per row, and re-persists
+    // surviving tombstones to a fresh stream file (so deletes of keys living in older bulk files keep
+    // suppressing them). Because reads resolve by write-time, the folded data needn't be older than the
+    // stream — the join sorts it out — so we just fold whole files, no cutoff split. Folded files are
+    // marked ignored (cleanup deletes them later); the new manifest is the atomic swap.
+    private async consolidate(): Promise<void> {
         const storage = await this.storage();
-        const names = await storage.getKeys();
-        const files = names.flatMap(n => {
-            const parsed = parseStreamFileName(n);
-            return parsed && [parsed] || [];
-        });
-        sort(files, f => f.timestamp);
-        return files;
+        const startTime = nextFileTime();
+        const view = await this.getValidFiles();
+        const cutoff = Date.now() - bulkDatabase2Timing.foldDataAgeMs;
+        const selected = view.streamFiles.filter(f => f.timestamp < cutoff);
+        if (!selected.length) return;
+        const { entries } = await this.loadStreamEntries(selected);
+        const ordered = this.orderStreamEntries(entries);
+
+        const byKey = new Map<string, Record<string, unknown>>();
+        const byKeyTime = new Map<string, number>();
+        const deleted = new Map<string, number>();
+        for (const e of ordered) {
+            if (e.deletedKey !== undefined) {
+                byKey.delete(e.deletedKey);
+                byKeyTime.delete(e.deletedKey);
+                deleted.set(e.deletedKey, e.time);
+            } else if (e.row) {
+                const key = e.row.key as string;
+                byKey.set(key, { ...byKey.get(key), ...e.row });
+                byKeyTime.set(key, e.time); // ordered ascending, so this ends up the latest write
+                deleted.delete(key);
+            }
+        }
+
+        const newBulkNames: string[] = [];
+        if (byKey.size) {
+            const rows = [...byKey.values()];
+            const times = rows.map(r => byKeyTime.get(r.key as string)!);
+            for (const buffer of buildFileBuffer(rows, times)) {
+                const name = newFileName(startTime);
+                await storage.set(name, encodeCompressedBlocks(buffer));
+                newBulkNames.push(name);
+            }
+        }
+
+        // Surviving tombstones -> a fresh stream file (always valid). Written BEFORE the manifest so the
+        // folded sources are never ignored while their deletes are missing.
+        if (deleted.size) {
+            const carryName = `stream_${Date.now()}_${Math.random().toString(36).slice(2, 10)}${STREAM_EXTENSION}`;
+            await storage.set(carryName, frameDeletes([...deleted].map(([key, time]) => ({ time, key }))));
+        }
+
+        const validBulkFiles = view.bulkFiles.map(f => f.fileName).concat(newBulkNames);
+        const ignoredStreamFiles = [...new Set([...(view.manifest?.ignoredStreamFiles || []), ...selected.map(f => f.fileName)])];
+        const readFiles = [...view.allBulkNames, ...view.allStreamNames, ...view.manifestNames];
+        await this.commitManifest(startTime, readFiles, validBulkFiles, ignoredStreamFiles);
+        this.resetReader();
+
+        if (validBulkFiles.length >= MERGE_FILE_COUNT) {
+            while (await this.mergeFiles() > 0) { /* consolidate accumulated bulk files */ }
+        }
+        await this.cleanup();
     }
 
     // Reads and parses every stream file in parallel. Returns per-write entries (each carrying its
@@ -345,7 +541,19 @@ export class BulkDatabaseBase<T extends { key: string }> {
     private async loadStreamEntries(streamFiles: StreamFileInfo[]): Promise<{ entries: { time: number; fileName: string; entry: StreamEntry }[]; totalBytes: number }> {
         if (!streamFiles.length) return { entries: [], totalBytes: 0 };
         const storage = await this.storage();
-        const buffers = await Promise.all(streamFiles.map(f => storage.get(f.fileName)));
+        // Read a bounded prefix [0, size) rather than the whole file: a foreign writer may be appending
+        // concurrently, and storage.get() errors when the file grows past the size it stat'd. Reading a
+        // prefix is tolerant — parseStream stops at the last complete frame, the file stays valid, and a
+        // later read picks up the rest. A file removed out from under us (cleanup) just yields undefined.
+        const buffers = await Promise.all(streamFiles.map(async f => {
+            try {
+                const info = await storage.getInfo(f.fileName);
+                if (!info || info.size === 0) return undefined;
+                return await storage.getRange(f.fileName, { start: 0, end: info.size });
+            } catch {
+                return undefined;
+            }
+        }));
         const entries: { time: number; fileName: string; entry: StreamEntry }[] = [];
         let totalBytes = 0;
         for (let i = 0; i < streamFiles.length; i++) {
@@ -372,51 +580,18 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return entries.map(e => e.entry);
     }
 
+    // Folding is purely age-driven and done in big infrequent batches: once some stream file is older
+    // than the trigger age, fold everything past the data cutoff. Throttled so we don't scan on every write.
     private async maybeRolloverStream(): Promise<void> {
-        const streamFiles = await this.listStreamFiles();
-        const storage = await this.storage();
-        let totalBytes = 0;
-        for (const f of streamFiles) {
-            const info = await storage.getInfo(f.fileName);
-            totalBytes += info?.size || 0;
-        }
-        if (streamFiles.length > ROLLOVER_FILES || totalBytes > ROLLOVER_BYTES || this.streamRowsWritten > ROLLOVER_ROWS) {
-            await this.rolloverStream(streamFiles);
+        const now = Date.now();
+        if (now - this.lastFoldCheck < bulkDatabase2Timing.foldCheckIntervalMs) return;
+        this.lastFoldCheck = now;
+        const { streamFiles } = await this.getValidFiles();
+        if (streamFiles.some(f => now - f.timestamp >= bulkDatabase2Timing.foldTriggerAgeMs)) {
+            await this.consolidate();
         }
     }
 
-    // Combine all tier-0 stream files into a single columnar bulk file (newest-wins per key, deletes
-    // applied), delete the consumed stream files, and re-persist surviving tombstones to a fresh
-    // stream file so deletes of keys that live in older bulk files are not lost.
-    private async rolloverStream(streamFiles: StreamFileInfo[]): Promise<void> {
-        const { entries } = await this.loadStreamEntries(streamFiles);
-        const ordered = this.orderStreamEntries(entries);
-        const byKey = new Map<string, Record<string, unknown>>();
-        const deleted = new Map<string, number>();
-        for (const e of ordered) {
-            if (e.deletedKey !== undefined) {
-                byKey.delete(e.deletedKey);
-                deleted.set(e.deletedKey, e.time);
-            } else if (e.row) {
-                let key = e.row.key as string;
-                byKey.set(key, e.row);
-                deleted.delete(key);
-            }
-        }
-        if (byKey.size) await this.writeBulkFile([...byKey.values()]);
-        const storage = await this.storage();
-        // Persist surviving tombstones (keeping their original timestamps) to a FRESH stream file
-        // before removing the consumed files, so a crash in between can't drop the deletes. The window
-        // is at worst redundant (deletes present in both old and new files), never missing.
-        this.streamFileName = undefined;
-        this.streamRowsWritten = 0;
-        if (deleted.size) {
-            await storage.append(this.getStreamFileName(), frameDeletes([...deleted].map(([key, time]) => ({ time, key }))));
-        }
-        for (const f of streamFiles) await storage.remove(f.fileName);
-        this.resetReader();
-    }
-
     // Consolidate as much as the caps allow: repeatedly merge contiguous non-sealed runs until nothing
     // more can be combined. Unlike a naive "merge everything into one file", this respects MERGE_MAX_BYTES
     // so it never loads the whole collection into memory — a multi-GB collection settles into several
@@ -427,23 +602,6 @@ export class BulkDatabaseBase<T extends { key: string }> {
         if (merged) this.resetReader();
     }
 
-    private async listFiles(): Promise<BulkFileInfo[]> {
-        const storage = await this.storage();
-        const names = await storage.getKeys();
-        const files = names.flatMap(n => {
-            const parsed = parseFileName(n);
-            return parsed && [parsed] || [];
-        });
-        // Newest-first by timestamp; ties broken by file name (descending) for a deterministic order.
-        // A merged file inherits the newest timestamp of the run it replaced, so it lands exactly where
-        // that run was — keeping newest-wins correct without any level bookkeeping.
-        files.sort((a, b) => {
-            if (a.timestamp !== b.timestamp) return b.timestamp - a.timestamp;
-            return a.fileName < b.fileName && 1 || a.fileName > b.fileName && -1 || 0;
-        });
-        return files;
-    }
-
     private async makeRawGetRange(fileName: string): Promise<{ rawGetRange: GetRange; size: number } | undefined> {
         const storage = await this.storage();
         const info = await storage.getInfo(fileName);
@@ -507,51 +665,84 @@ export class BulkDatabaseBase<T extends { key: string }> {
         console.warn(`${this.name}: skipping unreadable bulk file ${file.fileName} (recent — may be an in-progress write): ${message}`);
     }
 
-    // Merges exactly the files it's given (already newest-first), newest-wins per key, writing the
-    // result with `timestamp` so the new file takes the slot of the run it replaced, then deletes the
-    // consumed files. The caller is responsible for keeping the input under MERGE_MAX_BYTES — this
-    // function blindly reads it all into memory.
-    private async mergeFilesBase(files: BulkFileInfo[], timestamp: number): Promise<void> {
+    // Merges exactly the files it's given, returning the new file name(s). It does NOT delete the inputs
+    // or write a manifest — the caller (mergeFiles) commits one manifest for the whole pass. The merge is
+    // per-COLUMN by write-TIME: for each key, each column takes the value with the newest write-time
+    // across the merged files (non-ABSENT), and the output row's time is the newest of those — so the
+    // merge preserves correct time-resolution (only collapsing per-column times within a single output
+    // row, the accepted per-row corner). The caller keeps the input under MERGE_MAX_BYTES.
+    private async mergeFilesBase(files: BulkFileInfo[], timestamp: number): Promise<string[]> {
         const storage = await this.storage();
         const readers = await Promise.all(files.map(f => this.loadFileReader(f.fileName)));
 
-        const seen = new Set<string>();
-        const mergedRows: Record<string, unknown>[] = [];
-        for (const reader of readers) {
-            const colData: Record<string, unknown[]> = {};
+        const loaded = await Promise.all(readers.map(async reader => {
+            const cols = new Map<string, Map<string, { value: unknown; time: number }>>();
             for (const col of reader.columns) {
-                colData[col.column] = (await reader.getColumn(col.column)).map(r => r.value);
+                if (col.column === KEY_COLUMN) continue;
+                const entries = await reader.getColumn(col.column);
+                cols.set(col.column, new Map(entries.map(e => [e.key, { value: e.value, time: e.time }])));
             }
-            for (let i = 0; i < reader.keys.length; i++) {
-                const key = reader.keys[i];
-                if (seen.has(key)) continue;
-                seen.add(key);
-                const row: Record<string, unknown> = {};
-                for (const col of reader.columns) {
-                    row[col.column] = colData[col.column][i];
+            return { keyTimes: reader.keyTimes, cols };
+        }));
+
+        const allKeys = new Set<string>();
+        const allCols = new Set<string>();
+        for (const l of loaded) {
+            for (const k of l.keyTimes.keys()) allKeys.add(k);
+            for (const c of l.cols.keys()) allCols.add(c);
+        }
+
+        const mergedRows: Record<string, unknown>[] = [];
+        const mergedTimes: number[] = [];
+        for (const key of allKeys) {
+            const row: Record<string, unknown> = { [KEY_COLUMN]: key };
+            let rowTime = -Infinity;
+            for (const col of allCols) {
+                let bestTime = -Infinity;
+                let bestVal: unknown;
+                let found = false;
+                for (const l of loaded) {
+                    const cell = l.cols.get(col)?.get(key);
+                    if (!cell || cell.value === ABSENT) continue;
+                    if (cell.time > bestTime) { bestTime = cell.time; bestVal = cell.value; found = true; }
                 }
-                mergedRows.push(row);
+                if (found) { row[col] = bestVal; if (bestTime > rowTime) rowTime = bestTime; }
             }
+            for (const l of loaded) { const t = l.keyTimes.get(key); if (t !== undefined && t > rowTime) rowTime = t; }
+            mergedRows.push(row);
+            mergedTimes.push(rowTime === -Infinity ? 0 : rowTime);
         }
 
-        // The input is under the cap, so buildFileBuffer almost always returns a single buffer; the loop
-        // is only here to stay correct if a merge's deduped output still happens to exceed the split size.
-        for (const buffer of buildFileBuffer(mergedRows)) {
-            await storage.set(newFileName(timestamp), encodeCompressedBlocks(buffer));
-        }
-        for (const f of files) {
-            await storage.remove(f.fileName);
+        const names: string[] = [];
+        for (const buffer of buildFileBuffer(mergedRows, mergedTimes)) {
+            const name = newFileName(timestamp);
+            await storage.set(name, encodeCompressedBlocks(buffer));
+            names.push(name);
         }
+        return names;
     }
 
-    // The cap-aware merge planner. Walks the files newest-first and merges contiguous runs of
+    // The cap-aware merge planner. Walks the valid files newest-first and merges contiguous runs of
     // non-sealed files, each run capped at MERGE_MAX_BYTES of LOGICAL size so a single merge never
     // loads more than that into memory. A file at/over MERGE_MIN_BYTES is sealed (left untouched) and
-    // breaks the run, as does an unreadable file. Because each run is contiguous in the newest-first
-    // order and its merged file keeps the run's newest timestamp, newest-wins ordering is preserved
-    // with no inversions. Returns the number of runs merged (0 means nothing left to consolidate).
+    // breaks the run, as does an unreadable file. The whole pass is committed as ONE new manifest:
+    // valid bulk = (old valid - consumed) + merged outputs; the consumed files are left on disk for
+    // cleanup. Returns the number of runs merged (0 means nothing left to consolidate).
     private async mergeFiles(): Promise<number> {
-        const files = await this.listFiles();
+        // Best-effort cross-tab lock: if another tab is already merging, skip — the manifest backstop
+        // keeps us correct, and we'd only be racing to orphan each other's output. No-op in Node.
+        if (!tryAcquireMergeLock(this.name, writerId)) return 0;
+        try {
+            return await this.mergeFilesLocked();
+        } finally {
+            releaseMergeLock(this.name, writerId);
+        }
+    }
+
+    private async mergeFilesLocked(): Promise<number> {
+        const startTime = nextFileTime();
+        const view = await this.getValidFiles();
+        const files = view.bulkFiles;
         const sizes = await Promise.all(files.map(f => this.fileLogicalSize(f.fileName)));
 
         const batches: BulkFileInfo[][] = [];
@@ -574,25 +765,80 @@ export class BulkDatabaseBase<T extends { key: string }> {
             batchBytes += size;
         }
         flush();
+        if (!batches.length) return 0;
 
-        // batch[0] is the newest file in each (newest-first) run, so its timestamp is the run's slot.
+        const removed = new Set<string>();
+        const newBulkNames: string[] = [];
         for (const runFiles of batches) {
-            await this.mergeFilesBase(runFiles, runFiles[0].timestamp);
+            // batch[0] is the newest file in each (newest-first) run, so its timestamp is the run's slot.
+            const produced = await this.mergeFilesBase(runFiles, runFiles[0].timestamp);
+            for (const f of runFiles) removed.add(f.fileName);
+            newBulkNames.push(...produced);
         }
+        const validBulkFiles = files.map(f => f.fileName).filter(n => !removed.has(n)).concat(newBulkNames);
+        const ignoredStreamFiles = view.manifest?.ignoredStreamFiles || [];
+        const readFiles = [...view.allBulkNames, ...view.allStreamNames, ...view.manifestNames];
+        await this.commitManifest(startTime, readFiles, validBulkFiles, ignoredStreamFiles);
+        this.resetReader();
+        await this.cleanup();
         return batches.length;
     }
 
-    private formatInfo(reader: BaseBulkDatabaseReader): string {
+    // Deletes files no longer referenced by the authoritative manifest that have sat long enough that
+    // no reader still resolving an older manifest needs them: superseded/orphaned bulk files,
+    // folded-away (ignored) stream files, and every manifest but the newest. Age-gated by each file's
+    // own name timestamp and throttled per instance. Best-effort — a failed remove (another writer beat
+    // us to it) is ignored. We never delete a file whose name we can't parse for an age.
+    private async cleanup(): Promise<void> {
+        const now = Date.now();
+        if (now - this.lastCleanup < bulkDatabase2Timing.cleanupIntervalMs) return;
+        this.lastCleanup = now;
+        // Best-effort and must never throw: it runs fire-and-forget from reads, and the directory could
+        // even be removed out from under us (e.g. the collection is being deleted) mid-scan.
+        try {
+            const storage = await this.storage();
+            const view = await this.getValidFiles();
+            const validBulk = new Set(view.bulkFiles.map(f => f.fileName));
+            const validStream = new Set(view.streamFiles.map(f => f.fileName));
+            const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };
+
+            for (const name of view.allBulkNames) {
+                if (validBulk.has(name)) continue;
+                const info = parseFileName(name);
+                if (!info || now - info.timestamp < bulkDatabase2Timing.cleanupAgeMs) continue;
+                await remove(name);
+            }
+            for (const name of view.allStreamNames) {
+                if (validStream.has(name)) continue;
+                const info = parseStreamFileName(name);
+                if (!info || now - info.timestamp < bulkDatabase2Timing.cleanupAgeMs) continue;
+                await remove(name);
+            }
+            for (const name of view.manifestNames) {
+                if (name === view.manifestName) continue;
+                const startTime = parseManifestStartTime(name);
+                if (startTime === undefined || now - startTime < bulkDatabase2Timing.cleanupAgeMs) continue;
+                await remove(name);
+            }
+        } catch {
+            // ignore — cleanup is opportunistic; the next pass will catch up
+        }
+    }
+
+    private formatInfo(reader: ResolvedReader): string {
         return `(collection has ${blue(formatNumber(reader.rowCount))} rows, ${blue(formatNumber(reader.totalBytes))}B)`;
     }
 
-    // Applies the overlay (pending writes/deletes) on top of a base column. No-op when empty.
+    // Applies the overlay (pending writes/deletes) on top of a base column. No-op when empty. An
+    // overlay entry that doesn't include this column leaves the base (disk) value in place — a partial
+    // write/update only overrides the columns it set; everything else falls through.
     private patchColumn(base: { key: string; value: unknown }[], column: string): { key: string; value: unknown }[] {
         if (this.overlay.size === 0) return base;
         const map = new Map(base.map(e => [e.key, e.value]));
         for (const [key, entry] of this.overlay) {
-            if (entry.value === DELETED) map.delete(key);
-            else map.set(key, entry.value[column]);
+            if (entry.value === DELETED) { map.delete(key); continue; }
+            if (column in entry.value) map.set(key, entry.value[column]);
+            else if (!map.has(key)) map.set(key, undefined);
         }
         return [...map].map(([key, value]) => ({ key, value }));
     }
@@ -604,7 +850,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const entry = this.overlay.get(key);
         if (entry !== undefined) {
             if (entry.value === DELETED) return undefined;
-            return entry.value[String(column)] as T[Column];
+            if (String(column) in entry.value) return entry.value[String(column)] as T[Column];
+            // column not set in the overlay entry — fall through to disk
         }
         let time = Date.now();
         let reader = await this.reader();
@@ -689,7 +936,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
         let entry = this.overlay.get(key);
         if (entry !== undefined) {
             if (entry.value === DELETED) return undefined;
-            return entry.value[col] as T[Column];
+            if (col in entry.value) return entry.value[col] as T[Column];
+            // column not set in the overlay entry — fall through to the base field cache
         }
         let cacheKey = nullJoin(col, key);
         if (!this.baseFields.has(cacheKey)) {
@@ -730,24 +978,38 @@ export class BulkDatabaseBase<T extends { key: string }> {
     }
 }
 
-// Lowest indexes are read first (newest-wins). A reader's deletedKeys tombstone a key in all older
-// readers; the newest reader that has a key live wins.
-function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): BaseBulkDatabaseReader {
-    const keySets = databases.map(db => new Set(db.keys));
-    const deleted = new Set<string>();
+// The merged, time-resolved view over all readers. getColumn/getSingleField return plain resolved
+// values (not {value,time}); the base layers the overlay on top of these.
+type ResolvedReader = {
+    rowCount: number;
+    totalBytes: number;
+    keys: string[];
+    columns: { column: string; byteSize: number }[];
+    getColumn: (column: string) => Promise<{ key: string; value: unknown }[]>;
+    getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+};
+
+// Resolve every read by ACTUAL write-time across all readers (stream + bulk), per key and per column:
+//  - a column resolves to the value with the newest write-time among readers that set it (non-ABSENT);
+//    a reader that never set the column for that key falls through to an older reader.
+//  - a key is live iff its newest write is newer than its newest delete; per column, the value is
+//    suppressed if a delete is newer than that column's newest set.
+// No reliance on file order or partitioning — time is the only thing that decides.
+async function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): Promise<ResolvedReader> {
+    const deleteTime = new Map<string, number>();
     for (const db of databases) {
-        if (db.deletedKeys) for (const key of db.deletedKeys) deleted.add(key);
+        if (!db.deleteTimes) continue;
+        for (const [key, t] of db.deleteTimes) deleteTime.set(key, Math.max(deleteTime.get(key) ?? -Infinity, t));
     }
-
-    const keys: string[] = [];
-    const keySeen = new Set<string>();
+    const keyTime = new Map<string, number>();
     for (const db of databases) {
-        for (const key of db.keys) {
-            if (keySeen.has(key) || deleted.has(key)) continue;
-            keySeen.add(key);
-            keys.push(key);
-        }
+        for (const [key, t] of db.keyTimes) keyTime.set(key, Math.max(keyTime.get(key) ?? -Infinity, t));
     }
+    const delOf = (key: string) => deleteTime.get(key) ?? -Infinity;
+    // Live keys: newest write strictly newer than newest delete.
+    const keys: string[] = [];
+    for (const [key, t] of keyTime) if (t > delOf(key)) keys.push(key);
+
     const columns: { column: string; byteSize: number }[] = [];
     const columnByName = new Map<string, { column: string; byteSize: number }>();
     for (const db of databases) {
@@ -761,37 +1023,41 @@ function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): BaseBulkDatabas
             existing.byteSize += col.byteSize;
         }
     }
+
     return {
         totalBytes: databases.reduce((acc, db) => acc + db.totalBytes, 0),
         rowCount: keys.length,
         keys,
         columns,
         async getColumn(column) {
-            const result: { key: string; value: unknown }[] = [];
-            const taken = new Set<string>();
-            for (const db of databases) {
-                // NOTE: This is annoying logic that's needed so that if the column is removed and you write to it, it will clobber the old value. Otherwise, if we just start making something undefined, it might not clobber the old value because the column wouldn't exist. Ugh...
-                let values: unknown[] | undefined;
-                if (db.columns.some(c => c.column === column)) {
-                    values = (await db.getColumn(column)).map(r => r.value);
+            const perReader = await Promise.all(databases.map(async db => {
+                if (!db.columns.some(c => c.column === column)) return undefined;
+                const entries = await db.getColumn(column);
+                return new Map(entries.map(e => [e.key, { value: e.value, time: e.time }]));
+            }));
+            return keys.map(key => {
+                let bestTime = -Infinity;
+                let bestVal: unknown;
+                let found = false;
+                for (const m of perReader) {
+                    const cell = m && m.get(key);
+                    if (!cell || cell.value === ABSENT) continue;
+                    if (cell.time > bestTime) { bestTime = cell.time; bestVal = cell.value; found = true; }
                 }
-                for (let i = 0; i < db.keys.length; i++) {
-                    const key = db.keys[i];
-                    if (taken.has(key) || deleted.has(key)) continue;
-                    taken.add(key);
-                    result.push({ key, value: values && values[i] });
-                }
-            }
-            return result;
+                return { key, value: (found && bestTime > delOf(key)) ? bestVal : undefined };
+            });
         },
         async getSingleField(key, column) {
-            if (deleted.has(key)) return undefined;
-            for (let i = 0; i < databases.length; i++) {
-                if (!keySets[i].has(key)) continue;
-                if (!databases[i].columns.some(c => c.column === column)) return undefined;
-                return await databases[i].getSingleField(key, column);
+            let bestTime = -Infinity;
+            let bestVal: unknown;
+            let found = false;
+            for (const db of databases) {
+                if (!db.columns.some(c => c.column === column)) continue;
+                const r = await db.getSingleField(key, column);
+                if (r === ABSENT) continue;
+                if (r.time > bestTime) { bestTime = r.time; bestVal = r.value; found = true; }
             }
-            return undefined;
+            return (found && bestTime > delOf(key)) ? bestVal : undefined;
         },
     };
 }