sliftutils 1.7.121 → 1.7.122

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -16,7 +16,7 @@ import { blockCache, encodeCompressedBlocks } from "./blockCache";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { STREAM_EXTENSION, frameDeletes, frameRows, streamReaderFromEntries } from "./streamLog";
-import { broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, connect as syncConnect, isSyncSupported, RemoteWrite } from "./syncClient";
+import { broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, connect as syncConnect, isSyncSupported, queryLiveWriters, registerWriterId, RemoteWrite } from "./syncClient";
 import { DELETED } from "./WriteOverlay";
 import { MergeLockInfo, peekMergeFileLock, peekMergeLock, releaseMergeFileLock, releaseMergeLock, startMergeFileLockHeartbeat, tryAcquireMergeFileLock, tryAcquireMergeLock } from "./mergeLock";
 import { markerExclusions, processDeleteMarkers, readDeleteMarkers, writeDeleteMarker } from "./mergeMarkers";
@@ -40,10 +40,9 @@ const ROLLOVER_BYTES = 5 * 1024 * 1024;
 const MEMORY_WATCHDOG_INTERVAL_MS = 60 * 1000;
 const STALE_DELETE_MS = 24 * 60 * 60 * 1000;
 const MAX_INDEX_RELOAD_ATTEMPTS = 3;
-const FIRST_MERGE_BYTES = TARGET_FILE_BYTES / 2;
 const KEY_GROUP_BYTES = 800 * 1024 * 1024;
 const DUP_THRESHOLD = 0.4;
-// Whole-tier dedup short-circuit (after Pass 1): when the bulk tier is over this size AND the overall key duplication fraction is over this threshold, fold every bulk file in one merge instead of the per-key-group walk (which spaces merges 5 min apart — 16 h for 200 groups).
+// Whole-tier dedup short-circuit (start of phase 3): when the combined tier is over this size AND the overall key duplication fraction is over this threshold, fold every combined file in one merge instead of the per-key-group walk (which spaces merges 5 min apart — 16 h for 200 groups).
 const DEDUP_TRIGGER_BYTES = 512 * 1024 * 1024;
 const DEDUP_TRIGGER_FRACTION = 0.5;
 const WRITE_FLUSH_FIRST_STEP_MS = 250;
@@ -55,11 +54,13 @@ export const bulkDatabase2Timing = {
     // Wait this long after the tab becomes visible before the first merge check, then check this often afterwards. Tab being hidden cancels the timers (no merges while in background). Browser-only; Node compactors (e.g. remoteFileServer) drive their own polling.
     visibleMergeIntervalMs: 5 * 60 * 1000,
     mergeSpacingMs: 5 * 60 * 1000,
-    firstMergeTriggerFiles: 20,
-    firstMergeTriggerRangeMs: 3 * 24 * 60 * 60 * 1000,
-    streamFoldTriggerRows: 100,
+    // Phase 2 rolls the loose bulk files up once they pass either of these. Bytes because read cost grows with them; count because each loose file holds its whole key list in memory no matter how few bytes it has, so a trickle of tiny folds has to be collapsed long before it reaches the byte trigger.
+    looseBulkTriggerBytes: 1024 * 1024 * 1024,
+    looseBulkTriggerFiles: 20,
     streamFoldTriggerBytes: 64 * 1024 * 1024,
     streamFileMaxBytes: 50 * 1024 * 1024,
+    // How long to wait for peers to answer a stream-owner liveness probe. Runs inside a merge pass, which already waits 15s for the file lock, so a second here costs nothing.
+    liveWriterProbeMs: 1000,
     streamFoldHardLimitBytes: 768 * 1024 * 1024,
     // 0 = flush every write (Node — append is real and cheap); browser ramps to 15s to avoid rewriting the whole stream file per write.
     writeFlushMaxDelayMs: isNode() ? 0 : 15 * 1000,
@@ -120,8 +121,11 @@ export type MergeAttemptResult = {
 let networkCompactionEnabled = false;
 
 let fileNameCounter = 0;
-// Per-process ID so two writers picking the same timestamp + counter never collide on a name.
+// Per-process ID so two writers picking the same timestamp + counter never collide on a name. Also stamped into our stream file names and answered over the sync channel, so a peer can tell whether the writer of a given stream file is still running.
 const writerId = Math.random().toString(36).slice(2, 10);
+registerWriterId(writerId);
+// Reserved owner for the tombstone-carry file a merge emits. Nothing ever appends to it, but it must NOT read as abandoned: folding it would emit another carry file, which would read as abandoned in turn, and the pass would fold forever.
+const MERGE_OUTPUT_OWNER = "merged";
 function nextCounter(): number { return ++fileNameCounter; }
 
 let lastFileTime = 0;
@@ -135,13 +139,19 @@ function newFileName(timestamp: number): string {
     return `0_${timestamp}_${writerId}_${nextCounter()}${FILE_EXTENSION}`;
 }
 
+// Accept old 3-part (stream_timestamp_random) and new 4-part (stream_timestamp_ownerId_counter) shapes. A 3-part name carries no owner, so it can never be matched to a live writer — which is what we want: no build that produces those names is still appending to them.
 function parseStreamFileName(fileName: string): StreamFileInfo | undefined {
     if (!fileName.endsWith(STREAM_EXTENSION)) return undefined;
     const parts = fileName.slice(0, -STREAM_EXTENSION.length).split("_");
-    if (parts.length !== 3 || parts[0] !== "stream") return undefined;
+    if (parts[0] !== "stream") return undefined;
+    if (parts.length !== 3 && parts.length !== 4) return undefined;
     const timestamp = parseInt(parts[1], 10);
     if (!Number.isFinite(timestamp)) return undefined;
-    return { fileName, timestamp };
+    return { fileName, timestamp, ownerId: parts.length === 4 && parts[2] || undefined };
+}
+
+function newStreamFileName(ownerId: string): string {
+    return `stream_${Date.now()}_${ownerId}_${nextCounter()}${STREAM_EXTENSION}`;
 }
 
 // Accept old 3-part (level_timestamp_counter) and new 4-part (level_timestamp_writerId_counter) shapes.
@@ -155,6 +165,57 @@ function parseFileName(fileName: string): BulkFileInfo | undefined {
     return { fileName, level, timestamp };
 }
 
+/** One threshold a compaction step is measured against. `value` is where the collection stands now and `threshold` is what sets the step off, so `fraction` (value/threshold) reads as how close it is - 1 or more means met. Deliberately not clamped, so an overdue step reads as how far past due it is. */
+export type CompactionTrigger = {
+    name: string;
+    value: number;
+    threshold: number;
+    fraction: number;
+    met: boolean;
+    /** How to render value/threshold. "fraction" values are 0..1. */
+    unit: "bytes" | "count" | "fraction";
+};
+
+/** streamHardLimit and streamFold are phase 1 (stream -> bulk), looseCombine is phase 2 (loose bulk -> combined bulk), dedupAll and dedupKeyGroup are phase 3. */
+export type CompactionStepKind = "streamHardLimit" | "streamFold" | "looseCombine" | "dedupAll" | "dedupKeyGroup";
+
+export type CompactionStep = {
+    phase: 1 | 2 | 3;
+    kind: CompactionStepKind;
+    /** Whether this step runs on the next pass. Authoritative: on top of `requires` it accounts for inputs the step needs beyond its thresholds (e.g. two files to combine), so it can be false even with every trigger met. */
+    ready: boolean;
+    /** Whether every trigger has to be met for this step, or just one of them. */
+    requires: "any" | "all";
+    triggers: CompactionTrigger[];
+    /** When this step's merge starts, given merges are spaced mergeSpacingMs apart. Only set when ready. */
+    startTime?: number;
+    /** The files this step consumes, as of when the plan was made. */
+    bulkFiles: BulkFileInfo[];
+    streamFiles: StreamFileInfo[];
+    /** Total size of those inputs. */
+    bytes: number;
+    /** dedupKeyGroup only - the key range the step rewrites. */
+    keyRange?: { lo: string; hi: string };
+};
+
+/** Every compaction the current file set calls for, in the order a merge pass runs them, plus how close each not-yet-ready one is to its thresholds. */
+export type CompactionPlan = {
+    collection: string;
+    /** When the plan was computed; every startTime is measured from here. */
+    time: number;
+    steps: CompactionStep[];
+};
+
+function makeTrigger(config: { name: string; value: number; threshold: number; unit: CompactionTrigger["unit"] }): CompactionTrigger {
+    return { ...config, fraction: config.value / config.threshold, met: config.value >= config.threshold };
+}
+
+function fmtTriggerValue(value: number, unit: CompactionTrigger["unit"]): string {
+    if (unit === "bytes") return fmtBytes(value);
+    if (unit === "fraction") return `${Math.round(value * 100)}%`;
+    return formatNumber(value);
+}
+
 export class BulkDatabaseBase<T extends { key: string }> {
     constructor(
         public readonly name: string,
@@ -229,7 +290,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
 
     private subCaches: SubReaderCaches = { bulk: new Map(), stream: new Map() };
 
-    private pendingAppends: { framed: Buffer; rows: number }[] = [];
+    private pendingAppends: Buffer[] = [];
     private flushTimer: ReturnType<typeof setTimeout> | undefined;
     private flushChain: Promise<void> = Promise.resolve();
     private currentFlushDelay = 0;
@@ -241,8 +302,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // In-process re-entry guard: if a merge is running, additional triggers (visibility timer, writes, tryMergeNow calls) return immediately instead of queueing. Keeps the visibility-timer firings from stacking behind a long merge.
     private mergeInFlight = false;
     private lastMergeSkipLogMs = 0;
-    // Running counters of stream-tier rows + bytes on disk. Seeded from each LoadedIndex build, then incremented per flush so the fold-trigger checks current data without an extra directory listing.
-    private streamRowsOnDisk = 0;
+    // Running counter of stream-tier bytes on disk. Seeded from each LoadedIndex build, then incremented per flush so the fold-trigger checks current data without an extra directory listing.
     private streamBytesOnDisk = 0;
 
     private fileSetPollTimer: ReturnType<typeof setInterval> | undefined;
@@ -298,9 +358,36 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return !!(await this.storage()).isRemote;
     }
 
+    // Uncompacted stream bytes have to be read and decoded in full by every reader on every index build, so size alone decides — a fold is worth it even if those bytes are one enormous row.
     private streamNeedsFold(): boolean {
-        return this.streamRowsOnDisk >= bulkDatabase2Timing.streamFoldTriggerRows
-            && this.streamBytesOnDisk > bulkDatabase2Timing.streamFoldTriggerBytes;
+        return this.streamBytesOnDisk > bulkDatabase2Timing.streamFoldTriggerBytes;
+    }
+
+    // Stream files nobody can still be appending to, split by who owned them:
+    //   retirable - a fold may delete these the moment it has consumed them, instead of waiting out streamSealAgeMs. Includes our own sealed files and merge-carry output.
+    //   ownerGone - nobody is left to fold these (a closed tab's leftovers, or a legacy name with no owner stamp), and until one is folded every reader has to decode all of it just to build the index. Worth folding at any size. Our own files are deliberately NOT in here: every merge pass seals ours, so counting them would fold on every tick; the streamFileMaxBytes rollover and the streamFoldTriggerBytes gate cover ours instead.
+    //
+    // Foreign owners are probed over the sync channel. In Node there is no channel, so we cannot know - every foreign owner is assumed alive and the streamSealAgeMs rule stands.
+    //
+    // assumeSealed is for planning: a merge pass broadcasts a seal before it starts, so by the time it merges our current file IS final. The planner passes isSyncSupported() to predict that; anything deciding a real deletion passes false and goes by streamFileName as it actually stands.
+    private async findAbandonedStreams(streamFiles: StreamFileInfo[], assumeSealed: boolean): Promise<{ retirable: Set<string>; ownerGone: Set<string> }> {
+        const retirable = new Set<string>();
+        const ownerGone = new Set<string>();
+        const hasForeignOwner = streamFiles.some(f => f.ownerId && f.ownerId !== writerId && f.ownerId !== MERGE_OUTPUT_OWNER);
+        const live = hasForeignOwner && await queryLiveWriters(this.name, bulkDatabase2Timing.liveWriterProbeMs) || undefined;
+        for (const f of streamFiles) {
+            if (f.ownerId === MERGE_OUTPUT_OWNER) { retirable.add(f.fileName); continue; }
+            // streamFileName is the only "we will append here again" signal - once it moves on, getStreamFileName opens a fresh file and this one is final.
+            if (f.ownerId === writerId) {
+                if (assumeSealed || f.fileName !== this.streamFileName) retirable.add(f.fileName);
+                continue;
+            }
+            if (!f.ownerId || live && !live.has(f.ownerId)) {
+                retirable.add(f.fileName);
+                ownerGone.add(f.fileName);
+            }
+        }
+        return { retirable, ownerGone };
     }
 
     private async automaticCompactionAllowed(): Promise<boolean> {
@@ -356,7 +443,6 @@ export class BulkDatabaseBase<T extends { key: string }> {
         });
         const oldIndex = this.reader.index;
         this.reader.setIndex(newIndex, { dropStaleFallback: this.rebuildOptions.dropStaleFallback });
-        this.streamRowsOnDisk = newIndex.streamRowsOnDisk;
         this.streamBytesOnDisk = newIndex.streamBytesOnDisk;
         if (oldIndex) {
             for (const f of oldIndex.fileSet) {
@@ -437,7 +523,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
             for (const { time, row } of stamped) this.reader.applyWrite(row.key as string, row, time);
         });
         for (const { time, row } of stamped) syncBroadcast(this.name, { key: row.key as string, time, value: row });
-        await this.streamAppend(framed, stamped.length);
+        await this.streamAppend(framed);
         void this.maybeMerge({ onlyIfStreamHeavy: true });
     }
 
@@ -453,13 +539,13 @@ export class BulkDatabaseBase<T extends { key: string }> {
             for (const { time, key } of stamped) this.reader.applyDelete(key, time);
         });
         for (const { time, key } of stamped) syncBroadcast(this.name, { key, time, deleted: true });
-        await this.streamAppend(frameDeletes(stamped), stamped.length);
+        await this.streamAppend(frameDeletes(stamped));
         void this.maybeMerge({ onlyIfStreamHeavy: true });
     }
 
     // Coalesce stream appends on a ramping per-collection schedule (the browser rewrites the whole file per append). The first write after a lull flushes immediately so a single edit-then-close is saved at once; sustained writes ramp toward writeFlushMaxDelayMs.
-    private async streamAppend(framed: Buffer, rows: number): Promise<void> {
-        this.pendingAppends.push({ framed, rows });
+    private async streamAppend(framed: Buffer): Promise<void> {
+        this.pendingAppends.push(framed);
         const max = bulkDatabase2Timing.writeFlushMaxDelayMs;
         const now = Date.now();
         if (max <= 0 || this.currentFlushDelay <= 0 || now - this.lastWriteTime > max) {
@@ -490,7 +576,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
     private async doFlush(): Promise<void> {
         if (!this.pendingAppends.length) return;
         const batch = this.pendingAppends.slice();
-        const combined = Buffer.concat(batch.map(p => p.framed));
+        const combined = Buffer.concat(batch);
         const storage = await this.storage();
         const fileName = this.getStreamFileName();
         if (fileName !== this.currentStreamFileName) {
@@ -502,7 +588,6 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // New entries appended during the await are after `batch` — removing the front is exactly the flushed set.
         this.pendingAppends.splice(0, batch.length);
         this.streamBytesOnDisk += combined.length;
-        for (const p of batch) this.streamRowsOnDisk += p.rows;
         this.currentStreamFileBytes += combined.length;
         if (this.currentStreamFileBytes >= bulkDatabase2Timing.streamFileMaxBytes) {
             this.streamFileName = undefined;
@@ -519,7 +604,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
             if (info && Date.now() - info.timestamp >= bulkDatabase2Timing.streamSealAgeMs) this.streamFileName = undefined;
         }
         if (!this.streamFileName) {
-            this.streamFileName = `stream_${Date.now()}_${Math.random().toString(36).slice(2, 10)}${STREAM_EXTENSION}`;
+            this.streamFileName = newStreamFileName(writerId);
         }
         return this.streamFileName;
     }
@@ -772,9 +857,18 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const ordered = orderStreamEntries(streamData.entries);
         const streamReader = ordered.length ? streamReaderFromEntries(ordered, 0).reader : undefined;
 
+        // An abandoned stream that yielded no entries (zero bytes, or nothing but torn bytes) holds no data and has no writer left. Retire it here: the merge below never lists it as a used source, so the normal retirement path would skip it and it would re-trigger the abandoned-stream fold on every pass. replacedBy is empty because nothing supersedes it - the marker hides it from reads immediately and processMarkers deletes it once aged.
+        const { retirable: retirableStreams } = await this.findAbandonedStreams(streamFiles, false);
+        const contributingStreams = new Set(streamData.entries.map(e => e.fileName));
+        const emptyAbandoned = streamFiles.filter(f => retirableStreams.has(f.fileName) && !contributingStreams.has(f.fileName)).map(f => f.fileName);
+        if (emptyAbandoned.length) await writeDeleteMarker(storage, { deleteFiles: emptyAbandoned, replacedBy: [] });
+
         const readers = streamReader ? [streamReader, ...bulkReaders] : bulkReaders;
         const readerNames = streamReader ? ["(streams)", ...consumedBulk.map(f => f.fileName)] : consumedBulk.map(f => f.fileName);
-        if (!readers.length) return false;
+        if (!readers.length) {
+            if (emptyAbandoned.length) await this.triggerRebuild();
+            return emptyAbandoned.length > 0;
+        }
 
         const inputs = [
             ...await Promise.all(consumedBulk.map(async f => ({ name: f.fileName, size: (await storage.getInfo(f.fileName).catch(() => undefined))?.size ?? 0 }))),
@@ -816,7 +910,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const carriedDeletes = includesOldest ? 0 : mergeResult.carriedDeletes.size;
         const outNames = [...newNames];
         if (carriedDeletes) {
-            const carryName = `stream_${Date.now()}_${Math.random().toString(36).slice(2, 10)}${STREAM_EXTENSION}`;
+            const carryName = newStreamFileName(MERGE_OUTPUT_OWNER);
             await storage.set(carryName, frameDeletes([...mergeResult.carriedDeletes].map(([key, time]) => ({ time, key }))));
             outNames.push(carryName);
         }
@@ -835,6 +929,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // A stream may still be appended to after we read it (the writer hasn't sealed/moved on); only retire streams canDeleteStream clears — the rest stay live and a later merge re-folds them once sealed. Bulk files are immutable, so a used one is always safe to retire.
         const deletableStreams: string[] = [];
         for (const f of usedStreamFiles) {
+            if (retirableStreams.has(f.fileName)) { deletableStreams.push(f.fileName); continue; }
             if (await this.canDeleteStream(f, Date.now(), streamData.sizes, forceDeleteStreams)) deletableStreams.push(f.fileName);
         }
         const deleteFiles = [...usedConsumedBulk.map(f => f.fileName), ...deletableStreams];
@@ -871,118 +966,30 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return true;
     }
 
-    // The merge policy (two passes, spaced by mergeSpacingMs):
-    //   1) Consolidate recent fragmentation: take newest files up to ~FIRST_MERGE_BYTES; merge them into one when they fragment or span too wide a time range.
-    //   2) Key-stratify: walk all keys in ~KEY_GROUP_BYTES groups; rewrite groups whose duplicate-key fraction passes DUP_THRESHOLD, highest first. Use tryMerge instead
-    private async testMergeINTERNAL_DO_NOT_CALL(): Promise<boolean> {
-        let merged = false;
-        await this.flushPending();
-        const runMerge = async (bulk: BulkFileInfo[], stream: StreamFileInfo[]): Promise<boolean> => {
-            if (merged && !await this.mergeSpacingDelay()) return false;
-            if (await this.mergeFileSet(bulk, stream)) merged = true;
-            return true;
-        };
-
-        // Hard stream limit: a stream this big makes every read pull a huge file → fold ALL of it now, force-delete (canDeleteStream still requires size-stable, so an active writer never loses data).
-        {
-            const { streamFiles } = await this.listFiles();
-            if (streamFiles.length) {
-                const storage = await this.storage();
-                const sizes = await Promise.all(streamFiles.map(async f => { try { return (await storage.getInfo(f.fileName))?.size ?? 0; } catch { return 0; } }));
-                const totalStreamBytes = sizes.reduce((a, b) => a + b, 0);
-                if (totalStreamBytes > bulkDatabase2Timing.streamFoldHardLimitBytes) {
-                    console.log(`${blue(this.name)} ${magenta("fold")} stream tier ${fmtBytes(totalStreamBytes)} over hard limit ${fmtBytes(bulkDatabase2Timing.streamFoldHardLimitBytes)} - folding all streams now`);
-                    if (await this.mergeFileSet([], streamFiles, false, true)) merged = true;
-                }
-            }
-        }
-
-        // Pass 1: consolidate recent. Only seal when cross-tab sync can fold recent streams — in Node canDeleteStream needs them aged anyway, so sealing would just fragment streams every pass.
-        const foldRecentStreams = isSyncSupported();
-        if (foldRecentStreams) {
-            syncBroadcastSeal(this.name);
-            this.streamFileName = undefined;
-        }
-        {
-            const { bulkFiles, streamFiles } = await this.listFiles();
-            const bulkMeta = await Promise.all(bulkFiles.map(async f => {
-                const [size, header] = await Promise.all([this.fileLogicalSize(f.fileName), this.readBulkHeader(f.fileName)]);
-                return { kind: "bulk" as const, file: f, bytes: size ?? 0, time: header?.maxTime || f.timestamp };
-            }));
-            const streamMeta: { kind: "stream"; file: StreamFileInfo; bytes: number; time: number }[] = [];
-            for (const f of streamFiles) {
-                const aged = Date.now() - f.timestamp >= bulkDatabase2Timing.streamSealAgeMs;
-                if (!foldRecentStreams && !aged) continue;
-                let bytes = 0;
-                try { const info = await (await this.storage()).getInfo(f.fileName); bytes = info?.size ?? 0; } catch { bytes = 0; }
-                streamMeta.push({ kind: "stream", file: f, bytes, time: f.timestamp });
-            }
-            const items = [...bulkMeta, ...streamMeta].sort((a, b) => b.time - a.time);
-            const recent: typeof items = [];
-            let recentBytes = 0;
-            for (const it of items) {
-                recent.push(it);
-                recentBytes += it.bytes;
-                if (recentBytes >= FIRST_MERGE_BYTES) break;
-            }
-            const span = recent.length ? recent[0].time - recent[recent.length - 1].time : 0;
-            const recentStreamBytes = recent.reduce((a, it) => a + (it.kind === "stream" ? it.bytes : 0), 0);
-            const heavyStream = recentStreamBytes > bulkDatabase2Timing.streamFoldTriggerBytes;
-            const triggered =
-                recent.length >= 2 && (recent.length > bulkDatabase2Timing.firstMergeTriggerFiles || span > bulkDatabase2Timing.firstMergeTriggerRangeMs)
-                || heavyStream;
-            if (triggered) {
-                const rb = recent.filter(i => i.kind === "bulk").map(i => (i.file as BulkFileInfo));
-                const rs = recent.filter(i => i.kind === "stream").map(i => (i.file as StreamFileInfo));
-                if (!await runMerge(rb, rs)) return merged;
-            }
-        }
-
-        // Whole-tier dedup short-circuit (between Pass 1 and the per-group Pass 2): when the bulk tier is big enough AND mostly duplicates, fold every bulk file in one merge. Avoids the per-group walk's 5-min spacing × N-groups latency.
-        {
-            const { bulkFiles } = await this.listFiles();
-            if (bulkFiles.length >= 2) {
-                const storage = await this.storage();
-                let totalBytes = 0;
-                let totalSlots = 0;
-                const uniqueKeys = new Set<string>();
-                for (const f of bulkFiles) {
-                    try {
-                        const reader = await loadFileReader(this.name, storage, f, this.subCaches.bulk);
-                        totalBytes += reader.totalBytes;
-                        totalSlots += reader.keys.length;
-                        for (const k of reader.keys) uniqueKeys.add(k);
-                    } catch { /* skip unreadable */ }
-                }
-                const dupFraction = totalSlots ? (totalSlots - uniqueKeys.size) / totalSlots : 0;
-                if (totalBytes > DEDUP_TRIGGER_BYTES && dupFraction > DEDUP_TRIGGER_FRACTION) {
-                    console.log(`${blue(this.name)} ${magenta("dedup")}: ${fmtBytes(totalBytes)} across ${bulkFiles.length} bulk file(s), ${Math.round(dupFraction * 100)}% duplicate key-slots - folding all at once`);
-                    if (!await runMerge(bulkFiles, [])) return merged;
-                    return merged;
-                }
+    // Splits the bulk tier the way the merge writer does. A merge cuts an output file once it reaches TARGET_FILE_BYTES, so only its LAST output can be under that — meaning a sub-target file is either a single stream fold or the tail of an earlier merge, and is still waiting to be rolled up ("loose"). Anything at or over the target is done growing ("combined") and only phase 3 touches it again.
+    //
+    // A file whose size won't read is reported as combined: phase 2 can't consume one (its reader won't load, so the merge won't retire it), and calling it loose would re-trigger phase 2 on every pass until handleUnreadableFile finally deletes it.
+    private async splitBulkTier(bulkFiles: BulkFileInfo[]): Promise<{ loose: BulkFileInfo[]; looseBytes: number; combined: BulkFileInfo[]; sizes: Map<string, number> }> {
+        const logicalSizes = await Promise.all(bulkFiles.map(f => this.fileLogicalSize(f.fileName)));
+        const loose: BulkFileInfo[] = [];
+        const combined: BulkFileInfo[] = [];
+        const sizes = new Map<string, number>();
+        let looseBytes = 0;
+        for (let i = 0; i < bulkFiles.length; i++) {
+            const bytes = logicalSizes[i];
+            sizes.set(bulkFiles[i].fileName, bytes ?? 0);
+            if (bytes === undefined || bytes >= TARGET_FILE_BYTES) {
+                combined.push(bulkFiles[i]);
+                continue;
             }
+            loose.push(bulkFiles[i]);
+            looseBytes += bytes;
         }
-
-        // Pass 2: key-stratified deduplication. Disjoint key ranges → one group's merge doesn't change another's duplication; re-select each group's files at merge time (set shifts).
-        const groups = await this.findDuplicateGroups();
-        for (const g of groups) {
-            const { bulkFiles } = await this.listFiles();
-            const headers = await Promise.all(bulkFiles.map(f => this.readBulkHeader(f.fileName)));
-            const groupFiles = bulkFiles.filter((f, i) => {
-                const h = headers[i];
-                if (!h) return false;
-                if (h.minKey === undefined || h.maxKey === undefined) return true;
-                return h.minKey <= g.hi && h.maxKey >= g.lo;
-            });
-            if (groupFiles.length >= 2) { if (!await runMerge(groupFiles, [])) return merged; }
-        }
-
-        return merged;
+        return { loose, looseBytes, combined, sizes };
     }
 
-    private async findDuplicateGroups(): Promise<{ lo: string; hi: string; dup: number }[]> {
-        const { bulkFiles } = await this.listFiles();
-        if (bulkFiles.length < 2) return [];
+    // One walk of the combined tier's key lists, producing both of phase 3's inputs: the whole-tier duplicate fraction, and the per-key-range groups. Combined into one pass because walking every file's keys is the most expensive thing a merge pass does that isn't a merge, and both answers come from the same key counts.
+    private async analyzeDuplicates(bulkFiles: BulkFileInfo[]): Promise<{ dupFraction: number; groups: { lo: string; hi: string; dup: number }[] }> {
         const storage = await this.storage();
         const infos = await Promise.all(bulkFiles.map(async f => {
             try {
@@ -998,7 +1005,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
             totalBytes += i.bytes;
             for (const k of i.keys) { keyCount.set(k, (keyCount.get(k) || 0) + 1); totalSlots++; }
         }
-        if (totalSlots === 0) return [];
+        if (!totalSlots) return { dupFraction: 0, groups: [] };
         const bytesPerSlot = totalBytes / totalSlots;
         const sortedKeys = [...keyCount.keys()].sort();
         const groups: { lo: string; hi: string; dup: number }[] = [];
@@ -1007,13 +1014,157 @@ export class BulkDatabaseBase<T extends { key: string }> {
             const c = keyCount.get(sortedKeys[i]) ?? 0;
             gBytes += c * bytesPerSlot; gSlots += c; gUnique += 1;
             if (gBytes >= KEY_GROUP_BYTES || i === sortedKeys.length - 1) {
-                const dup = (gSlots - gUnique) / gSlots;
-                if (dup > DUP_THRESHOLD) groups.push({ lo: sortedKeys[gStart], hi: sortedKeys[i], dup });
+                groups.push({ lo: sortedKeys[gStart], hi: sortedKeys[i], dup: (gSlots - gUnique) / gSlots });
                 gStart = i + 1; gBytes = 0; gSlots = 0; gUnique = 0;
             }
         }
-        groups.sort((a, b) => b.dup - a.dup);
-        return groups;
+        sort(groups, g => -g.dup);
+        return { dupFraction: (totalSlots - keyCount.size) / totalSlots, groups };
+    }
+
+    // Which files a key-group step rewrites. Re-run at merge time as well as at plan time: groups are disjoint by KEY range, not by file, so an earlier step in the same pass can consume a file this range still lists.
+    private async filesForKeyRange(bulkFiles: BulkFileInfo[], keyRange: { lo: string; hi: string }): Promise<BulkFileInfo[]> {
+        const headers = await Promise.all(bulkFiles.map(f => this.readBulkHeader(f.fileName)));
+        return bulkFiles.filter((f, i) => {
+            const h = headers[i];
+            if (!h) return false;
+            if (h.minKey === undefined || h.maxKey === undefined) return true;
+            return h.minKey <= keyRange.hi && h.maxKey >= keyRange.lo;
+        });
+    }
+
+    /**
+     * Every compaction the files on disk currently call for, without performing any of them. A merge pass
+     * builds exactly this and then runs the steps whose `ready` is true, so the plan is precisely what the
+     * database is about to do — and a step that isn't ready still reports its `triggers`, so a caller can
+     * see how close it is (50MB of stream data out of the 64MB that would fold it, and so on).
+     *
+     * O(total keys): the phase 3 steps need every combined file's key list walked.
+     */
+    public async planCompaction(): Promise<CompactionPlan> {
+        const time = Date.now();
+        const steps: CompactionStep[] = [];
+        const storage = await this.storage();
+        const { bulkFiles, streamFiles } = await this.listFiles();
+
+        const streamSizes = new Map<string, number>();
+        await Promise.all(streamFiles.map(async f => {
+            try { streamSizes.set(f.fileName, (await storage.getInfo(f.fileName))?.size ?? 0); } catch { streamSizes.set(f.fileName, 0); }
+        }));
+        const streamBytes = (files: StreamFileInfo[]) => files.reduce((a, f) => a + (streamSizes.get(f.fileName) ?? 0), 0);
+
+        // ── Phase 1: stream -> bulk ──────────────────────────────────────────────────────────────────
+        // The ENTIRE stream tier is parsed into memory and held there (subCaches.stream keeps every decoded entry) just to build the index, so it is the biggest single lever on our heap - a fold turns it into a bulk file we only read an index of. Folds streams and nothing else: no bulk file is dragged in, so the work is proportional to the memory reclaimed.
+        const hardLimit = makeTrigger({ name: "streamBytes", value: streamBytes(streamFiles), threshold: bulkDatabase2Timing.streamFoldHardLimitBytes, unit: "bytes" });
+        // Past the hard limit every read pulls an enormous file, so fold the whole tier regardless of who owns what. mergeFileSet force-deletes for this one; canDeleteStream still requires size-stability, so an active writer never loses data.
+        steps.push({
+            phase: 1, kind: "streamHardLimit", requires: "all", triggers: [hardLimit],
+            ready: hardLimit.met && streamFiles.length > 0,
+            bulkFiles: [], streamFiles, bytes: hardLimit.value,
+        });
+
+        // A pass seals before it merges, so by then our own current file is final too - predict that rather than reporting it as still-open.
+        const { retirable, ownerGone } = await this.findAbandonedStreams(streamFiles, isSyncSupported());
+        // Only fold what we can also retire. Folding a stream a live foreign owner may still append to would copy it into bulk without removing it, so the bytes would stay in memory and just get re-folded next pass; that owner rolls its own file over at streamFileMaxBytes instead.
+        // Aged past streamSealAgeMs counts as retirable too (canDeleteStream's own first rule): no writer appends past the seal age, and this is the only thing that frees the tier in Node, where liveness cannot be probed at all.
+        const foldable = streamFiles.filter(f => f.ownerId !== MERGE_OUTPUT_OWNER
+            && (retirable.has(f.fileName) || time - f.timestamp >= bulkDatabase2Timing.streamSealAgeMs));
+        // Merge-carry files hold nothing but tombstones and are never a REASON to fold - folding one alone would just rewrite it into another carry file, forever. They ride along whenever something else folds, which collapses however many have piled up into one.
+        const carry = streamFiles.filter(f => f.ownerId === MERGE_OUTPUT_OWNER);
+        // A stream whose writer is gone will never grow and nothing else will ever fold it, so fold it at any size - even one row. Zero-byte ones don't count: they hold no data to fold, and mergeFileSetInner retires them directly.
+        const abandoned = foldable.filter(f => ownerGone.has(f.fileName) && streamSizes.get(f.fileName));
+        const foldTriggers = [
+            makeTrigger({ name: "foldableBytes", value: streamBytes(foldable), threshold: bulkDatabase2Timing.streamFoldTriggerBytes, unit: "bytes" }),
+            makeTrigger({ name: "abandonedFiles", value: abandoned.length, threshold: 1, unit: "count" }),
+        ];
+        steps.push({
+            phase: 1, kind: "streamFold", requires: "any", triggers: foldTriggers,
+            // Skipped when the hard limit already folds everything this would have.
+            ready: !hardLimit.met && foldTriggers.some(t => t.met),
+            bulkFiles: [], streamFiles: [...foldable, ...carry], bytes: streamBytes([...foldable, ...carry]),
+        });
+
+        // ── Phase 2: loose bulk -> combined bulk ─────────────────────────────────────────────────────
+        // Phase 1 emits one small bulk file per fold. Each is cheap to read (index only) but holds its whole key list in memory and joins into every read, so they have to be rolled up. Merging just the loose ones also dedupes them for free - a rewrite-heavy workload collapses a gigabyte of near-identical folds into almost nothing - and it always terminates, because a merge cuts every output but the last at TARGET_FILE_BYTES, so one pass can leave at most one loose file behind.
+        const { loose, looseBytes, combined, sizes } = await this.splitBulkTier(bulkFiles);
+        const looseTriggers = [
+            makeTrigger({ name: "looseBytes", value: looseBytes, threshold: bulkDatabase2Timing.looseBulkTriggerBytes, unit: "bytes" }),
+            makeTrigger({ name: "looseFiles", value: loose.length, threshold: bulkDatabase2Timing.looseBulkTriggerFiles, unit: "count" }),
+        ];
+        steps.push({
+            phase: 2, kind: "looseCombine", requires: "any", triggers: looseTriggers,
+            // Under two files there is nothing to combine, and rewriting one file into an identical one would re-trigger forever.
+            ready: loose.length >= 2 && looseTriggers.some(t => t.met),
+            bulkFiles: loose, streamFiles: [], bytes: looseBytes,
+        });
+
+        // ── Phase 3: dedup the combined files ────────────────────────────────────────────────────────
+        // Loose files are excluded throughout: phase 2 rewrites them anyway, and that rewrite already dedupes them, so pulling one in here would do the same work twice at key-group scale.
+        const combinedBytes = combined.reduce((a, f) => a + (sizes.get(f.fileName) ?? 0), 0);
+        const { dupFraction, groups } = await this.analyzeDuplicates(combined);
+        // Whole-tier short-circuit: when the combined tier is big enough AND mostly duplicates, fold all of it in one merge rather than paying the per-group walk's 5-min spacing x N groups.
+        const dedupAllTriggers = [
+            makeTrigger({ name: "combinedBytes", value: combinedBytes, threshold: DEDUP_TRIGGER_BYTES, unit: "bytes" }),
+            makeTrigger({ name: "duplicateFraction", value: dupFraction, threshold: DEDUP_TRIGGER_FRACTION, unit: "fraction" }),
+        ];
+        const dedupAllReady = combined.length >= 2 && dedupAllTriggers.every(t => t.met);
+        steps.push({
+            phase: 3, kind: "dedupAll", requires: "all", triggers: dedupAllTriggers,
+            ready: dedupAllReady,
+            bulkFiles: combined, streamFiles: [], bytes: combinedBytes,
+        });
+
+        // Then key-stratified: disjoint key ranges, so one group's merge doesn't change another's duplication. Only groups over the threshold are worth a step; the best one below it is listed anyway so a caller can see how close the tier is.
+        const groupSteps = groups.filter(g => g.dup >= DUP_THRESHOLD);
+        if (!groupSteps.length && groups.length) groupSteps.push(groups[0]);
+        for (const g of groupSteps) {
+            const groupFiles = await this.filesForKeyRange(combined, g);
+            const dup = makeTrigger({ name: "duplicateFraction", value: g.dup, threshold: DUP_THRESHOLD, unit: "fraction" });
+            steps.push({
+                phase: 3, kind: "dedupKeyGroup", requires: "all", triggers: [dup],
+                // Moot if the whole tier is about to be folded in one go.
+                ready: !dedupAllReady && dup.met && groupFiles.length >= 2,
+                bulkFiles: groupFiles, streamFiles: [], bytes: groupFiles.reduce((a, f) => a + (sizes.get(f.fileName) ?? 0), 0),
+                keyRange: { lo: g.lo, hi: g.hi },
+            });
+        }
+
+        // Merges are spaced mergeSpacingMs apart, and the first one runs immediately.
+        let readyCount = 0;
+        for (const step of steps) {
+            if (!step.ready) continue;
+            step.startTime = time + readyCount * bulkDatabase2Timing.mergeSpacingMs;
+            readyCount++;
+        }
+        return { collection: this.name, time, steps };
+    }
+
+    // Runs the compaction plan: builds it, then performs every step it marks ready, in order. All the deciding lives in planCompaction - this only executes, so what the database does and what planCompaction reports can never drift apart.
+    private async testMergeINTERNAL_DO_NOT_CALL(): Promise<boolean> {
+        let merged = false;
+        await this.flushPending();
+
+        // Seal before planning, so every writer's current stream file is final and the plan can count it as foldable. Only when cross-tab sync is there to carry the seal - in Node canDeleteStream needs streams aged anyway, so sealing would just fragment them every pass.
+        if (isSyncSupported()) {
+            syncBroadcastSeal(this.name);
+            this.streamFileName = undefined;
+        }
+
+        const plan = await this.planCompaction();
+        for (const step of plan.steps) {
+            if (!step.ready) continue;
+            let bulkFiles = step.bulkFiles;
+            if (step.keyRange) {
+                bulkFiles = await this.filesForKeyRange(bulkFiles, step.keyRange);
+                if (bulkFiles.length < 2) continue;
+            }
+            const why = step.triggers.filter(t => t.met)
+                .map(t => `${t.name} ${fmtTriggerValue(t.value, t.unit)} of ${fmtTriggerValue(t.threshold, t.unit)}`).join(", ");
+            console.log(`${blue(this.name)} ${magenta(step.kind)} phase ${step.phase}: ${bulkFiles.length} bulk + ${step.streamFiles.length} stream file(s), ${fmtBytes(step.bytes)} - ${why}`);
+            if (merged && !await this.mergeSpacingDelay()) return merged;
+            if (await this.mergeFileSet(bulkFiles, step.streamFiles, false, step.kind === "streamHardLimit")) merged = true;
+        }
+        return merged;
     }
 
     // ── reads — forwarded to BulkDatabaseReader, with rebuild-on-missing retry ───────────────────────

package/storage/BulkDatabase2/LoadedIndex.d.ts

@@ -10,6 +10,7 @@ export type BulkFileInfo = {
 export type StreamFileInfo = {
     fileName: string;
     timestamp: number;
+    ownerId?: string;
 };
 export type StreamReaderCacheEntry = {
     readSize: number;

package/storage/BulkDatabase2/LoadedIndex.ts

@@ -9,7 +9,8 @@ import { blue, red } from "socket-function/src/formatting/logColors";
 const BULK_ROOT_FOLDER = "bulkDatabases2";
 
 export type BulkFileInfo = { fileName: string; level: number; timestamp: number };
-export type StreamFileInfo = { fileName: string; timestamp: number };
+// ownerId is the writer that may still be appending to this file; undefined for legacy names that predate the stamp (nothing appends to those anymore).
+export type StreamFileInfo = { fileName: string; timestamp: number; ownerId?: string };
 export type StreamReaderCacheEntry = { readSize: number; parsedPos: number; entries: StreamEntry[] };
 
 export class MissingFileError extends Error { }

package/storage/BulkDatabase2/blockCache.ts

@@ -11,7 +11,12 @@ import { LZ4 } from "socket-function/src/lz4/LZ4";
 // Per-block compression mainly helps slow storage (HDD): fewer bytes off disk per block, at the cost of a fast in-memory LZ4 decompress.
 
 const BLOCK_SIZE = 256 * 1024;
-const MAX_BLOCKS = Math.floor((512 * 1024 * 1024) / BLOCK_SIZE);
+// 128MB per context. This cache exists per browsing context (each tab, each
+// worker), so a generous cap multiplies fast — at 512MB, three same-process
+// tabs could pin 1.5GB of block data, and the resulting GC pressure visibly
+// janks video playback. 128MB still holds ~500 hot blocks per context; misses
+// just re-read from local disk, which is cheap.
+const MAX_BLOCKS = Math.floor((128 * 1024 * 1024) / BLOCK_SIZE);
 const MIN_COMPRESSION_RATIO = 2;
 
 const EMPTY = Buffer.alloc(0) as Buffer;