sliftutils 1.4.66 → 1.4.68

package/index.d.ts

@@ -853,6 +853,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
         isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
         /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
         isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+        /**
+         * Reactive: true while a merge is rewriting this collection's files (background `maybeMerge` or
+         * an explicit `compact`/`merge`/`tryMergeNow`). Becomes false as soon as the new index is swapped
+         * in — the deferred-delete cleanup window is NOT counted. Use this in a UI to show a per-database
+         * "compacting…" indicator.
+         */
+        isCompactingSync(): boolean;
         /**
          * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
          * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;
@@ -926,7 +933,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
     export declare const BULK_ROOT_FOLDER = "bulkDatabases2";
     export declare const bulkDatabase2Timing: {
         streamSealAgeMs: number;
-        mergeCheckIntervalMs: number;
+        visibleMergeIntervalMs: number;
         mergeSpacingMs: number;
         firstMergeTriggerFiles: number;
         firstMergeTriggerRangeMs: number;
@@ -960,6 +967,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private storageFactory;
         private config;
         constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+        private setupVisibilityMergeCheck;
         private reader;
         private subCaches;
         private pendingAppends;
@@ -970,7 +978,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private streamFileName;
         private currentStreamFileName;
         private currentStreamFileBytes;
-        private lastMergeCheck;
+        private mergeInFlight;
         private streamRowsOnDisk;
         private streamBytesOnDisk;
         private fileSetPollTimer;
@@ -1030,6 +1038,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private fileLogicalSize;
         private handleUnreadableFile;
         private mergeFileSet;
+        private mergeFileSetInner;
         private canDeleteStream;
         private mergeSpacingDelay;
         private testMerge;
@@ -1059,6 +1068,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         }[] | undefined;
         isFieldLoadedSync<C extends keyof T>(key: string, column: C): boolean;
         isColumnLoadedSync<C extends keyof T>(column: C): boolean;
+        isCompactingSync(): boolean;
         getColumnInfo(): Promise<{
             column: string;
             byteSize: number;
@@ -1240,6 +1250,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseMerge" {
     }): Promise<{
         outputs: PlannedMergeOutput[];
         carriedDeletes: Map<string, number>;
+        usedSourceNames: Set<string>;
     }>;
     export {};
 
@@ -1279,6 +1290,10 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseReader" {
         isKeyWatched(key: string): boolean;
         isLiveNow(key: string): boolean;
         localTime(key: string): number;
+        private compactingCount;
+        beginCompaction(): void;
+        endCompaction(): void;
+        isCompactingSync(): boolean;
         private notifyOverlayMutation;
         getKeys(): Promise<string[]>;
         getColumn<C extends keyof T>(column: C): Promise<{
@@ -1514,8 +1529,12 @@ declare module "sliftutils/storage/BulkDatabase2/blockCache" {
 }
 
 declare module "sliftutils/storage/BulkDatabase2/mergeLock" {
+    import type { FileStorage } from "../FileFolderAPI";
     export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
     export declare function releaseMergeLock(collection: string, holderId: string): void;
+    export declare function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean>;
+    export declare function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void;
+    export declare function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void>;
 
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.66",
+    "version": "1.4.68",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -88,6 +88,13 @@ export interface IBulkDatabase2<T extends {
     isFieldLoadedSync<Column extends keyof T>(key: string, column: Column): boolean;
     /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
     isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
+    /**
+     * Reactive: true while a merge is rewriting this collection's files (background `maybeMerge` or
+     * an explicit `compact`/`merge`/`tryMergeNow`). Becomes false as soon as the new index is swapped
+     * in — the deferred-delete cleanup window is NOT counted. Use this in a UI to show a per-database
+     * "compacting…" indicator.
+     */
+    isCompactingSync(): boolean;
     /**
      * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
      * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -74,6 +74,14 @@ export interface IBulkDatabase2<T extends { key: string }> {
     /** Reactive: whether a whole column is loaded yet (see isFieldLoadedSync). */
     isColumnLoadedSync<Column extends keyof T>(column: Column): boolean;
 
+    /**
+     * Reactive: true while a merge is rewriting this collection's files (background `maybeMerge` or
+     * an explicit `compact`/`merge`/`tryMergeNow`). Becomes false as soon as the new index is swapped
+     * in — the deferred-delete cleanup window is NOT counted. Use this in a UI to show a per-database
+     * "compacting…" indicator.
+     */
+    isCompactingSync(): boolean;
+
     /**
      * Whether a row (key) is currently being watched by some reactive observer (getSingleFieldObjSync /
      * getSingleFieldSync). Lets callers skip per-row work when nothing's watching. Non-reactive query;

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -2,7 +2,7 @@ import type { FileStorage } from "../FileFolderAPI";
 export declare const BULK_ROOT_FOLDER = "bulkDatabases2";
 export declare const bulkDatabase2Timing: {
     streamSealAgeMs: number;
-    mergeCheckIntervalMs: number;
+    visibleMergeIntervalMs: number;
     mergeSpacingMs: number;
     firstMergeTriggerFiles: number;
     firstMergeTriggerRangeMs: number;
@@ -36,6 +36,7 @@ export declare class BulkDatabaseBase<T extends {
     private storageFactory;
     private config;
     constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+    private setupVisibilityMergeCheck;
     private reader;
     private subCaches;
     private pendingAppends;
@@ -46,7 +47,7 @@ export declare class BulkDatabaseBase<T extends {
     private streamFileName;
     private currentStreamFileName;
     private currentStreamFileBytes;
-    private lastMergeCheck;
+    private mergeInFlight;
     private streamRowsOnDisk;
     private streamBytesOnDisk;
     private fileSetPollTimer;
@@ -106,6 +107,7 @@ export declare class BulkDatabaseBase<T extends {
     private fileLogicalSize;
     private handleUnreadableFile;
     private mergeFileSet;
+    private mergeFileSetInner;
     private canDeleteStream;
     private mergeSpacingDelay;
     private testMerge;
@@ -135,6 +137,7 @@ export declare class BulkDatabaseBase<T extends {
     }[] | undefined;
     isFieldLoadedSync<C extends keyof T>(key: string, column: C): boolean;
     isColumnLoadedSync<C extends keyof T>(column: C): boolean;
+    isCompactingSync(): boolean;
     getColumnInfo(): Promise<{
         column: string;
         byteSize: number;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -1,4 +1,5 @@
 import { sort } from "socket-function/src/misc";
+import { runInSerial } from "socket-function/src/batching";
 import { getTimeUnique } from "socket-function/src/bits";
 import { lazy } from "socket-function/src/caching";
 import { isNode } from "typesafecss";
@@ -17,7 +18,7 @@ import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { STREAM_EXTENSION, frameDeletes, frameRows, streamReaderFromEntries } from "./streamLog";
 import { broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, connect as syncConnect, isSyncSupported, RemoteWrite } from "./syncClient";
 import { DELETED } from "./WriteOverlay";
-import { releaseMergeLock, tryAcquireMergeLock } from "./mergeLock";
+import { releaseMergeFileLock, releaseMergeLock, startMergeFileLockHeartbeat, tryAcquireMergeFileLock, tryAcquireMergeLock } from "./mergeLock";
 import {
     BulkFileInfo,
     LoadedIndex,
@@ -41,11 +42,19 @@ const MAX_INDEX_RELOAD_ATTEMPTS = 3;
 const FIRST_MERGE_BYTES = TARGET_FILE_BYTES / 2;
 const KEY_GROUP_BYTES = 800 * 1024 * 1024;
 const DUP_THRESHOLD = 0.4;
+// Emergency Pass 2 short-circuit: once Pass 1 has had a chance to fold streams, if the bulk tier is
+// big enough AND the overall key duplication fraction is high, do ONE merge across every bulk file
+// instead of the per-key-group walk (which spaces merges 5 min apart — 16 hours for 200 groups).
+const EMERGENCY_DEDUP_BYTES = 512 * 1024 * 1024;
+const EMERGENCY_DEDUP_FRACTION = 0.5;
 const WRITE_FLUSH_FIRST_STEP_MS = 250;
 
 export const bulkDatabase2Timing = {
     streamSealAgeMs: 10 * 60 * 60 * 1000,
-    mergeCheckIntervalMs: 30 * 60 * 1000,
+    // Wait this long after the tab becomes visible before the first merge check, then check this often
+    // afterwards. Tab being hidden cancels the timers (no merges while in background). Browser-only;
+    // Node compactors (e.g. remoteFileServer) drive their own polling.
+    visibleMergeIntervalMs: 5 * 60 * 1000,
     mergeSpacingMs: 5 * 60 * 1000,
     firstMergeTriggerFiles: 20,
     firstMergeTriggerRangeMs: 3 * 24 * 60 * 60 * 1000,
@@ -164,10 +173,43 @@ export class BulkDatabaseBase<T extends { key: string }> {
         }
         this.fileSetPollTimer = setInterval(() => void this.pollFileSet(), bulkDatabase2Timing.fileSetPollIntervalMs);
         (this.fileSetPollTimer as { unref?: () => void }).unref?.();
+        this.setupVisibilityMergeCheck();
         BulkDatabaseBase.liveInstances.add(this);
         BulkDatabaseBase.startMemoryWatchdog();
     }
 
+    // Every `visibleMergeIntervalMs` of being-visible time, run a merge check. First check fires
+    // `visibleMergeIntervalMs` after the tab becomes visible (not on initial load — gives the user a
+    // moment of session commitment before we start churning the FS), then every interval after. Tab
+    // hiding cancels the timers; visible-again restarts. Node has no document — there we consider
+    // ourselves always-visible and just install the interval directly.
+    private setupVisibilityMergeCheck(): void {
+        let firstTimer: ReturnType<typeof setTimeout> | undefined;
+        let intervalTimer: ReturnType<typeof setInterval> | undefined;
+        const stop = () => {
+            if (firstTimer) { clearTimeout(firstTimer); firstTimer = undefined; }
+            if (intervalTimer) { clearInterval(intervalTimer); intervalTimer = undefined; }
+        };
+        const start = () => {
+            if (firstTimer || intervalTimer) return;
+            firstTimer = setTimeout(() => {
+                firstTimer = undefined;
+                void this.maybeMerge();
+                intervalTimer = setInterval(() => void this.maybeMerge(), bulkDatabase2Timing.visibleMergeIntervalMs);
+                (intervalTimer as { unref?: () => void }).unref?.();
+            }, bulkDatabase2Timing.visibleMergeIntervalMs);
+            (firstTimer as { unref?: () => void }).unref?.();
+        };
+        if (typeof document === "undefined") { start(); return; }
+        try {
+            document.addEventListener("visibilitychange", () => {
+                if (document.visibilityState === "visible") start();
+                else stop();
+            });
+            if (document.visibilityState === "visible") start();
+        } catch { /* not in a DOM context */ }
+    }
+
     private reader: BulkDatabaseReader<T>;
     private subCaches: SubReaderCaches = { bulk: new Map(), stream: new Map() };
 
@@ -180,7 +222,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
     private streamFileName: string | undefined;
     private currentStreamFileName: string | undefined;
     private currentStreamFileBytes = 0;
-    private lastMergeCheck = Date.now();
+    // In-process re-entry guard: if a merge is running, additional triggers (visibility timer, writes,
+    // tryMergeNow calls) return immediately instead of queueing. Keeps the visibility-timer firings
+    // from stacking behind a long merge.
+    private mergeInFlight = false;
     // Running counters of stream-tier rows + bytes on disk. Seeded from each LoadedIndex build, then
     // incremented per flush so the fold-trigger checks current data without an extra directory listing.
     private streamRowsOnDisk = 0;
@@ -380,7 +425,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         });
         for (const { time, row } of stamped) syncBroadcast(this.name, { key: row.key as string, time, value: row });
         await this.streamAppend(framed, stamped.length);
-        void this.maybeMerge();
+        void this.maybeMerge({ onlyIfStreamHeavy: true });
     }
 
     public async delete(key: string): Promise<void> {
@@ -396,7 +441,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         });
         for (const { time, key } of stamped) syncBroadcast(this.name, { key, time, deleted: true });
         await this.streamAppend(frameDeletes(stamped), stamped.length);
-        void this.maybeMerge();
+        void this.maybeMerge({ onlyIfStreamHeavy: true });
     }
 
     // Coalesce stream appends on a ramping per-collection schedule (the browser rewrites the whole
@@ -533,11 +578,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
     }
 
     // ── merge policy ─────────────────────────────────────────────────────────────────────────────────
-    private async maybeMerge(): Promise<void> {
+    // Called both periodically (visibility timer) and on writes (with the streamNeedsFold gate so a
+    // small write doesn't drag the merge through). The mergeInFlight check in tryMergeNow skips any
+    // call that arrives while a previous one is still running — so 5-min timer ticks don't pile up.
+    private async maybeMerge(opts: { onlyIfStreamHeavy?: boolean } = {}): Promise<void> {
         if (!await this.automaticCompactionAllowed()) return;
-        const now = Date.now();
-        if (!this.streamNeedsFold() && now - this.lastMergeCheck < bulkDatabase2Timing.mergeCheckIntervalMs) return;
-        this.lastMergeCheck = now;
+        if (opts.onlyIfStreamHeavy && !this.streamNeedsFold()) return;
         try {
             await this.tryMergeNow();
         } catch (e) {
@@ -545,17 +591,38 @@ export class BulkDatabaseBase<T extends { key: string }> {
         }
     }
 
+    // Returns immediately with { lockFailed: true } if another call is already in flight (same
+    // instance), if the localStorage lock is held by another tab, or if the cross-process file lock
+    // says another process owns it. Otherwise: acquires both locks, runs testMerge, releases.
     public async tryMergeNow(): Promise<{ merged: boolean; lockFailed: boolean }> {
+        if (this.mergeInFlight) return { merged: false, lockFailed: true };
         if (!tryAcquireMergeLock(this.name, writerId)) return { merged: false, lockFailed: true };
+        const storage = await this.storage();
+        const haveFileLock = await tryAcquireMergeFileLock(storage, writerId);
+        if (!haveFileLock) {
+            releaseMergeLock(this.name, writerId);
+            return { merged: false, lockFailed: true };
+        }
+        this.mergeInFlight = true;
+        const stopHeartbeat = startMergeFileLockHeartbeat(storage, writerId);
         try {
             return { merged: await this.testMerge(), lockFailed: false };
         } finally {
+            stopHeartbeat();
+            await releaseMergeFileLock(storage, writerId);
             releaseMergeLock(this.name, writerId);
+            this.mergeInFlight = false;
         }
     }
 
     public async compact(): Promise<void> {
+        if (this.mergeInFlight) return;
         if (!tryAcquireMergeLock(this.name, writerId)) return;
+        const storage = await this.storage();
+        const haveFileLock = await tryAcquireMergeFileLock(storage, writerId);
+        if (!haveFileLock) { releaseMergeLock(this.name, writerId); return; }
+        this.mergeInFlight = true;
+        const stopHeartbeat = startMergeFileLockHeartbeat(storage, writerId);
         try {
             await this.flushPending();
             syncBroadcastSeal(this.name);
@@ -565,7 +632,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
             // can be dropped (nothing left to suppress).
             if (bulkFiles.length + streamFiles.length >= 1) await this.mergeFileSet(bulkFiles, streamFiles, true);
         } finally {
+            stopHeartbeat();
+            await releaseMergeFileLock(storage, writerId);
             releaseMergeLock(this.name, writerId);
+            this.mergeInFlight = false;
         }
     }
 
@@ -631,10 +701,40 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // leaves duplicates (next merge dedupes) rather than a gap. After the file set changes on disk,
     // we trigger an index rebuild + atomic swap; once swap completes, the consumed files' block-cache
     // entries are evicted (no consumer can ask for them now).
-    private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false, forceDeleteStreams = false): Promise<boolean> {
+    //
+    // Serialized: foldOwnStream + merge(timeLo, timeHi) + testMerge's runMerge all hit this; the lock
+    // covers tryMergeNow/compact but those two paths bypass it. runInSerial queues so they never
+    // collide on the file set.
+    private mergeFileSet = runInSerial(async (bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false, forceDeleteStreams = false): Promise<boolean> => {
+        this.reader.beginCompaction();
+        try {
+            return await this.mergeFileSetInner(bulkFiles, streamFiles, includesOldest, forceDeleteStreams);
+        } finally {
+            this.reader.endCompaction();
+        }
+    });
+
+    private async mergeFileSetInner(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest: boolean, forceDeleteStreams: boolean): Promise<boolean> {
         const storage = await this.storage();
         const timestamp = nextFileTime();
 
+        // The caller's bulkFiles came from a `listFiles()` snapshot. Between then and now any file
+        // could be gone (another tab's deferred delete, manual cleanup). The subCaches reader for a
+        // deleted file is silently stale — its in-memory metadata still resolves, but the very next
+        // getRange against disk throws. Re-verify existence up-front and drop the cache entry for
+        // anything that vanished, so we don't even try to plan around a ghost file.
+        const verifiedBulkFiles = (await Promise.all(bulkFiles.map(async f => {
+            try {
+                const info = await storage.getInfo(f.fileName);
+                if (!info) { this.subCaches.bulk.delete(f.fileName); return undefined; }
+                return f;
+            } catch {
+                this.subCaches.bulk.delete(f.fileName);
+                return undefined;
+            }
+        }))).filter((f): f is BulkFileInfo => f !== undefined);
+        bulkFiles = verifiedBulkFiles;
+
         const consumedBulk: BulkFileInfo[] = [];
         const bulkReaders: BaseBulkDatabaseReader[] = [];
         await Promise.all(bulkFiles.map(async f => {
@@ -712,6 +812,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // correct). Block-cache eviction for inputs happens on the SECOND rebuild after deletion.
         await this.triggerRebuild();
 
+        // Only the sources runPlannedMerge actually used are safe to delete — a source it dropped
+        // mid-plan (file gone, corruption) still holds data we couldn't merge in, so leave it on disk
+        // and let a future merge re-attempt it.
+        const usedConsumedBulk = consumedBulk.filter(f => mergeResult.usedSourceNames.has(f.fileName));
+        const usedStreamFiles = streamFiles.filter(f => mergeResult.usedSourceNames.has(f.fileName) || mergeResult.usedSourceNames.has("(streams)"));
+
         // Deferred delete: give the FS time to durably flush new writes before removing the inputs.
         // If we crash before the timer fires, the inputs survive as duplicates and a later key-stratify
         // pass dedupes them. We DON'T await this — the merge returns "done" as soon as the new outputs
@@ -720,8 +826,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const timer = setTimeout(() => {
             void (async () => {
                 try {
-                    for (const f of consumedBulk) await remove(f.fileName);
-                    for (const f of streamFiles) {
+                    for (const f of usedConsumedBulk) await remove(f.fileName);
+                    for (const f of usedStreamFiles) {
                         if (await this.canDeleteStream(f, Date.now(), streamData.sizes, forceDeleteStreams)) await remove(f.fileName);
                     }
                     await this.triggerRebuild();
@@ -832,6 +938,33 @@ export class BulkDatabaseBase<T extends { key: string }> {
             }
         }
 
+        // Pass 1.5 (emergency dedup): Pass 1 has had its chance. If the bulk tier is fat AND mostly
+        // duplicates, the per-group walk below (5 min spacing per group) takes hours for an extreme
+        // state (1000+ files, 99% dup) — fold every bulk file in one merge and skip the walk.
+        {
+            const { bulkFiles } = await this.listFiles();
+            if (bulkFiles.length >= 2) {
+                const storage = await this.storage();
+                let totalBytes = 0;
+                let totalSlots = 0;
+                const uniqueKeys = new Set<string>();
+                for (const f of bulkFiles) {
+                    try {
+                        const reader = await loadFileReader(this.name, storage, f, this.subCaches.bulk);
+                        totalBytes += reader.totalBytes;
+                        totalSlots += reader.keys.length;
+                        for (const k of reader.keys) uniqueKeys.add(k);
+                    } catch { /* skip unreadable */ }
+                }
+                const dupFraction = totalSlots ? (totalSlots - uniqueKeys.size) / totalSlots : 0;
+                if (totalBytes > EMERGENCY_DEDUP_BYTES && dupFraction > EMERGENCY_DEDUP_FRACTION) {
+                    console.log(`${blue(this.name)} ${magenta("emergency-dedup")}: ${fmtBytes(totalBytes)} across ${bulkFiles.length} bulk file(s), ${Math.round(dupFraction * 100)}% duplicate key-slots — folding all at once`);
+                    if (!await runMerge(bulkFiles, [])) return merged;
+                    return merged;
+                }
+            }
+        }
+
         // Pass 2: key-stratified deduplication. Disjoint key ranges → one group's merge doesn't
         // change another's duplication; re-select each group's files at merge time (set shifts).
         const groups = await this.findDuplicateGroups();
@@ -932,6 +1065,13 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return this.reader.isColumnLoadedSync(column);
     }
 
+    // Reactive: true while a merge is rewriting this collection's files. Use this in UI to show a
+    // "compacting…" indicator. Counts both background merges (maybeMerge) and explicit compact/merge
+    // calls. Becomes false once the new index is swapped in (the deferred-delete window is NOT counted).
+    public isCompactingSync(): boolean {
+        return this.reader.isCompactingSync();
+    }
+
     public async getColumnInfo() {
         const index = await this.ensureIndex();
         return index.reader.columns;

package/storage/BulkDatabase2/BulkDatabaseMerge.d.ts

@@ -46,5 +46,6 @@ export declare function runPlannedMerge(config: {
 }): Promise<{
     outputs: PlannedMergeOutput[];
     carriedDeletes: Map<string, number>;
+    usedSourceNames: Set<string>;
 }>;
 export {};

package/storage/BulkDatabase2/BulkDatabaseMerge.ts

@@ -92,16 +92,42 @@ export async function runPlannedMerge(config: {
     // console.log so standalone callers still get identifiable output.
     log?: (line: string) => void;
     writeFile: (data: Buffer) => Promise<{ name: string; size: number }>;
-}): Promise<{ outputs: PlannedMergeOutput[]; carriedDeletes: Map<string, number> }> {
+}): Promise<{ outputs: PlannedMergeOutput[]; carriedDeletes: Map<string, number>; usedSourceNames: Set<string> }> {
     const targetFileBytes = config.targetFileBytes ?? TARGET_FILE_BYTES;
     const targetBatchBytes = config.targetBatchBytes ?? DEFAULT_OUTPUT_BATCH_BYTES;
     const log = config.log ?? (line => console.log(`${blue(config.collectionName)} ${line}`));
 
     // ─────────────────────────────────────────── Phase 1: plan ───────────────────────────────────────────
-    // Aggregate keyTimes + deleteTimes across all sources (max per key).
+    // Load per-source column indexes FIRST so a file that's gone (deleted between caller's listFiles and
+    // now) gets caught here and excluded from the merge entirely. Each source's load is wrapped in a
+    // try/catch; on failure we mark the source bad, log a "skipped" line, and the rest of planning +
+    // execute pretends the source doesn't exist. The caller only deletes inputs that were actually used.
+    const sourceOk = new Array<boolean>(config.sources.length).fill(true);
+    const indexesPerSource: Map<string, ColumnIndex>[] = await Promise.all(config.sources.map(async (src, si) => {
+        const map = new Map<string, ColumnIndex>();
+        try {
+            await Promise.all(src.columns
+                .filter(c => c.column !== KEY_COLUMN)
+                .map(async c => { map.set(c.column, await src.getColumnIndex(c.column)); }));
+        } catch (e) {
+            sourceOk[si] = false;
+            log(`${magenta("skipped")} ${config.sourceNames[si]}: ${(e as Error).message}`);
+        }
+        return map;
+    }));
+    const usedSourceNames = new Set<string>();
+    for (let si = 0; si < config.sources.length; si++) {
+        if (sourceOk[si]) usedSourceNames.add(config.sourceNames[si]);
+    }
+
+    // Aggregate keyTimes + deleteTimes across GOOD sources only (max per key). A bad source's cached
+    // metadata might claim a key as newest, but we can't actually read its cells — letting it win the
+    // time resolution would just drop that key out of the output.
     const deleteTime = new Map<string, number>();
     const keyTime = new Map<string, number>();
-    for (const src of config.sources) {
+    for (let si = 0; si < config.sources.length; si++) {
+        if (!sourceOk[si]) continue;
+        const src = config.sources[si];
         for (const [k, t] of src.keyTimes) {
             const prev = keyTime.get(k);
             if (prev === undefined || t > prev) keyTime.set(k, t);
@@ -127,28 +153,19 @@ export async function runPlannedMerge(config: {
         if (dT >= sT) carriedDeletes.set(k, dT);
     }
 
-    // All distinct value columns (KEY_COLUMN excluded — it's added at file-assembly time). Order: first-
-    // seen across sources, matching the existing builders.
+    // All distinct value columns from GOOD sources (KEY_COLUMN excluded — it's added at file-assembly
+    // time). Order: first-seen, matching the existing builders.
     const allColumns: string[] = [];
     const seenCols = new Set<string>();
-    for (const src of config.sources) {
-        for (const c of src.columns) {
+    for (let si = 0; si < config.sources.length; si++) {
+        if (!sourceOk[si]) continue;
+        for (const c of config.sources[si].columns) {
             if (c.column === KEY_COLUMN || seenCols.has(c.column)) continue;
             seenCols.add(c.column);
             allColumns.push(c.column);
         }
     }
 
-    // Load every (source, column) index in parallel — small data (offsets+types), no values pulled. Each
-    // index is kept for the executor too, so it can read contiguous row-ranges from the right source.
-    const indexesPerSource: Map<string, ColumnIndex>[] = await Promise.all(config.sources.map(async src => {
-        const map = new Map<string, ColumnIndex>();
-        await Promise.all(allColumns.map(async col => {
-            map.set(col, await src.getColumnIndex(col));
-        }));
-        return map;
-    }));
-
     // For each live key, for each column, find the winning source: among sources that have this key, the
     // one whose keyTime is largest AND whose column-index reports non-ABSENT. Record source + sourceRow +
     // byteLen + type so the planner can size offsets and the executor can copy the bytes.
@@ -161,6 +178,7 @@ export async function runPlannedMerge(config: {
             let bestTime = -Infinity;
             let best: CellChoice | undefined;
             for (let si = 0; si < config.sources.length; si++) {
+                if (!sourceOk[si]) continue;
                 const src = config.sources[si];
                 const t = src.keyTimes.get(key);
                 if (t === undefined) continue;
@@ -251,7 +269,7 @@ export async function runPlannedMerge(config: {
     const writtenBytes = outputs.reduce((a, o) => a + o.size, 0);
     log(`${magenta("execute")}: ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written`);
 
-    return { outputs, carriedDeletes };
+    return { outputs, carriedDeletes, usedSourceNames };
 }
 
 function buildOutputPlan(

package/storage/BulkDatabase2/BulkDatabaseReader.d.ts

@@ -31,6 +31,10 @@ export declare class BulkDatabaseReader<T extends {
     isKeyWatched(key: string): boolean;
     isLiveNow(key: string): boolean;
     localTime(key: string): number;
+    private compactingCount;
+    beginCompaction(): void;
+    endCompaction(): void;
+    isCompactingSync(): boolean;
     private notifyOverlayMutation;
     getKeys(): Promise<string[]>;
     getColumn<C extends keyof T>(column: C): Promise<{

package/storage/BulkDatabase2/BulkDatabaseReader.ts

@@ -7,6 +7,7 @@ import { blue, red } from "socket-function/src/formatting/logColors";
 const NULL = String.fromCharCode(0);
 const LOAD_SIGNAL = NULL + "load";
 const OVERLAY_SIGNAL = NULL + "overlay";
+const COMPACTING_SIGNAL = NULL + "compacting";
 const TRIGGER_THROTTLE_FIRST_STEP_MS = 16;
 
 function nullJoin(a: string, b: string): string {
@@ -88,6 +89,22 @@ export class BulkDatabaseReader<T extends { key: string }> {
         return -Infinity;
     }
 
+    // Counter (so concurrent / nested merges in the same instance compose correctly). Signal fires
+    // immediately, not through the trigger throttle — UI spinners should show right away.
+    private compactingCount = 0;
+    beginCompaction(): void {
+        this.compactingCount++;
+        if (this.compactingCount === 1) this.cfg.deps.invalidate(COMPACTING_SIGNAL);
+    }
+    endCompaction(): void {
+        this.compactingCount--;
+        if (this.compactingCount === 0) this.cfg.deps.invalidate(COMPACTING_SIGNAL);
+    }
+    isCompactingSync(): boolean {
+        this.cfg.deps.observe(COMPACTING_SIGNAL);
+        return this.compactingCount > 0;
+    }
+
     private notifyOverlayMutation(key: string, columns: Iterable<string> | "all"): void {
         this.dataGen++;
         if (columns === "all") this.columnCache.clear();

package/storage/BulkDatabase2/mergeLock.d.ts

@@ -1,2 +1,6 @@
+import type { FileStorage } from "../FileFolderAPI";
 export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
 export declare function releaseMergeLock(collection: string, holderId: string): void;
+export declare function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean>;
+export declare function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void;
+export declare function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void>;

package/storage/BulkDatabase2/mergeLock.ts

@@ -6,6 +6,14 @@
 // localStorage is unavailable (Node) — there concurrent merges are harmless, which the Node stress
 // tests exercise. localStorage has no atomic compare-and-swap, so we write-then-reread to shrink the
 // race window, and a TTL frees a lock left behind by a tab that crashed or closed mid-merge.
+//
+// File-lock layer (see acquireMergeFileLock at the bottom): for cross-PROCESS safety (different
+// browsers, different machines pointing at the same remote storage), we ALSO write a `.merge-lock`
+// file inside the collection's storage with our ID. The protocol is check-write-wait-recheck — last
+// writer wins after the 15s settle window. The localStorage lock above covers same-origin tabs; this
+// covers everything else (separate processes, remote shared storage).
+
+import type { FileStorage } from "../FileFolderAPI";
 
 const LOCK_TTL_MS = 30 * 1000;
 
@@ -48,3 +56,57 @@ export function releaseMergeLock(collection: string, holderId: string): void {
     const existing = ls.getItem(key);
     if (existing && existing.startsWith(holderId + ":")) ls.removeItem(key);
 }
+
+// ── File-based lock (cross-process / shared storage) ──────────────────────────────────────────────
+const FILE_LOCK_KEY = ".merge-lock";
+const FILE_LOCK_TTL_MS = 5 * 60 * 1000;
+const FILE_LOCK_SETTLE_MS = 15 * 1000;
+const FILE_LOCK_HEARTBEAT_MS = 2 * 60 * 1000;
+
+async function readMergeFileLock(storage: FileStorage): Promise<{ id: string; time: number } | undefined> {
+    try {
+        const buf = await storage.get(FILE_LOCK_KEY);
+        if (!buf) return undefined;
+        const s = buf.toString("utf8");
+        const colon = s.lastIndexOf(":");
+        if (colon < 0) return undefined;
+        const time = parseInt(s.slice(colon + 1), 10);
+        if (!Number.isFinite(time)) return undefined;
+        return { id: s.slice(0, colon), time };
+    } catch { return undefined; }
+}
+
+async function writeMergeFileLock(storage: FileStorage, holderId: string, time: number): Promise<void> {
+    await storage.set(FILE_LOCK_KEY, Buffer.from(`${holderId}:${time}`, "utf8") as Buffer);
+}
+
+// Acquire the cross-process file lock. Resolves to true ONLY if, after writing our ID and waiting
+// FILE_LOCK_SETTLE_MS for concurrent writers to settle, the file still names us. Caller must
+// releaseMergeFileLock() on the same storage + holderId when done.
+export async function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean> {
+    const now = Date.now();
+    const existing = await readMergeFileLock(storage);
+    if (existing && existing.id !== holderId && now - existing.time < FILE_LOCK_TTL_MS) return false;
+    await writeMergeFileLock(storage, holderId, now);
+    await new Promise(r => setTimeout(r, FILE_LOCK_SETTLE_MS));
+    const after = await readMergeFileLock(storage);
+    return !!after && after.id === holderId;
+}
+
+// Keep the lock fresh. The merge might run longer than FILE_LOCK_TTL_MS; without periodic re-stamping,
+// another process would see the lock as stale and acquire it. Returns a stop function the caller calls
+// in `finally`. Heartbeat is best-effort: a failed write is logged but not propagated.
+export function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void {
+    const interval = setInterval(() => {
+        void writeMergeFileLock(storage, holderId, Date.now()).catch(() => { /* best-effort */ });
+    }, FILE_LOCK_HEARTBEAT_MS);
+    (interval as { unref?: () => void }).unref?.();
+    return () => clearInterval(interval);
+}
+
+export async function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void> {
+    try {
+        const existing = await readMergeFileLock(storage);
+        if (existing && existing.id === holderId) await storage.remove(FILE_LOCK_KEY);
+    } catch { /* ignore */ }
+}