sliftutils 1.4.67 → 1.4.69

package/index.d.ts

@@ -903,10 +903,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
          * whether it bailed because another tab/process holds the merge lock — so a scheduler can call this
          * (e.g. every 30 minutes) and tell "nothing to do" from "someone else is already merging".
          */
-        tryMergeNow(): Promise<{
-            merged: boolean;
-            lockFailed: boolean;
-        }>;
+        tryMergeNow(): Promise<void>;
         /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
          * most callers want compact() or tryMergeNow(). */
         merge(timeLo: number, timeHi: number): Promise<void>;
@@ -933,7 +930,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
     export declare const BULK_ROOT_FOLDER = "bulkDatabases2";
     export declare const bulkDatabase2Timing: {
         streamSealAgeMs: number;
-        mergeCheckIntervalMs: number;
+        visibleMergeIntervalMs: number;
         mergeSpacingMs: number;
         firstMergeTriggerFiles: number;
         firstMergeTriggerRangeMs: number;
@@ -967,6 +964,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private storageFactory;
         private config;
         constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+        private setupVisibilityMergeCheck;
         private reader;
         private subCaches;
         private pendingAppends;
@@ -977,7 +975,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private streamFileName;
         private currentStreamFileName;
         private currentStreamFileBytes;
-        private lastMergeCheck;
+        private mergeInFlight;
         private streamRowsOnDisk;
         private streamBytesOnDisk;
         private fileSetPollTimer;
@@ -1027,10 +1025,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private listFiles;
         private writeBulkFile;
         private maybeMerge;
-        tryMergeNow(): Promise<{
-            merged: boolean;
-            lockFailed: boolean;
-        }>;
+        tryMergeNow: () => Promise<void>;
         compact(): Promise<void>;
         merge(timeLo: number, timeHi: number): Promise<void>;
         private readBulkHeader;
@@ -1040,7 +1035,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private mergeFileSetInner;
         private canDeleteStream;
         private mergeSpacingDelay;
-        private testMerge;
+        private testMergeINTERNAL_DO_NOT_CALL;
         private findDuplicateGroups;
         getSingleField<C extends keyof T>(key: string, column: C): Promise<T[C] | undefined>;
         getSingleFieldObj<C extends keyof T>(key: string, column: C): Promise<{
@@ -1249,6 +1244,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseMerge" {
     }): Promise<{
         outputs: PlannedMergeOutput[];
         carriedDeletes: Map<string, number>;
+        usedSourceNames: Set<string>;
     }>;
     export {};
 
@@ -1527,8 +1523,12 @@ declare module "sliftutils/storage/BulkDatabase2/blockCache" {
 }
 
 declare module "sliftutils/storage/BulkDatabase2/mergeLock" {
+    import type { FileStorage } from "../FileFolderAPI";
     export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
     export declare function releaseMergeLock(collection: string, holderId: string): void;
+    export declare function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean>;
+    export declare function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void;
+    export declare function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void>;
 
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.67",
+    "version": "1.4.69",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -138,10 +138,7 @@ export interface IBulkDatabase2<T extends {
      * whether it bailed because another tab/process holds the merge lock — so a scheduler can call this
      * (e.g. every 30 minutes) and tell "nothing to do" from "someone else is already merging".
      */
-    tryMergeNow(): Promise<{
-        merged: boolean;
-        lockFailed: boolean;
-    }>;
+    tryMergeNow(): Promise<void>;
     /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
      * most callers want compact() or tryMergeNow(). */
     merge(timeLo: number, timeHi: number): Promise<void>;

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -131,7 +131,7 @@ export interface IBulkDatabase2<T extends { key: string }> {
      * whether it bailed because another tab/process holds the merge lock — so a scheduler can call this
      * (e.g. every 30 minutes) and tell "nothing to do" from "someone else is already merging".
      */
-    tryMergeNow(): Promise<{ merged: boolean; lockFailed: boolean }>;
+    tryMergeNow(): Promise<void>;
 
     /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
      * most callers want compact() or tryMergeNow(). */

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -2,7 +2,7 @@ import type { FileStorage } from "../FileFolderAPI";
 export declare const BULK_ROOT_FOLDER = "bulkDatabases2";
 export declare const bulkDatabase2Timing: {
     streamSealAgeMs: number;
-    mergeCheckIntervalMs: number;
+    visibleMergeIntervalMs: number;
     mergeSpacingMs: number;
     firstMergeTriggerFiles: number;
     firstMergeTriggerRangeMs: number;
@@ -36,6 +36,7 @@ export declare class BulkDatabaseBase<T extends {
     private storageFactory;
     private config;
     constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory, config?: BulkDatabase2Config);
+    private setupVisibilityMergeCheck;
     private reader;
     private subCaches;
     private pendingAppends;
@@ -46,7 +47,7 @@ export declare class BulkDatabaseBase<T extends {
     private streamFileName;
     private currentStreamFileName;
     private currentStreamFileBytes;
-    private lastMergeCheck;
+    private mergeInFlight;
     private streamRowsOnDisk;
     private streamBytesOnDisk;
     private fileSetPollTimer;
@@ -96,10 +97,7 @@ export declare class BulkDatabaseBase<T extends {
     private listFiles;
     private writeBulkFile;
     private maybeMerge;
-    tryMergeNow(): Promise<{
-        merged: boolean;
-        lockFailed: boolean;
-    }>;
+    tryMergeNow: () => Promise<void>;
     compact(): Promise<void>;
     merge(timeLo: number, timeHi: number): Promise<void>;
     private readBulkHeader;
@@ -109,7 +107,7 @@ export declare class BulkDatabaseBase<T extends {
     private mergeFileSetInner;
     private canDeleteStream;
     private mergeSpacingDelay;
-    private testMerge;
+    private testMergeINTERNAL_DO_NOT_CALL;
     private findDuplicateGroups;
     getSingleField<C extends keyof T>(key: string, column: C): Promise<T[C] | undefined>;
     getSingleFieldObj<C extends keyof T>(key: string, column: C): Promise<{

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -1,4 +1,5 @@
-import { sort } from "socket-function/src/misc";
+import { sort, throttleFunction } from "socket-function/src/misc";
+import { runInSerial } from "socket-function/src/batching";
 import { getTimeUnique } from "socket-function/src/bits";
 import { lazy } from "socket-function/src/caching";
 import { isNode } from "typesafecss";
@@ -17,7 +18,7 @@ import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { STREAM_EXTENSION, frameDeletes, frameRows, streamReaderFromEntries } from "./streamLog";
 import { broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, connect as syncConnect, isSyncSupported, RemoteWrite } from "./syncClient";
 import { DELETED } from "./WriteOverlay";
-import { releaseMergeLock, tryAcquireMergeLock } from "./mergeLock";
+import { releaseMergeFileLock, releaseMergeLock, startMergeFileLockHeartbeat, tryAcquireMergeFileLock, tryAcquireMergeLock } from "./mergeLock";
 import {
     BulkFileInfo,
     LoadedIndex,
@@ -41,11 +42,19 @@ const MAX_INDEX_RELOAD_ATTEMPTS = 3;
 const FIRST_MERGE_BYTES = TARGET_FILE_BYTES / 2;
 const KEY_GROUP_BYTES = 800 * 1024 * 1024;
 const DUP_THRESHOLD = 0.4;
+// Emergency Pass 2 short-circuit: once Pass 1 has had a chance to fold streams, if the bulk tier is
+// big enough AND the overall key duplication fraction is high, do ONE merge across every bulk file
+// instead of the per-key-group walk (which spaces merges 5 min apart — 16 hours for 200 groups).
+const EMERGENCY_DEDUP_BYTES = 512 * 1024 * 1024;
+const EMERGENCY_DEDUP_FRACTION = 0.5;
 const WRITE_FLUSH_FIRST_STEP_MS = 250;
 
 export const bulkDatabase2Timing = {
     streamSealAgeMs: 10 * 60 * 60 * 1000,
-    mergeCheckIntervalMs: 30 * 60 * 1000,
+    // Wait this long after the tab becomes visible before the first merge check, then check this often
+    // afterwards. Tab being hidden cancels the timers (no merges while in background). Browser-only;
+    // Node compactors (e.g. remoteFileServer) drive their own polling.
+    visibleMergeIntervalMs: 5 * 60 * 1000,
     mergeSpacingMs: 5 * 60 * 1000,
     firstMergeTriggerFiles: 20,
     firstMergeTriggerRangeMs: 3 * 24 * 60 * 60 * 1000,
@@ -164,10 +173,43 @@ export class BulkDatabaseBase<T extends { key: string }> {
         }
         this.fileSetPollTimer = setInterval(() => void this.pollFileSet(), bulkDatabase2Timing.fileSetPollIntervalMs);
         (this.fileSetPollTimer as { unref?: () => void }).unref?.();
+        this.setupVisibilityMergeCheck();
         BulkDatabaseBase.liveInstances.add(this);
         BulkDatabaseBase.startMemoryWatchdog();
     }
 
+    // Every `visibleMergeIntervalMs` of being-visible time, run a merge check. First check fires
+    // `visibleMergeIntervalMs` after the tab becomes visible (not on initial load — gives the user a
+    // moment of session commitment before we start churning the FS), then every interval after. Tab
+    // hiding cancels the timers; visible-again restarts. Node has no document — there we consider
+    // ourselves always-visible and just install the interval directly.
+    private setupVisibilityMergeCheck(): void {
+        let firstTimer: ReturnType<typeof setTimeout> | undefined;
+        let intervalTimer: ReturnType<typeof setInterval> | undefined;
+        const stop = () => {
+            if (firstTimer) { clearTimeout(firstTimer); firstTimer = undefined; }
+            if (intervalTimer) { clearInterval(intervalTimer); intervalTimer = undefined; }
+        };
+        const start = () => {
+            if (firstTimer || intervalTimer) return;
+            firstTimer = setTimeout(() => {
+                firstTimer = undefined;
+                void this.maybeMerge();
+                intervalTimer = setInterval(() => void this.maybeMerge(), bulkDatabase2Timing.visibleMergeIntervalMs);
+                (intervalTimer as { unref?: () => void }).unref?.();
+            }, bulkDatabase2Timing.visibleMergeIntervalMs);
+            (firstTimer as { unref?: () => void }).unref?.();
+        };
+        if (typeof document === "undefined") { start(); return; }
+        try {
+            document.addEventListener("visibilitychange", () => {
+                if (document.visibilityState === "visible") start();
+                else stop();
+            });
+            if (document.visibilityState === "visible") start();
+        } catch { /* not in a DOM context */ }
+    }
+
     private reader: BulkDatabaseReader<T>;
     private subCaches: SubReaderCaches = { bulk: new Map(), stream: new Map() };
 
@@ -180,7 +222,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
     private streamFileName: string | undefined;
     private currentStreamFileName: string | undefined;
     private currentStreamFileBytes = 0;
-    private lastMergeCheck = Date.now();
+    // In-process re-entry guard: if a merge is running, additional triggers (visibility timer, writes,
+    // tryMergeNow calls) return immediately instead of queueing. Keeps the visibility-timer firings
+    // from stacking behind a long merge.
+    private mergeInFlight = false;
     // Running counters of stream-tier rows + bytes on disk. Seeded from each LoadedIndex build, then
     // incremented per flush so the fold-trigger checks current data without an extra directory listing.
     private streamRowsOnDisk = 0;
@@ -380,7 +425,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         });
         for (const { time, row } of stamped) syncBroadcast(this.name, { key: row.key as string, time, value: row });
         await this.streamAppend(framed, stamped.length);
-        void this.maybeMerge();
+        void this.maybeMerge({ onlyIfStreamHeavy: true });
     }
 
     public async delete(key: string): Promise<void> {
@@ -396,7 +441,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         });
         for (const { time, key } of stamped) syncBroadcast(this.name, { key, time, deleted: true });
         await this.streamAppend(frameDeletes(stamped), stamped.length);
-        void this.maybeMerge();
+        void this.maybeMerge({ onlyIfStreamHeavy: true });
     }
 
     // Coalesce stream appends on a ramping per-collection schedule (the browser rewrites the whole
@@ -533,11 +578,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
     }
 
     // ── merge policy ─────────────────────────────────────────────────────────────────────────────────
-    private async maybeMerge(): Promise<void> {
+    // Called both periodically (visibility timer) and on writes (with the streamNeedsFold gate so a
+    // small write doesn't drag the merge through). The mergeInFlight check in tryMergeNow skips any
+    // call that arrives while a previous one is still running — so 5-min timer ticks don't pile up.
+    private async maybeMerge(opts: { onlyIfStreamHeavy?: boolean } = {}): Promise<void> {
         if (!await this.automaticCompactionAllowed()) return;
-        const now = Date.now();
-        if (!this.streamNeedsFold() && now - this.lastMergeCheck < bulkDatabase2Timing.mergeCheckIntervalMs) return;
-        this.lastMergeCheck = now;
+        if (opts.onlyIfStreamHeavy && !this.streamNeedsFold()) return;
         try {
             await this.tryMergeNow();
         } catch (e) {
@@ -545,17 +591,40 @@ export class BulkDatabaseBase<T extends { key: string }> {
         }
     }
 
-    public async tryMergeNow(): Promise<{ merged: boolean; lockFailed: boolean }> {
-        if (!tryAcquireMergeLock(this.name, writerId)) return { merged: false, lockFailed: true };
+    // Returns immediately with { lockFailed: true } if another call is already in flight (same
+    // instance), if the localStorage lock is held by another tab, or if the cross-process file lock
+    // says another process owns it. Otherwise: acquires both locks, runs testMerge, releases.
+    public tryMergeNow = throttleFunction(1000, async () => {
+        if (this.mergeInFlight) return;
+        if (!tryAcquireMergeLock(this.name, writerId)) return;
+        const storage = await this.storage();
+        const haveFileLock = await tryAcquireMergeFileLock(storage, writerId);
+        if (!haveFileLock) {
+            releaseMergeLock(this.name, writerId);
+            return;
+        }
+        this.mergeInFlight = true;
+        const stopHeartbeat = startMergeFileLockHeartbeat(storage, writerId);
         try {
-            return { merged: await this.testMerge(), lockFailed: false };
+            let t = Date.now();
+            await this.testMergeINTERNAL_DO_NOT_CALL();
+            console.log(`  [autocompact] ${this.name}: merged (${Date.now() - t}ms)`);
         } finally {
+            stopHeartbeat();
+            await releaseMergeFileLock(storage, writerId);
             releaseMergeLock(this.name, writerId);
+            this.mergeInFlight = false;
         }
-    }
+    });
 
     public async compact(): Promise<void> {
+        if (this.mergeInFlight) return;
         if (!tryAcquireMergeLock(this.name, writerId)) return;
+        const storage = await this.storage();
+        const haveFileLock = await tryAcquireMergeFileLock(storage, writerId);
+        if (!haveFileLock) { releaseMergeLock(this.name, writerId); return; }
+        this.mergeInFlight = true;
+        const stopHeartbeat = startMergeFileLockHeartbeat(storage, writerId);
         try {
             await this.flushPending();
             syncBroadcastSeal(this.name);
@@ -565,7 +634,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
             // can be dropped (nothing left to suppress).
             if (bulkFiles.length + streamFiles.length >= 1) await this.mergeFileSet(bulkFiles, streamFiles, true);
         } finally {
+            stopHeartbeat();
+            await releaseMergeFileLock(storage, writerId);
             releaseMergeLock(this.name, writerId);
+            this.mergeInFlight = false;
         }
     }
 
@@ -631,19 +703,40 @@ export class BulkDatabaseBase<T extends { key: string }> {
     // leaves duplicates (next merge dedupes) rather than a gap. After the file set changes on disk,
     // we trigger an index rebuild + atomic swap; once swap completes, the consumed files' block-cache
     // entries are evicted (no consumer can ask for them now).
-    private async mergeFileSet(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false, forceDeleteStreams = false): Promise<boolean> {
+    //
+    // Serialized: foldOwnStream + merge(timeLo, timeHi) + testMerge's runMerge all hit this; the lock
+    // covers tryMergeNow/compact but those two paths bypass it. runInSerial queues so they never
+    // collide on the file set.
+    private mergeFileSet = runInSerial(async (bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest = false, forceDeleteStreams = false): Promise<boolean> => {
         this.reader.beginCompaction();
         try {
             return await this.mergeFileSetInner(bulkFiles, streamFiles, includesOldest, forceDeleteStreams);
         } finally {
             this.reader.endCompaction();
         }
-    }
+    });
 
     private async mergeFileSetInner(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], includesOldest: boolean, forceDeleteStreams: boolean): Promise<boolean> {
         const storage = await this.storage();
         const timestamp = nextFileTime();
 
+        // The caller's bulkFiles came from a `listFiles()` snapshot. Between then and now any file
+        // could be gone (another tab's deferred delete, manual cleanup). The subCaches reader for a
+        // deleted file is silently stale — its in-memory metadata still resolves, but the very next
+        // getRange against disk throws. Re-verify existence up-front and drop the cache entry for
+        // anything that vanished, so we don't even try to plan around a ghost file.
+        const verifiedBulkFiles = (await Promise.all(bulkFiles.map(async f => {
+            try {
+                const info = await storage.getInfo(f.fileName);
+                if (!info) { this.subCaches.bulk.delete(f.fileName); return undefined; }
+                return f;
+            } catch {
+                this.subCaches.bulk.delete(f.fileName);
+                return undefined;
+            }
+        }))).filter((f): f is BulkFileInfo => f !== undefined);
+        bulkFiles = verifiedBulkFiles;
+
         const consumedBulk: BulkFileInfo[] = [];
         const bulkReaders: BaseBulkDatabaseReader[] = [];
         await Promise.all(bulkFiles.map(async f => {
@@ -721,6 +814,12 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // correct). Block-cache eviction for inputs happens on the SECOND rebuild after deletion.
         await this.triggerRebuild();
 
+        // Only the sources runPlannedMerge actually used are safe to delete — a source it dropped
+        // mid-plan (file gone, corruption) still holds data we couldn't merge in, so leave it on disk
+        // and let a future merge re-attempt it.
+        const usedConsumedBulk = consumedBulk.filter(f => mergeResult.usedSourceNames.has(f.fileName));
+        const usedStreamFiles = streamFiles.filter(f => mergeResult.usedSourceNames.has(f.fileName) || mergeResult.usedSourceNames.has("(streams)"));
+
         // Deferred delete: give the FS time to durably flush new writes before removing the inputs.
         // If we crash before the timer fires, the inputs survive as duplicates and a later key-stratify
         // pass dedupes them. We DON'T await this — the merge returns "done" as soon as the new outputs
@@ -729,8 +828,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const timer = setTimeout(() => {
             void (async () => {
                 try {
-                    for (const f of consumedBulk) await remove(f.fileName);
-                    for (const f of streamFiles) {
+                    for (const f of usedConsumedBulk) await remove(f.fileName);
+                    for (const f of usedStreamFiles) {
                         if (await this.canDeleteStream(f, Date.now(), streamData.sizes, forceDeleteStreams)) await remove(f.fileName);
                     }
                     await this.triggerRebuild();
@@ -775,7 +874,8 @@ export class BulkDatabaseBase<T extends { key: string }> {
     //      into one when they fragment or span too wide a time range.
     //   2) Key-stratify: walk all keys in ~KEY_GROUP_BYTES groups; rewrite groups whose duplicate-key
     //      fraction passes DUP_THRESHOLD, highest first.
-    private async testMerge(): Promise<boolean> {
+    // Use tryMerge instead
+    private async testMergeINTERNAL_DO_NOT_CALL(): Promise<boolean> {
         let merged = false;
         await this.flushPending();
         const runMerge = async (bulk: BulkFileInfo[], stream: StreamFileInfo[]): Promise<boolean> => {
@@ -841,6 +941,33 @@ export class BulkDatabaseBase<T extends { key: string }> {
             }
         }
 
+        // Pass 1.5 (emergency dedup): Pass 1 has had its chance. If the bulk tier is fat AND mostly
+        // duplicates, the per-group walk below (5 min spacing per group) takes hours for an extreme
+        // state (1000+ files, 99% dup) — fold every bulk file in one merge and skip the walk.
+        {
+            const { bulkFiles } = await this.listFiles();
+            if (bulkFiles.length >= 2) {
+                const storage = await this.storage();
+                let totalBytes = 0;
+                let totalSlots = 0;
+                const uniqueKeys = new Set<string>();
+                for (const f of bulkFiles) {
+                    try {
+                        const reader = await loadFileReader(this.name, storage, f, this.subCaches.bulk);
+                        totalBytes += reader.totalBytes;
+                        totalSlots += reader.keys.length;
+                        for (const k of reader.keys) uniqueKeys.add(k);
+                    } catch { /* skip unreadable */ }
+                }
+                const dupFraction = totalSlots ? (totalSlots - uniqueKeys.size) / totalSlots : 0;
+                if (totalBytes > EMERGENCY_DEDUP_BYTES && dupFraction > EMERGENCY_DEDUP_FRACTION) {
+                    console.log(`${blue(this.name)} ${magenta("emergency-dedup")}: ${fmtBytes(totalBytes)} across ${bulkFiles.length} bulk file(s), ${Math.round(dupFraction * 100)}% duplicate key-slots — folding all at once`);
+                    if (!await runMerge(bulkFiles, [])) return merged;
+                    return merged;
+                }
+            }
+        }
+
         // Pass 2: key-stratified deduplication. Disjoint key ranges → one group's merge doesn't
         // change another's duplication; re-select each group's files at merge time (set shifts).
         const groups = await this.findDuplicateGroups();

package/storage/BulkDatabase2/BulkDatabaseMerge.d.ts

@@ -46,5 +46,6 @@ export declare function runPlannedMerge(config: {
 }): Promise<{
     outputs: PlannedMergeOutput[];
     carriedDeletes: Map<string, number>;
+    usedSourceNames: Set<string>;
 }>;
 export {};

package/storage/BulkDatabase2/BulkDatabaseMerge.ts

@@ -92,16 +92,42 @@ export async function runPlannedMerge(config: {
     // console.log so standalone callers still get identifiable output.
     log?: (line: string) => void;
     writeFile: (data: Buffer) => Promise<{ name: string; size: number }>;
-}): Promise<{ outputs: PlannedMergeOutput[]; carriedDeletes: Map<string, number> }> {
+}): Promise<{ outputs: PlannedMergeOutput[]; carriedDeletes: Map<string, number>; usedSourceNames: Set<string> }> {
     const targetFileBytes = config.targetFileBytes ?? TARGET_FILE_BYTES;
     const targetBatchBytes = config.targetBatchBytes ?? DEFAULT_OUTPUT_BATCH_BYTES;
     const log = config.log ?? (line => console.log(`${blue(config.collectionName)} ${line}`));
 
     // ─────────────────────────────────────────── Phase 1: plan ───────────────────────────────────────────
-    // Aggregate keyTimes + deleteTimes across all sources (max per key).
+    // Load per-source column indexes FIRST so a file that's gone (deleted between caller's listFiles and
+    // now) gets caught here and excluded from the merge entirely. Each source's load is wrapped in a
+    // try/catch; on failure we mark the source bad, log a "skipped" line, and the rest of planning +
+    // execute pretends the source doesn't exist. The caller only deletes inputs that were actually used.
+    const sourceOk = new Array<boolean>(config.sources.length).fill(true);
+    const indexesPerSource: Map<string, ColumnIndex>[] = await Promise.all(config.sources.map(async (src, si) => {
+        const map = new Map<string, ColumnIndex>();
+        try {
+            await Promise.all(src.columns
+                .filter(c => c.column !== KEY_COLUMN)
+                .map(async c => { map.set(c.column, await src.getColumnIndex(c.column)); }));
+        } catch (e) {
+            sourceOk[si] = false;
+            log(`${magenta("skipped")} ${config.sourceNames[si]}: ${(e as Error).message}`);
+        }
+        return map;
+    }));
+    const usedSourceNames = new Set<string>();
+    for (let si = 0; si < config.sources.length; si++) {
+        if (sourceOk[si]) usedSourceNames.add(config.sourceNames[si]);
+    }
+
+    // Aggregate keyTimes + deleteTimes across GOOD sources only (max per key). A bad source's cached
+    // metadata might claim a key as newest, but we can't actually read its cells — letting it win the
+    // time resolution would just drop that key out of the output.
     const deleteTime = new Map<string, number>();
     const keyTime = new Map<string, number>();
-    for (const src of config.sources) {
+    for (let si = 0; si < config.sources.length; si++) {
+        if (!sourceOk[si]) continue;
+        const src = config.sources[si];
         for (const [k, t] of src.keyTimes) {
             const prev = keyTime.get(k);
             if (prev === undefined || t > prev) keyTime.set(k, t);
@@ -127,28 +153,19 @@ export async function runPlannedMerge(config: {
         if (dT >= sT) carriedDeletes.set(k, dT);
     }
 
-    // All distinct value columns (KEY_COLUMN excluded — it's added at file-assembly time). Order: first-
-    // seen across sources, matching the existing builders.
+    // All distinct value columns from GOOD sources (KEY_COLUMN excluded — it's added at file-assembly
+    // time). Order: first-seen, matching the existing builders.
     const allColumns: string[] = [];
     const seenCols = new Set<string>();
-    for (const src of config.sources) {
-        for (const c of src.columns) {
+    for (let si = 0; si < config.sources.length; si++) {
+        if (!sourceOk[si]) continue;
+        for (const c of config.sources[si].columns) {
             if (c.column === KEY_COLUMN || seenCols.has(c.column)) continue;
             seenCols.add(c.column);
             allColumns.push(c.column);
         }
     }
 
-    // Load every (source, column) index in parallel — small data (offsets+types), no values pulled. Each
-    // index is kept for the executor too, so it can read contiguous row-ranges from the right source.
-    const indexesPerSource: Map<string, ColumnIndex>[] = await Promise.all(config.sources.map(async src => {
-        const map = new Map<string, ColumnIndex>();
-        await Promise.all(allColumns.map(async col => {
-            map.set(col, await src.getColumnIndex(col));
-        }));
-        return map;
-    }));
-
     // For each live key, for each column, find the winning source: among sources that have this key, the
     // one whose keyTime is largest AND whose column-index reports non-ABSENT. Record source + sourceRow +
     // byteLen + type so the planner can size offsets and the executor can copy the bytes.
@@ -161,6 +178,7 @@ export async function runPlannedMerge(config: {
             let bestTime = -Infinity;
             let best: CellChoice | undefined;
             for (let si = 0; si < config.sources.length; si++) {
+                if (!sourceOk[si]) continue;
                 const src = config.sources[si];
                 const t = src.keyTimes.get(key);
                 if (t === undefined) continue;
@@ -251,7 +269,7 @@ export async function runPlannedMerge(config: {
     const writtenBytes = outputs.reduce((a, o) => a + o.size, 0);
     log(`${magenta("execute")}: ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written`);
 
-    return { outputs, carriedDeletes };
+    return { outputs, carriedDeletes, usedSourceNames };
 }
 
 function buildOutputPlan(

package/storage/BulkDatabase2/mergeLock.d.ts

@@ -1,2 +1,6 @@
+import type { FileStorage } from "../FileFolderAPI";
 export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
 export declare function releaseMergeLock(collection: string, holderId: string): void;
+export declare function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean>;
+export declare function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void;
+export declare function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void>;

package/storage/BulkDatabase2/mergeLock.ts

@@ -6,6 +6,14 @@
 // localStorage is unavailable (Node) — there concurrent merges are harmless, which the Node stress
 // tests exercise. localStorage has no atomic compare-and-swap, so we write-then-reread to shrink the
 // race window, and a TTL frees a lock left behind by a tab that crashed or closed mid-merge.
+//
+// File-lock layer (see acquireMergeFileLock at the bottom): for cross-PROCESS safety (different
+// browsers, different machines pointing at the same remote storage), we ALSO write a `.merge-lock`
+// file inside the collection's storage with our ID. The protocol is check-write-wait-recheck — last
+// writer wins after the 15s settle window. The localStorage lock above covers same-origin tabs; this
+// covers everything else (separate processes, remote shared storage).
+
+import type { FileStorage } from "../FileFolderAPI";
 
 const LOCK_TTL_MS = 30 * 1000;
 
@@ -48,3 +56,57 @@ export function releaseMergeLock(collection: string, holderId: string): void {
     const existing = ls.getItem(key);
     if (existing && existing.startsWith(holderId + ":")) ls.removeItem(key);
 }
+
+// ── File-based lock (cross-process / shared storage) ──────────────────────────────────────────────
+const FILE_LOCK_KEY = ".merge-lock";
+const FILE_LOCK_TTL_MS = 5 * 60 * 1000;
+const FILE_LOCK_SETTLE_MS = 15 * 1000;
+const FILE_LOCK_HEARTBEAT_MS = 2 * 60 * 1000;
+
+async function readMergeFileLock(storage: FileStorage): Promise<{ id: string; time: number } | undefined> {
+    try {
+        const buf = await storage.get(FILE_LOCK_KEY);
+        if (!buf) return undefined;
+        const s = buf.toString("utf8");
+        const colon = s.lastIndexOf(":");
+        if (colon < 0) return undefined;
+        const time = parseInt(s.slice(colon + 1), 10);
+        if (!Number.isFinite(time)) return undefined;
+        return { id: s.slice(0, colon), time };
+    } catch { return undefined; }
+}
+
+async function writeMergeFileLock(storage: FileStorage, holderId: string, time: number): Promise<void> {
+    await storage.set(FILE_LOCK_KEY, Buffer.from(`${holderId}:${time}`, "utf8") as Buffer);
+}
+
+// Acquire the cross-process file lock. Resolves to true ONLY if, after writing our ID and waiting
+// FILE_LOCK_SETTLE_MS for concurrent writers to settle, the file still names us. Caller must
+// releaseMergeFileLock() on the same storage + holderId when done.
+export async function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean> {
+    const now = Date.now();
+    const existing = await readMergeFileLock(storage);
+    if (existing && existing.id !== holderId && now - existing.time < FILE_LOCK_TTL_MS) return false;
+    await writeMergeFileLock(storage, holderId, now);
+    await new Promise(r => setTimeout(r, FILE_LOCK_SETTLE_MS));
+    const after = await readMergeFileLock(storage);
+    return !!after && after.id === holderId;
+}
+
+// Keep the lock fresh. The merge might run longer than FILE_LOCK_TTL_MS; without periodic re-stamping,
+// another process would see the lock as stale and acquire it. Returns a stop function the caller calls
+// in `finally`. Heartbeat is best-effort: a failed write is logged but not propagated.
+export function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void {
+    const interval = setInterval(() => {
+        void writeMergeFileLock(storage, holderId, Date.now()).catch(() => { /* best-effort */ });
+    }, FILE_LOCK_HEARTBEAT_MS);
+    (interval as { unref?: () => void }).unref?.();
+    return () => clearInterval(interval);
+}
+
+export async function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void> {
+    try {
+        const existing = await readMergeFileLock(storage);
+        if (existing && existing.id === holderId) await storage.remove(FILE_LOCK_KEY);
+    } catch { /* ignore */ }
+}

package/storage/remoteFileServer.ts

@@ -22,176 +22,176 @@ import { wrapHandle, NodeJSDirectoryHandleWrapper, DirectoryWrapper } from "./Fi
 
 // Common, memorable words (frequency-ordered subset) for passphrase generation.
 const WORDS: string[] = [
-    "that","with","which","this","from","have","they","were","their","there","when","them","said","been",
-    "would","will","what","more","then","into","some","could","time","very","than","your","other","upon",
-    "about","only","little","like","these","great","after","well","made","before","such","over","should",
-    "first","good","must","much","down","where","know","most","here","come","those","never","life","long",
-    "came","being","many","through","even","himself","back","every","shall","again","make","without","might",
-    "while","same","under","just","still","people","place","think","house","take","last","found","away",
-    "hand","went","thought","also","though","three","another","eyes","years","work","right","once","night",
-    "young","nothing","against","small","head","left","part","ever","world","each","father","give","between",
-    "face","king","things","love","because","took","always","tell","called","water","both","side","look",
-    "having","room","mind","half","heart","name","home","country","whole","however","find","among","going",
-    "thing","lord","looked","seen","mother","general","done","seemed","told","whom","days","soon","better",
-    "letter","woman","heard","asked","course","thus","moment","knew","light","enough","white","almost",
-    "until","quite","hands","words","death","large","taken","since","gave","given","best","state","brought",
-    "does","whose","door","others","power","perhaps","present","next","morning","poor","lady","four","high",
-    "year","turned","less","word","full","during","rather","want","order","near","feet","true","miss",
-    "matter","began","cannot","used","known","felt","above","round","thou","voice","till","case","nature",
-    "indeed","church","kind","certain","fire","often","stood","fact","friend","girl","five","land","says",
-    "john","myself","along","point","dear","wife","city","within","sent","times","keep","passed","form",
-    "second","body","money","believe","hundred","open","several","means","child","english","herself","sure",
-    "looking","women","already","black","alone","least","gone","held","itself","whether","hope","river",
-    "ground","either","number","chapter","england","leave","rest","town","hear","greek","friends","book",
-    "hour","short","cried","read","behind","became","making","family","earth","captain","around","dead",
-    "reason","call","become","lost","line","replied","help","coming","speak","manner","french","twenty",
-    "spirit","really","early","story","hard","close","human","public","truth","strong","master","care",
-    "towards","history","kept","later","states","dark","able","mean","return","brother","person","subject",
-    "soul","party","arms","thee","seems","common","fell","fine","feel","show","table","turn","wish","evening",
-    "free","cause","south","ready","north","across","rose","live","account","doubt","company","miles","road",
-    "bring","london","sense","horse","carried","hold","sight","fear","answer","idea","force","need","deep",
-    "further","nearly","past","army","blood","street","court","reached","view","school","sort","taking",
-    "else","chief","hours","beyond","cold","none","longer","strange","fellow","clear","service","natural",
-    "suppose","late","talk","front","stand","purpose","seem","neither","ought","west","real","except","sound",
-    "gold","forward","feeling","added","boys","self","peace","happy","living","husband","toward","spoke",
-    "fair","france","trees","effect","latter","length","change","died","green","united","fall","pretty",
-    "placed","meet","forth","office","comes","pass","written","ship","enemy","saying","tree","foot","blue",
-    "note","prince","hair","heaven","wild","play","entered","society","laid","wind","doing","paper","opened",
-    "bear","third","queen","mine","greater","various","faith","wanted","boat","stone","lived","george",
-    "window","doctor","action","books","tried","letters","makes","minutes","parts","wood","period","duty",
-    "york","instead","heavy","persons","battle","field","british","object","century","island","beauty",
-    "christ","sister","glad","below","east","opinion","save","places","months","food","sweet","trouble",
-    "system","born","desire","works","mary","chance","william","seven","single","hardly","sleep","mouth",
-    "horses","silence","ancient","broken","hall","send","rich","girls","besides","lips","henry","figure",
-    "slowly","hill","wall","future","lines","follow","german","march","filled","brown","deal","eight",
-    "covered","smile","former","week","drew","easy","paris","camp","stay","cross","seeing","result","started",
-    "caught","houses","appear","wrote","giving","simple","arrived","formed","bright","wide","uncle","piece",
-    "walked","knows","carry","middle","village","holy","merely","getting","raised","afraid","struck",
-    "charles","board","visit","royal","spring","dinner","evil","outside","wrong","summer","moved","danger",
-    "mark","fresh","plain","thirty","scene","quickly","wonder","jack","indian","walk","learned","class",
-    "secret","wait","command","easily","garden","loved","married","fight","leaving","music","usual","cases",
-    "winter","respect","value","quiet","please","stopped","write","laughed","america","chair","paid","duke",
-    "leaves","lower","colonel","perfect","james","worth","author","finally","youth","regard","glass",
-    "picture","built","modern","success","race","showed","tears","unless","mere","lives","grew","charge",
-    "beside","remain","waiting","study","hath","shot","unto","tone","iron","watch","grace","flowers","killed",
-    "allowed","news","step","proper","sitting","floor","justice","noble","goes","walls","laws","meant",
-    "bound","forms","fifty","drawn","private","fast","indians","judge","meeting","usually","sudden","gives",
-    "reach","bank","attempt","shore","stop","plan","silent","special","spot","officer","silver","passing",
-    "broke","lake","soft","journey","beneath","shown","turning","members","enter","lead","trade","names",
-    "escape","troops","bill","corner","rule","ladies","species","rock","similar","running","rate","cast",
-    "nation","post","likely","simply","orders","dress","fish","minute","birds","europe","passage","surface",
-    "snow","attack","higher","rise","grand","reply","honour","notice","speech","trying","game","writing",
-    "closed","rome","example","aunt","learn","morrow","offered","peter","equal","space","page","wished",
-    "changed","animal","social","twelve","appears","receive","valley","degree","train","steps","fixed",
-    "count","forest","matters","foreign","native","broad","moral","animals","safe","roman","break","pale",
-    "memory","warm","trust","yellow","sides","main","bird","reading","coast","pain","grave","forget","move",
-    "fortune","madame","proved","forty","lying","quick","wise","jesus","working","surely","looks","seat",
-    "divine","touch","loss","paul","heads","spent","ordered","report","caused","thomas","ones","drawing",
-    "talking","breath","meaning","month","union","pleased","powers","emperor","laugh","affairs","narrow",
-    "square","science","begin","promise","takes","conduct","serious","thank","curious","pieces","health",
-    "exactly","ways","upper","decided","nine","stream","wine","engaged","points","amount","named","song",
-    "worse","size","castle","crown","mass","liberty","stage","guard","servant","greatly","price","weeks",
-    "neck","ears","drink","crowd","thick","fallen","hills","spoken","golden","capital","sign","spite","terms",
-    "sake","council","threw","ships","portion","support","clothes","effort","fully","holding","majesty",
-    "glory","pure","facts","sword","sorry","fate","bread","prove","station","sharp","dream","vain","major",
-    "smiled","instant","spread","serve","sick","june","ideas","thrown","start","weather","gray","courage",
-    "anxious","woods","path","rising","grass","moon","gate","forced","fancy","bottom","expect","remains",
-    "shook","bishop","watched","settled","events","rain","quarter","heat","palace","glance","kingdom",
-    "papers","aside","worthy","taste","pride","july","opening","daily","leading","wounded","famous","offer",
-    "group","distant","weight","highest","vast","popular","passion","knowing","allow","sought","lies",
-    "marked","series","growing","measure","hearts","robert","obliged","western","hence","style","dropped",
-    "nations","grown","honor","edge","pray","tall","frank","poet","streets","season","pointed","mighty",
-    "temple","extent","thin","blow","freedom","entire","keeping","prevent","shut","storm","lose","college",
-    "cost","marry","spirits","worked","colour","ring","listen","fruit","waters","fashion","share","hung",
-    "proud","fingers","method","served","grow","fort","draw","dollars","quietly","yours","vessel","direct",
-    "refused","bridge","gods","inside","title","advance","priest","becomes","dick","double","seek","guess",
-    "spanish","larger","ball","finding","edward","removed","brave","tired","agreed","sacred","sons","guns",
-    "played","richard","devil","pounds","honest","false","cloth","soldier","tongue","smith","wealth","failed",
-    "height","faces","equally","legs","pocket","twice","volume","prayer","county","notes","louis","unknown",
-    "delight","rights","nice","taught","coat","dare","slight","rocks","enemies","control","forces","germany",
-    "kill","process","david","rapidly","comfort","islands","centre","salt","april","windows","noticed",
-    "truly","color","fields","date","august","desired","breast","skin","gentle","smoke","search","joined",
-    "nobody","results","needed","weak","type","stones","follows","theory","shape","waited","telling","relief",
-    "bearing","bent","mile","dressed","birth","spain","female","clearly","moving","drive","crossed","member",
-    "saved","teeth","flesh","cover","shows","farther","stands","figures","numbers","bodies","supply","motion",
-    "grey","bell","alive","seized","shadow","produce","touched","driven","talked","empire","sunday","pair",
-    "fourth","slave","regular","dozen","beat","cousin","sand","soil","rough","claim","angry","pity","walking",
-    "acts","pope","rooms","task","stock","tender","throw","reader","india","hotel","fifteen","arrival",
-    "plate","stars","willing","stories","saint","amongst","virtue","chamber","civil","yards","habit","writer",
-    "putting","origin","italy","hearing","genius","milk","busy","fool","burning","duties","brain","slow",
-    "belief","bore","mention","grant","library","inches","press","calm","total","ride","lands","labor",
-    "parties","clean","catch","treated","rode","whilst","level","efforts","minds","welcome","wisdom","supper",
-    "final","mercy","aware","lovely","empty","wants","midst","central","rank","proof","burst","term","farm",
-    "applied","loud","nearer","harry","closely","kings","explain","deck","noise","hurt","advice","absence",
-    "circle","objects","secure","plants","parents","falling","base","fellows","sorrow","flat","flower",
-    "calling","imagine","range","sold","favour","happen","showing","earl","hole","careful","dust","prison",
-    "useful","accept","firm","sing","port","seated","custom","wore","doors","lifted","edition","tale",
-    "affair","policy","roof","express","ahead","worship","partly","address","highly","victory","ocean",
-    "begun","buried","content","smiling","liked","active","quality","philip","current","divided","growth",
-    "huge","moments","article","speed","poetry","machine","join","nose","unable","ceased","gained","worn",
-    "eastern","safety","ages","vessels","hopes","corn","slaves","drop","plant","evident","local","police",
-    "october","obtain","italian","deeply","dying","wholly","dogs","playing","plenty","apart","suffer","maid",
-    "record","fairly","kindly","terror","drove","ends","sugar","capable","irish","message","chosen","flight",
-    "reasons","couple","meat","printed","blind","energy","bitter","baby","mistake","tower","ireland","trial",
-    "wear","brief","market","band","causes","clouds","goods","admit","cities","earlier","tells","stated",
-    "pressed","crime","fought","guide","hoped","list","banks","knight","fleet","pulled","tail","patient",
-    "mental","boats","kinds","stairs","star","cattle","rare","wings","disease","section","actual","bare",
-    "extreme","cruel","worst","escaped","dance","strike","views","eggs","needs","store","choice","fond",
-    "adopted","suit","singing","fail","souls","thanks","hidden","shame","faint","shop","older","joseph",
-    "knees","gently","blessed","yard","cent","grief","male","sooner","january","excited","region","praise",
-    "throne","classes","effects","demand","gain","fault","labour","variety","loose","cook","devoted","ended",
-    "changes","witness","bought","lover","visited","club","sheep","cloud","enjoy","asleep","cool","dull",
-    "painted","arose","armed","dignity","chiefly","voices","sail","event","signs","hero","schools","branch",
-    "hurried","arthur","kitchen","stick","coffee","thinks","eager","natives","flying","boston","eternal",
-    "source","fill","anger","vision","murder","viii","park","avoid","awful","raise","desert","plans","pardon",
-    "assured","wound","finger","bible","rear","details","cabin","whence","reign","weary","slavery","visible",
-    "shouted","seldom","skill","hast","throat","reality","leader","egypt","credit","smaller","severe","calls",
-    "younger","shell","sprang","anybody","handed","despair","asking","inch","mystery","manners","knife",
-    "design","sounds","wishes","stepped","towns","sixty","immense","china","favor","latin","hate","setting",
-    "excuse","chinese","behold","rushed","choose","ease","lights","paused","mission","voyage","gift","bold",
-    "meal","britain","smooth","mounted","utterly","lest","helped","picked","falls","pipe","bones","earnest",
-    "granted","swept","anxiety","teach","waves","softly","savage","hunting","defence","rapid","artist",
-    "recent","supreme","russian","created","habits","exist","guilty","pages","crew","hell","remark","request",
-    "harm","solemn","observe","trail","copy","violent","dreams","proceed","luck","steel","sell","temper",
-    "appeal","hide","fled","belong","triumph","abroad","waste","consent","writers","possess","jews","require",
-    "priests","railway","founded","pushed","host","solid","maybe","degrees","cheeks","mount","ruin","owing",
-    "fierce","reduced","mankind","text","swift","riding","silk","shade","career","wicked","afford","alas",
-    "rules","treaty","correct","spend","foolish","methods","remarks","horror","shining","enjoyed","tribes",
-    "lack","frame","gets","eagerly","russia","alike","somehow","nodded","problem","fever","sees","stern",
-    "dawn","forgive","flag","managed","fame","travel","estate","cotton","hurry","agree","feared","kiss",
-    "million","martin","mixed","risk","butter","jane","assumed","pause","benefit","unhappy","lighted",
-    "scheme","slept","plainly","forgot","delay","merry","rush","wooden","secured","poem","rolled","angel",
-    "guests","farmer","humble","glanced","shortly","rope","image","lately","africa","fired","retired","bull",
-    "steam","keen","loving","borne","readily","beloved","average","teacher","attend","flame","area","burned",
-    "thrust","negro","nurse","spare","fifth","thence","roads","refuse","tent","kissed","gates","gaze","fatal",
-    "israel","verse","scenes","confess","uttered","build","acted","hanging","reward","issue","utmost",
-    "fathers","noted","widow","related","urged","mode","failure","nervous","render","masters","tribe",
-    "colored","turns","alarm","rivers","using","eleven","rage","hers","lamp","retreat","amid","gospel",
-    "exposed","johnson","marks","tide","emotion","destroy","inner","sisters","lion","helen","tied","grounds",
-    "acid","wheel","issued","invited","gazed","senate","finds","seed","regret","clay","chain","crowded",
-    "bosom","limited","steady","chap","anne","francis","cavalry","freely","charm","faced","ideal","useless",
-    "warning","shelter","vote","xpage","cottage","beast","outer","folks","misery","anyone","expense","firmly",
-    "eating","lonely","owner","trace","coal","hastily","elder","utter","begins","chapel","nights","dared",
-    "signal","stared","track","lords","speaks","bottle","blame","claims","shoes","sigh","ghost","folk",
-    "dutch","flung","flew","cheek","staff","walter","clever","acting","hungry","poems","cutting","forever",
-    "exact","haste","dancing","trip","lincoln","pull","vice","slipped","thine","loves","permit","mill",
-    "engine","hollow","seeking","bowed","largely","charged","painful","madam","roll","leaf","prayers",
-    "element","agent","cries","songs","theatre","creek","match","ashamed","runs","aspect","canada","treat",
-    "legal","arts","corps","noon","nearest","scale","hunt","shadows","winds","landed","beings","tiny",
-    "gardens","bless","clerk","doth","derived","hang","heavens","cape","ours","brow","test","senses",
-    "opposed","error","driving","nest","headed","groups","princes","feast","admiral","poured","brings",
-    "pursued","germans","bears","lesson","journal","unusual","marched","wing","remove","studied","passes",
-    "actions","burden","colony","scott","bride","yield","crying","forming","rifle","powder","rested","shake",
-    "guest","begged","occur","altar","sorts","beaten","wave","papa","sang","depth","aloud","contact",
-    "intense","marble","poverty","detail","writes","root","metal","jean","existed","ability","purple",
-    "counsel","lane","wives","sending","heavily","leaders","queer","tales","sins","mexico","protect","clock",
-    "stuff","jones","pine","records","cave","baron","medical","pick","stayed","magic","seventy","pour",
-    "saddle","sank","dread","deny","wire","rude","holds","strain","autumn","shed","shoot","settle","basis",
-    "readers","route","beach","safely","oxford","notion","thunder","sale","saints","pound","shock","elected",
-    "signed","whereas","maiden","courts","rarely","wolf","mamma","billy","student","charity","impulse",
-    "tones","mortal","raising","wedding","belongs","chest","sharply","sounded","poets","lawyer","hugh",
-    "plays","awake","landing","debt","alice","venture","idle","uniform","contain","tobacco","boots","fears",
-    "leaned","studies","customs","prize","stir","obey","oath","badly","pursuit","resting",
+    "that", "with", "which", "this", "from", "have", "they", "were", "their", "there", "when", "them", "said", "been",
+    "would", "will", "what", "more", "then", "into", "some", "could", "time", "very", "than", "your", "other", "upon",
+    "about", "only", "little", "like", "these", "great", "after", "well", "made", "before", "such", "over", "should",
+    "first", "good", "must", "much", "down", "where", "know", "most", "here", "come", "those", "never", "life", "long",
+    "came", "being", "many", "through", "even", "himself", "back", "every", "shall", "again", "make", "without", "might",
+    "while", "same", "under", "just", "still", "people", "place", "think", "house", "take", "last", "found", "away",
+    "hand", "went", "thought", "also", "though", "three", "another", "eyes", "years", "work", "right", "once", "night",
+    "young", "nothing", "against", "small", "head", "left", "part", "ever", "world", "each", "father", "give", "between",
+    "face", "king", "things", "love", "because", "took", "always", "tell", "called", "water", "both", "side", "look",
+    "having", "room", "mind", "half", "heart", "name", "home", "country", "whole", "however", "find", "among", "going",
+    "thing", "lord", "looked", "seen", "mother", "general", "done", "seemed", "told", "whom", "days", "soon", "better",
+    "letter", "woman", "heard", "asked", "course", "thus", "moment", "knew", "light", "enough", "white", "almost",
+    "until", "quite", "hands", "words", "death", "large", "taken", "since", "gave", "given", "best", "state", "brought",
+    "does", "whose", "door", "others", "power", "perhaps", "present", "next", "morning", "poor", "lady", "four", "high",
+    "year", "turned", "less", "word", "full", "during", "rather", "want", "order", "near", "feet", "true", "miss",
+    "matter", "began", "cannot", "used", "known", "felt", "above", "round", "thou", "voice", "till", "case", "nature",
+    "indeed", "church", "kind", "certain", "fire", "often", "stood", "fact", "friend", "girl", "five", "land", "says",
+    "john", "myself", "along", "point", "dear", "wife", "city", "within", "sent", "times", "keep", "passed", "form",
+    "second", "body", "money", "believe", "hundred", "open", "several", "means", "child", "english", "herself", "sure",
+    "looking", "women", "already", "black", "alone", "least", "gone", "held", "itself", "whether", "hope", "river",
+    "ground", "either", "number", "chapter", "england", "leave", "rest", "town", "hear", "greek", "friends", "book",
+    "hour", "short", "cried", "read", "behind", "became", "making", "family", "earth", "captain", "around", "dead",
+    "reason", "call", "become", "lost", "line", "replied", "help", "coming", "speak", "manner", "french", "twenty",
+    "spirit", "really", "early", "story", "hard", "close", "human", "public", "truth", "strong", "master", "care",
+    "towards", "history", "kept", "later", "states", "dark", "able", "mean", "return", "brother", "person", "subject",
+    "soul", "party", "arms", "thee", "seems", "common", "fell", "fine", "feel", "show", "table", "turn", "wish", "evening",
+    "free", "cause", "south", "ready", "north", "across", "rose", "live", "account", "doubt", "company", "miles", "road",
+    "bring", "london", "sense", "horse", "carried", "hold", "sight", "fear", "answer", "idea", "force", "need", "deep",
+    "further", "nearly", "past", "army", "blood", "street", "court", "reached", "view", "school", "sort", "taking",
+    "else", "chief", "hours", "beyond", "cold", "none", "longer", "strange", "fellow", "clear", "service", "natural",
+    "suppose", "late", "talk", "front", "stand", "purpose", "seem", "neither", "ought", "west", "real", "except", "sound",
+    "gold", "forward", "feeling", "added", "boys", "self", "peace", "happy", "living", "husband", "toward", "spoke",
+    "fair", "france", "trees", "effect", "latter", "length", "change", "died", "green", "united", "fall", "pretty",
+    "placed", "meet", "forth", "office", "comes", "pass", "written", "ship", "enemy", "saying", "tree", "foot", "blue",
+    "note", "prince", "hair", "heaven", "wild", "play", "entered", "society", "laid", "wind", "doing", "paper", "opened",
+    "bear", "third", "queen", "mine", "greater", "various", "faith", "wanted", "boat", "stone", "lived", "george",
+    "window", "doctor", "action", "books", "tried", "letters", "makes", "minutes", "parts", "wood", "period", "duty",
+    "york", "instead", "heavy", "persons", "battle", "field", "british", "object", "century", "island", "beauty",
+    "christ", "sister", "glad", "below", "east", "opinion", "save", "places", "months", "food", "sweet", "trouble",
+    "system", "born", "desire", "works", "mary", "chance", "william", "seven", "single", "hardly", "sleep", "mouth",
+    "horses", "silence", "ancient", "broken", "hall", "send", "rich", "girls", "besides", "lips", "henry", "figure",
+    "slowly", "hill", "wall", "future", "lines", "follow", "german", "march", "filled", "brown", "deal", "eight",
+    "covered", "smile", "former", "week", "drew", "easy", "paris", "camp", "stay", "cross", "seeing", "result", "started",
+    "caught", "houses", "appear", "wrote", "giving", "simple", "arrived", "formed", "bright", "wide", "uncle", "piece",
+    "walked", "knows", "carry", "middle", "village", "holy", "merely", "getting", "raised", "afraid", "struck",
+    "charles", "board", "visit", "royal", "spring", "dinner", "evil", "outside", "wrong", "summer", "moved", "danger",
+    "mark", "fresh", "plain", "thirty", "scene", "quickly", "wonder", "jack", "indian", "walk", "learned", "class",
+    "secret", "wait", "command", "easily", "garden", "loved", "married", "fight", "leaving", "music", "usual", "cases",
+    "winter", "respect", "value", "quiet", "please", "stopped", "write", "laughed", "america", "chair", "paid", "duke",
+    "leaves", "lower", "colonel", "perfect", "james", "worth", "author", "finally", "youth", "regard", "glass",
+    "picture", "built", "modern", "success", "race", "showed", "tears", "unless", "mere", "lives", "grew", "charge",
+    "beside", "remain", "waiting", "study", "hath", "shot", "unto", "tone", "iron", "watch", "grace", "flowers", "killed",
+    "allowed", "news", "step", "proper", "sitting", "floor", "justice", "noble", "goes", "walls", "laws", "meant",
+    "bound", "forms", "fifty", "drawn", "private", "fast", "indians", "judge", "meeting", "usually", "sudden", "gives",
+    "reach", "bank", "attempt", "shore", "stop", "plan", "silent", "special", "spot", "officer", "silver", "passing",
+    "broke", "lake", "soft", "journey", "beneath", "shown", "turning", "members", "enter", "lead", "trade", "names",
+    "escape", "troops", "bill", "corner", "rule", "ladies", "species", "rock", "similar", "running", "rate", "cast",
+    "nation", "post", "likely", "simply", "orders", "dress", "fish", "minute", "birds", "europe", "passage", "surface",
+    "snow", "attack", "higher", "rise", "grand", "reply", "honour", "notice", "speech", "trying", "game", "writing",
+    "closed", "rome", "example", "aunt", "learn", "morrow", "offered", "peter", "equal", "space", "page", "wished",
+    "changed", "animal", "social", "twelve", "appears", "receive", "valley", "degree", "train", "steps", "fixed",
+    "count", "forest", "matters", "foreign", "native", "broad", "moral", "animals", "safe", "roman", "break", "pale",
+    "memory", "warm", "trust", "yellow", "sides", "main", "bird", "reading", "coast", "pain", "grave", "forget", "move",
+    "fortune", "madame", "proved", "forty", "lying", "quick", "wise", "jesus", "working", "surely", "looks", "seat",
+    "divine", "touch", "loss", "paul", "heads", "spent", "ordered", "report", "caused", "thomas", "ones", "drawing",
+    "talking", "breath", "meaning", "month", "union", "pleased", "powers", "emperor", "laugh", "affairs", "narrow",
+    "square", "science", "begin", "promise", "takes", "conduct", "serious", "thank", "curious", "pieces", "health",
+    "exactly", "ways", "upper", "decided", "nine", "stream", "wine", "engaged", "points", "amount", "named", "song",
+    "worse", "size", "castle", "crown", "mass", "liberty", "stage", "guard", "servant", "greatly", "price", "weeks",
+    "neck", "ears", "drink", "crowd", "thick", "fallen", "hills", "spoken", "golden", "capital", "sign", "spite", "terms",
+    "sake", "council", "threw", "ships", "portion", "support", "clothes", "effort", "fully", "holding", "majesty",
+    "glory", "pure", "facts", "sword", "sorry", "fate", "bread", "prove", "station", "sharp", "dream", "vain", "major",
+    "smiled", "instant", "spread", "serve", "sick", "june", "ideas", "thrown", "start", "weather", "gray", "courage",
+    "anxious", "woods", "path", "rising", "grass", "moon", "gate", "forced", "fancy", "bottom", "expect", "remains",
+    "shook", "bishop", "watched", "settled", "events", "rain", "quarter", "heat", "palace", "glance", "kingdom",
+    "papers", "aside", "worthy", "taste", "pride", "july", "opening", "daily", "leading", "wounded", "famous", "offer",
+    "group", "distant", "weight", "highest", "vast", "popular", "passion", "knowing", "allow", "sought", "lies",
+    "marked", "series", "growing", "measure", "hearts", "robert", "obliged", "western", "hence", "style", "dropped",
+    "nations", "grown", "honor", "edge", "pray", "tall", "frank", "poet", "streets", "season", "pointed", "mighty",
+    "temple", "extent", "thin", "blow", "freedom", "entire", "keeping", "prevent", "shut", "storm", "lose", "college",
+    "cost", "marry", "spirits", "worked", "colour", "ring", "listen", "fruit", "waters", "fashion", "share", "hung",
+    "proud", "fingers", "method", "served", "grow", "fort", "draw", "dollars", "quietly", "yours", "vessel", "direct",
+    "refused", "bridge", "gods", "inside", "title", "advance", "priest", "becomes", "dick", "double", "seek", "guess",
+    "spanish", "larger", "ball", "finding", "edward", "removed", "brave", "tired", "agreed", "sacred", "sons", "guns",
+    "played", "richard", "devil", "pounds", "honest", "false", "cloth", "soldier", "tongue", "smith", "wealth", "failed",
+    "height", "faces", "equally", "legs", "pocket", "twice", "volume", "prayer", "county", "notes", "louis", "unknown",
+    "delight", "rights", "nice", "taught", "coat", "dare", "slight", "rocks", "enemies", "control", "forces", "germany",
+    "kill", "process", "david", "rapidly", "comfort", "islands", "centre", "salt", "april", "windows", "noticed",
+    "truly", "color", "fields", "date", "august", "desired", "breast", "skin", "gentle", "smoke", "search", "joined",
+    "nobody", "results", "needed", "weak", "type", "stones", "follows", "theory", "shape", "waited", "telling", "relief",
+    "bearing", "bent", "mile", "dressed", "birth", "spain", "female", "clearly", "moving", "drive", "crossed", "member",
+    "saved", "teeth", "flesh", "cover", "shows", "farther", "stands", "figures", "numbers", "bodies", "supply", "motion",
+    "grey", "bell", "alive", "seized", "shadow", "produce", "touched", "driven", "talked", "empire", "sunday", "pair",
+    "fourth", "slave", "regular", "dozen", "beat", "cousin", "sand", "soil", "rough", "claim", "angry", "pity", "walking",
+    "acts", "pope", "rooms", "task", "stock", "tender", "throw", "reader", "india", "hotel", "fifteen", "arrival",
+    "plate", "stars", "willing", "stories", "saint", "amongst", "virtue", "chamber", "civil", "yards", "habit", "writer",
+    "putting", "origin", "italy", "hearing", "genius", "milk", "busy", "fool", "burning", "duties", "brain", "slow",
+    "belief", "bore", "mention", "grant", "library", "inches", "press", "calm", "total", "ride", "lands", "labor",
+    "parties", "clean", "catch", "treated", "rode", "whilst", "level", "efforts", "minds", "welcome", "wisdom", "supper",
+    "final", "mercy", "aware", "lovely", "empty", "wants", "midst", "central", "rank", "proof", "burst", "term", "farm",
+    "applied", "loud", "nearer", "harry", "closely", "kings", "explain", "deck", "noise", "hurt", "advice", "absence",
+    "circle", "objects", "secure", "plants", "parents", "falling", "base", "fellows", "sorrow", "flat", "flower",
+    "calling", "imagine", "range", "sold", "favour", "happen", "showing", "earl", "hole", "careful", "dust", "prison",
+    "useful", "accept", "firm", "sing", "port", "seated", "custom", "wore", "doors", "lifted", "edition", "tale",
+    "affair", "policy", "roof", "express", "ahead", "worship", "partly", "address", "highly", "victory", "ocean",
+    "begun", "buried", "content", "smiling", "liked", "active", "quality", "philip", "current", "divided", "growth",
+    "huge", "moments", "article", "speed", "poetry", "machine", "join", "nose", "unable", "ceased", "gained", "worn",
+    "eastern", "safety", "ages", "vessels", "hopes", "corn", "slaves", "drop", "plant", "evident", "local", "police",
+    "october", "obtain", "italian", "deeply", "dying", "wholly", "dogs", "playing", "plenty", "apart", "suffer", "maid",
+    "record", "fairly", "kindly", "terror", "drove", "ends", "sugar", "capable", "irish", "message", "chosen", "flight",
+    "reasons", "couple", "meat", "printed", "blind", "energy", "bitter", "baby", "mistake", "tower", "ireland", "trial",
+    "wear", "brief", "market", "band", "causes", "clouds", "goods", "admit", "cities", "earlier", "tells", "stated",
+    "pressed", "crime", "fought", "guide", "hoped", "list", "banks", "knight", "fleet", "pulled", "tail", "patient",
+    "mental", "boats", "kinds", "stairs", "star", "cattle", "rare", "wings", "disease", "section", "actual", "bare",
+    "extreme", "cruel", "worst", "escaped", "dance", "strike", "views", "eggs", "needs", "store", "choice", "fond",
+    "adopted", "suit", "singing", "fail", "souls", "thanks", "hidden", "shame", "faint", "shop", "older", "joseph",
+    "knees", "gently", "blessed", "yard", "cent", "grief", "male", "sooner", "january", "excited", "region", "praise",
+    "throne", "classes", "effects", "demand", "gain", "fault", "labour", "variety", "loose", "cook", "devoted", "ended",
+    "changes", "witness", "bought", "lover", "visited", "club", "sheep", "cloud", "enjoy", "asleep", "cool", "dull",
+    "painted", "arose", "armed", "dignity", "chiefly", "voices", "sail", "event", "signs", "hero", "schools", "branch",
+    "hurried", "arthur", "kitchen", "stick", "coffee", "thinks", "eager", "natives", "flying", "boston", "eternal",
+    "source", "fill", "anger", "vision", "murder", "viii", "park", "avoid", "awful", "raise", "desert", "plans", "pardon",
+    "assured", "wound", "finger", "bible", "rear", "details", "cabin", "whence", "reign", "weary", "slavery", "visible",
+    "shouted", "seldom", "skill", "hast", "throat", "reality", "leader", "egypt", "credit", "smaller", "severe", "calls",
+    "younger", "shell", "sprang", "anybody", "handed", "despair", "asking", "inch", "mystery", "manners", "knife",
+    "design", "sounds", "wishes", "stepped", "towns", "sixty", "immense", "china", "favor", "latin", "hate", "setting",
+    "excuse", "chinese", "behold", "rushed", "choose", "ease", "lights", "paused", "mission", "voyage", "gift", "bold",
+    "meal", "britain", "smooth", "mounted", "utterly", "lest", "helped", "picked", "falls", "pipe", "bones", "earnest",
+    "granted", "swept", "anxiety", "teach", "waves", "softly", "savage", "hunting", "defence", "rapid", "artist",
+    "recent", "supreme", "russian", "created", "habits", "exist", "guilty", "pages", "crew", "hell", "remark", "request",
+    "harm", "solemn", "observe", "trail", "copy", "violent", "dreams", "proceed", "luck", "steel", "sell", "temper",
+    "appeal", "hide", "fled", "belong", "triumph", "abroad", "waste", "consent", "writers", "possess", "jews", "require",
+    "priests", "railway", "founded", "pushed", "host", "solid", "maybe", "degrees", "cheeks", "mount", "ruin", "owing",
+    "fierce", "reduced", "mankind", "text", "swift", "riding", "silk", "shade", "career", "wicked", "afford", "alas",
+    "rules", "treaty", "correct", "spend", "foolish", "methods", "remarks", "horror", "shining", "enjoyed", "tribes",
+    "lack", "frame", "gets", "eagerly", "russia", "alike", "somehow", "nodded", "problem", "fever", "sees", "stern",
+    "dawn", "forgive", "flag", "managed", "fame", "travel", "estate", "cotton", "hurry", "agree", "feared", "kiss",
+    "million", "martin", "mixed", "risk", "butter", "jane", "assumed", "pause", "benefit", "unhappy", "lighted",
+    "scheme", "slept", "plainly", "forgot", "delay", "merry", "rush", "wooden", "secured", "poem", "rolled", "angel",
+    "guests", "farmer", "humble", "glanced", "shortly", "rope", "image", "lately", "africa", "fired", "retired", "bull",
+    "steam", "keen", "loving", "borne", "readily", "beloved", "average", "teacher", "attend", "flame", "area", "burned",
+    "thrust", "negro", "nurse", "spare", "fifth", "thence", "roads", "refuse", "tent", "kissed", "gates", "gaze", "fatal",
+    "israel", "verse", "scenes", "confess", "uttered", "build", "acted", "hanging", "reward", "issue", "utmost",
+    "fathers", "noted", "widow", "related", "urged", "mode", "failure", "nervous", "render", "masters", "tribe",
+    "colored", "turns", "alarm", "rivers", "using", "eleven", "rage", "hers", "lamp", "retreat", "amid", "gospel",
+    "exposed", "johnson", "marks", "tide", "emotion", "destroy", "inner", "sisters", "lion", "helen", "tied", "grounds",
+    "acid", "wheel", "issued", "invited", "gazed", "senate", "finds", "seed", "regret", "clay", "chain", "crowded",
+    "bosom", "limited", "steady", "chap", "anne", "francis", "cavalry", "freely", "charm", "faced", "ideal", "useless",
+    "warning", "shelter", "vote", "xpage", "cottage", "beast", "outer", "folks", "misery", "anyone", "expense", "firmly",
+    "eating", "lonely", "owner", "trace", "coal", "hastily", "elder", "utter", "begins", "chapel", "nights", "dared",
+    "signal", "stared", "track", "lords", "speaks", "bottle", "blame", "claims", "shoes", "sigh", "ghost", "folk",
+    "dutch", "flung", "flew", "cheek", "staff", "walter", "clever", "acting", "hungry", "poems", "cutting", "forever",
+    "exact", "haste", "dancing", "trip", "lincoln", "pull", "vice", "slipped", "thine", "loves", "permit", "mill",
+    "engine", "hollow", "seeking", "bowed", "largely", "charged", "painful", "madam", "roll", "leaf", "prayers",
+    "element", "agent", "cries", "songs", "theatre", "creek", "match", "ashamed", "runs", "aspect", "canada", "treat",
+    "legal", "arts", "corps", "noon", "nearest", "scale", "hunt", "shadows", "winds", "landed", "beings", "tiny",
+    "gardens", "bless", "clerk", "doth", "derived", "hang", "heavens", "cape", "ours", "brow", "test", "senses",
+    "opposed", "error", "driving", "nest", "headed", "groups", "princes", "feast", "admiral", "poured", "brings",
+    "pursued", "germans", "bears", "lesson", "journal", "unusual", "marched", "wing", "remove", "studied", "passes",
+    "actions", "burden", "colony", "scott", "bride", "yield", "crying", "forming", "rifle", "powder", "rested", "shake",
+    "guest", "begged", "occur", "altar", "sorts", "beaten", "wave", "papa", "sang", "depth", "aloud", "contact",
+    "intense", "marble", "poverty", "detail", "writes", "root", "metal", "jean", "existed", "ability", "purple",
+    "counsel", "lane", "wives", "sending", "heavily", "leaders", "queer", "tales", "sins", "mexico", "protect", "clock",
+    "stuff", "jones", "pine", "records", "cave", "baron", "medical", "pick", "stayed", "magic", "seventy", "pour",
+    "saddle", "sank", "dread", "deny", "wire", "rude", "holds", "strain", "autumn", "shed", "shoot", "settle", "basis",
+    "readers", "route", "beach", "safely", "oxford", "notion", "thunder", "sale", "saints", "pound", "shock", "elected",
+    "signed", "whereas", "maiden", "courts", "rarely", "wolf", "mamma", "billy", "student", "charity", "impulse",
+    "tones", "mortal", "raising", "wedding", "belongs", "chest", "sharply", "sounded", "poets", "lawyer", "hugh",
+    "plays", "awake", "landing", "debt", "alice", "venture", "idle", "uniform", "contain", "tobacco", "boots", "fears",
+    "leaned", "studies", "customs", "prize", "stir", "obey", "oath", "badly", "pursuit", "resting",
 ];
 
 export function generatePassword(wordCount: number): string {
@@ -493,6 +493,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
                 ws.send(Buffer.concat([len, h, body || EMPTY]));
             };
             void (async () => {
+                // TODO: Fix this code. It's some of the worst code I've ever seen. All the try-catches are bad. There should just be a single try-catch. The comparisons are bad. They're pretending like null can exist in places when it can't. We're using the any type for no reason.
                 let header: any;
                 let body: Buffer;
                 try {
@@ -608,9 +609,7 @@ export async function autocompactBulkDatabases(root: string): Promise<void> {
         } catch { continue; } // no bulkDatabases2 here
         for (const name of names) {
             try {
-                const t = Date.now();
-                const res = await getCompactor(baseDir, name).tryMergeNow();
-                if (res.merged) console.log(`  [autocompact] ${name}: merged (${Date.now() - t}ms)`);
+                await getCompactor(baseDir, name).tryMergeNow();
             } catch (e) {
                 console.warn(`  [autocompact] ${name}: failed — ${(e as Error).message}`);
             }