sliftutils 1.7.9 → 1.7.11

package/storage/remoteStorage/blobStore.ts

@@ -1,362 +1,802 @@
-import fs from "fs";
 import path from "path";
 import { lazy } from "socket-function/src/caching";
-import { runInSerial, runInfinitePoll } from "socket-function/src/batching";
-import { timeInMinute, sort } from "socket-function/src/misc";
+import { runInfinitePoll, delay } from "socket-function/src/batching";
+import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
 import { formatNumber } from "socket-function/src/formatting/format";
-import { TransactionStorage } from "../TransactionStorage";
-import { JSONStorage } from "../JSONStorage";
-import { getFileStorageNested2 } from "../FileFolderAPI";
-import { ArchiveFileInfo } from "../IArchives";
-
-// Disk engine for the remote storage server. Files are packed into large append-only blob files
-// (instead of one file on disk per stored file), with a transaction-log index mapping
-// key -> (blob, offset, length). This keeps us efficient with many small files and scales to
-// terabytes: reads are single pread calls, writes are appends, and deleted space is reclaimed by
-// compacting blobs that are mostly dead.
-
-// Roll to a new blob file once the current one reaches this size
-const MAX_BLOB_SIZE = 4 * 1024 * 1024 * 1024;
-// Whole-file memory cache (only files up to MEMORY_CACHE_MAX_FILE are cached)
-const MEMORY_CACHE_BYTES = 256 * 1024 * 1024;
-const MEMORY_CACHE_MAX_FILE = 16 * 1024 * 1024;
+import {
+    IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, assertValidLastModified,
+    windowAcceptsWrites, SyncActivity,
+} from "../IArchives";
+import { ArchivesDisk, applyFindInfoShape } from "../ArchivesDisk";
+import { ArchivesBackblaze } from "../backblaze";
+import { ROUTING_FILE, getRoute, routeContains } from "./remoteConfig";
+import { BulkDatabaseBase, noopReactiveDeps } from "../BulkDatabase2/BulkDatabaseBase";
+import { wrapHandle, NodeJSDirectoryHandleWrapper, DirectoryWrapper } from "../FileFolderAPI";
+
+// The storage engine of the remote storage server. Data lives in synchronization sources (at
+// minimum an ArchivesDisk, the local disk); BlobStore keeps an index of every file (path, last
+// modified time, size, and which source currently holds the data) in a BulkDatabase2, and
+// synchronizes the index from all sources (see ArchivesSource in IArchives.ts).
+// Every startup fully rescans each source's metadata, so the index self-heals; the file with the
+// highest write time wins across all sources, so multiple sources need no stacking order.
+
 export const DEFAULT_FAST_WRITE_DELAY = timeInMinute * 5;
 const FAST_FLUSH_POLL = 1000 * 15;
-const COMPACTION_INTERVAL = timeInMinute * 10;
-// Compact a blob once this fraction of its bytes are dead (deleted/overwritten)
-const COMPACTION_DEAD_FRACTION = 0.5;
-const MAX_OPEN_BLOBS = 64;
-
-type IndexEntry = {
-    // Blob file name
-    f: string;
-    // Offset + length within the blob
-    o: number;
-    l: number;
-    // Write time
-    t: number;
-};
+// Index changes are buffered in memory and written to the BulkDatabase2 in batches
+const INDEX_FLUSH_INTERVAL = 1000 * 30;
+// Sources that support getChangesAfter are polled this often
+const CHANGES_POLL_INTERVAL = 1000 * 60;
+// Sources that don't support getChangesAfter get a full metadata rescan this often
+const FULL_RESCAN_INTERVAL = 1000 * 60 * 60;
+// On a request for a file the index doesn't know, changes-after sources are re-polled, at most
+// this often
+const MISS_CHECK_INTERVAL = 1000 * 5;
+// Change polls re-request this much overlap, so clock skew between us and a source can't drop changes
+const CHANGES_POLL_OVERLAP = timeInMinute;
+const SCAN_RETRY_DELAY = 1000 * 30;
+// Deletes are tombstones (an empty file IS a missing file): the size-0 index entry is what lets a
+// deletion propagate/reconcile like any other write, and it expires after this long
+const TOMBSTONE_EXPIRY = 1000 * 60 * 60 * 24 * 7;
+const TOMBSTONE_CLEANUP_INTERVAL = 1000 * 60 * 60;
+// While a metadata scan or full sync is running, its progress is logged this often
+const SYNC_PROGRESS_LOG_INTERVAL = 1000 * 60;
+const DISK_LIMIT_CHECK_INTERVAL = 1000 * 60;
+// Full syncs download this many files concurrently (high-latency sources like backblaze would
+// otherwise crawl one round-trip at a time)
+const FULL_SYNC_PARALLEL = 8;
 
 export type WriteConfig = {
-    // Resolve once the write is in memory; flush to disk after writeDelay, coalescing writes to
-    // the same key (only the latest is written). Data is lost if the process crashes first.
+    // Resolve once the write is in memory; flush to the sources after writeDelay, coalescing
+    // writes to the same key (only the latest is written). Data is lost if the process crashes first.
     fast?: boolean;
     writeDelay?: number;
+    // Stamps the write with this last-write time instead of now. Older than the current file's
+    // time no-ops; more than 15 minutes in the future throws. See IArchives.set.
+    lastModified?: number;
+};
+
+// What the storage server needs from a bucket's store. BlobStore implements it fully; ArchivesDisk
+// also satisfies it (used directly for rawDisk buckets), minus the optional index-backed methods.
+export type IBucketStore = {
+    get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined>;
+    get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined>;
+    set(fileName: string, data: Buffer, config?: WriteConfig): Promise<void>;
+    del(fileName: string, config?: WriteConfig): Promise<void>;
+    getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined>;
+    findInfo(prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]>;
+    getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+    getSyncStatus?(): Promise<ArchivesSyncStatus>;
+    getSyncProgress?(): { index: { fileCount: number; byteCount: number }; syncing: SyncActivity[] };
+    computeIndexTotals?(): Promise<{ fileCount: number; byteCount: number }>;
+    startLargeUpload(): Promise<string>;
+    appendLargeUpload(id: string, data: Buffer): Promise<void>;
+    finishLargeUpload(id: string, key: string): Promise<void>;
+    cancelLargeUpload(id: string): Promise<void>;
 };
 
 type OverlayEntry = {
-    // undefined data means a pending delete
-    data: Buffer | undefined;
+    // A zero-length buffer is a pending delete (tombstone)
+    data: Buffer;
     t: number;
     flushAt: number;
 };
 
-class ByteLRU {
-    private map = new Map<string, Buffer>();
-    private bytes = 0;
-    constructor(private budget: number, private maxEntry: number) { }
-    public get(key: string): Buffer | undefined {
-        let value = this.map.get(key);
-        if (value) {
-            this.map.delete(key);
-            this.map.set(key, value);
-        }
-        return value;
-    }
-    public set(key: string, value: Buffer) {
-        this.delete(key);
-        if (value.length > this.maxEntry) return;
-        this.map.set(key, value);
-        this.bytes += value.length;
-        while (this.bytes > this.budget && this.map.size > 0) {
-            let oldest = this.map.keys().next().value;
-            if (oldest === undefined) break;
-            this.delete(oldest);
-        }
-    }
-    public delete(key: string) {
-        let existing = this.map.get(key);
-        if (!existing) return;
-        this.bytes -= existing.length;
-        this.map.delete(key);
-    }
-}
-
-export class BlobStore {
-    constructor(private folder: string) { }
+// One row of the BulkDatabase2 index (key is the file path)
+type BlobIndexEntry = {
+    key: string;
+    writeTime: number;
+    size: number;
+    // Which synchronization source currently holds the data (an index into the sources array)
+    source: number;
+};
+type IndexEntry = {
+    writeTime: number;
+    size: number;
+    source: number;
+    // When WE last changed this entry (not the file's write time) — what getChangesAfter filters
+    // on, so late-arriving files with old write times are still reported as changes
+    changedAt: number;
+    // In-memory only (not persisted): when the file was last served, for readerDiskLimit's LRU
+    // eviction. Starts as changedAt on load.
+    lastAccess: number;
+};
 
-    private memCache = new ByteLRU(MEMORY_CACHE_BYTES, MEMORY_CACHE_MAX_FILE);
-    private overlay = new Map<string, OverlayEntry>();
-    private writeQueue = runInSerial(async (fnc: () => Promise<void>) => fnc());
-    private openBlobs = new Map<string, Promise<fs.promises.FileHandle>>();
-    private largeUploads = new Map<string, { fd: fs.promises.FileHandle; tmpPath: string; size: number }>();
-    private nextLargeUploadId = 1;
+type SourceState = {
+    supportsChangesAfter: boolean;
+    initialScan: ReturnType<typeof promiseObj>;
+    scanComplete: boolean;
+    // Files seen in this source's scans / change polls so far
+    scannedCount: number;
+    // Watermark for getChangesAfter polls
+    changesAfterTime: number;
+    lastMissCheck: number;
+};
 
-    private currentBlobNumber = 0;
-    private currentBlobOffset = 0;
-    private currentBlobFd: fs.promises.FileHandle | undefined;
+export class BlobStore implements IBucketStore {
+    constructor(
+        private folder: string,
+        private sources: ArchivesSource[],
+        private config?: {
+            // Called whenever a key's index entry changes (our own writes AND files pulled in via
+            // synchronization) — how the storage server notices routing config updates.
+            onIndexChanged?: (key: string) => void;
+            // LRU-bound the disk (base source) to this many bytes; see CommonConfig.readerDiskLimit
+            readerDiskLimit?: number;
+        }
+    ) { }
 
-    private index!: JSONStorage<IndexEntry>;
-    // Dead (deleted/overwritten) byte count per blob file, for compaction
-    private deadBytes!: JSONStorage<number>;
+    private stopped = { stop: false };
 
-    private blobsDir = path.join(this.folder, "blobs");
+    // The index's BulkDatabase2 files live under <folder>/index
+    private index = new BulkDatabaseBase<BlobIndexEntry>("blobIndex", noopReactiveDeps, async (p: string) => {
+        let base: DirectoryWrapper = new NodeJSDirectoryHandleWrapper(path.join(this.folder, "index"));
+        for (let part of p.split("/")) {
+            if (part) base = await base.getDirectoryHandle(part, { create: true });
+        }
+        return wrapHandle(base);
+    });
+    // The in-memory copy of the index. All reads are served from it; changes are buffered in
+    // dirty and flushed to the BulkDatabase2 in batches (undefined = delete).
+    private mem = new Map<string, IndexEntry>();
+    // Live totals over mem (tombstones excluded), adjusted on every mutation and recomputed on
+    // load - so any drift heals on restart. computeIndexTotals gives the walk-the-index truth.
+    private indexFileCount = 0;
+    private indexByteCount = 0;
+    // The same totals per holding source (index 0 = our disk, which readerDiskLimit bounds)
+    private sourceFileCounts = this.sources.map(() => 0);
+    private sourceByteCounts = this.sources.map(() => 0);
+    // Background scans / full syncs currently in progress (a Set - one source can have a change
+    // poll's full sync and a rescan overlapping)
+    private syncActivities = new Set<SyncActivity>();
+    private dirty = new Map<string, IndexEntry | undefined>();
+    private overlay = new Map<string, OverlayEntry>();
+    private sourceStates = this.sources.map((): SourceState => ({
+        supportsChangesAfter: false,
+        initialScan: promiseObj(),
+        scanComplete: false,
+        scannedCount: 0,
+        changesAfterTime: 0,
+        lastMissCheck: 0,
+    }));
 
     public init = lazy(async () => {
-        fs.mkdirSync(this.blobsDir, { recursive: true });
-        let root = await getFileStorageNested2(path.resolve(this.folder));
-        let indexRaw = await root.folder.getStorage("index");
-        this.index = new JSONStorage<IndexEntry>(new TransactionStorage(indexRaw, "blobStoreIndex"));
-        let metaRaw = await root.folder.getStorage("meta");
-        this.deadBytes = new JSONStorage<number>(new TransactionStorage(metaRaw, "blobStoreDeadBytes"));
+        await this.loadIndex();
+        for (let i = 0; i < this.sources.length; i++) {
+            void this.runSourceSync(i);
+        }
+        runInfinitePoll(FAST_FLUSH_POLL, () => this.flushOverlay(), this.stopped);
+        runInfinitePoll(INDEX_FLUSH_INTERVAL, () => this.flushIndex(), this.stopped);
+        runInfinitePoll(TOMBSTONE_CLEANUP_INTERVAL, () => this.cleanupTombstones(), this.stopped);
+        if (this.config?.readerDiskLimit) {
+            runInfinitePoll(DISK_LIMIT_CHECK_INTERVAL, () => this.enforceDiskLimit(), this.stopped);
+        }
+    });
+
+    // Stops all synchronization scans/polls and flushes pending writes. Used when a bucket's
+    // routing config changes and the store is rebuilt with new sources.
+    public async dispose(): Promise<void> {
+        this.stopped.stop = true;
+        await this.flushOverlay(true);
+        await this.flushIndex();
+    }
 
-        for (let file of fs.readdirSync(this.blobsDir)) {
-            let match = /^blob_(\d+)\.bin$/.exec(file);
-            if (!match) continue;
-            this.currentBlobNumber = Math.max(this.currentBlobNumber, +match[1]);
+    private async loadIndex(): Promise<void> {
+        let [writeTimes, sizes, sources] = await Promise.all([
+            this.index.getColumn("writeTime"),
+            this.index.getColumn("size"),
+            this.index.getColumn("source"),
+        ]);
+        let sizeMap = new Map(sizes.map(x => [x.key, x.value]));
+        let sourceMap = new Map(sources.map(x => [x.key, x.value]));
+        for (let entry of writeTimes) {
+            let size = sizeMap.get(entry.key);
+            let source = sourceMap.get(entry.key);
+            // Explicit checks, as 0 is a valid size and a valid source number
+            if (size === undefined || source === undefined) continue;
+            let full: IndexEntry = { writeTime: entry.value, size, source, changedAt: entry.time, lastAccess: entry.time };
+            this.mem.set(entry.key, full);
+            this.countEntry(full, 1);
         }
-        if (this.currentBlobNumber > 0) {
-            this.currentBlobOffset = fs.statSync(path.join(this.blobsDir, this.blobName(this.currentBlobNumber))).size;
+    }
+
+    private countEntry(entry: IndexEntry | undefined, direction: number): void {
+        if (!entry || entry.size === 0) return;
+        this.indexFileCount += direction;
+        this.indexByteCount += entry.size * direction;
+        // A stale entry can reference a source that no longer exists (getIndexEntry cleans those)
+        if (this.sources[entry.source]) {
+            this.sourceFileCounts[entry.source] += direction;
+            this.sourceByteCounts[entry.source] += entry.size * direction;
         }
+    }
 
-        runInfinitePoll(FAST_FLUSH_POLL, () => this.flushOverlay());
-        runInfinitePoll(COMPACTION_INTERVAL, () => this.compact());
-    });
+    private setIndexEntry(key: string, entry: { writeTime: number; size: number; source: number }): void {
+        let full: IndexEntry = { ...entry, changedAt: Date.now(), lastAccess: Date.now() };
+        this.countEntry(this.mem.get(key), -1);
+        this.countEntry(full, 1);
+        this.mem.set(key, full);
+        this.dirty.set(key, full);
+        this.config?.onIndexChanged?.(key);
+    }
+    private deleteIndexEntry(key: string): void {
+        let existing = this.mem.get(key);
+        if (!existing) return;
+        this.countEntry(existing, -1);
+        this.mem.delete(key);
+        this.dirty.set(key, undefined);
+    }
+
+    /** The cheap always-current totals plus any in-progress background synchronization. */
+    public getSyncProgress(): {
+        index: { fileCount: number; byteCount: number };
+        sources: { debugName: string; fileCount: number; byteCount: number }[];
+        readerDiskLimit?: number;
+        syncing: SyncActivity[];
+    } {
+        return {
+            index: { fileCount: this.indexFileCount, byteCount: this.indexByteCount },
+            sources: this.sources.map((x, i) => ({
+                debugName: x.source.getDebugName(),
+                fileCount: this.sourceFileCounts[i],
+                byteCount: this.sourceByteCounts[i],
+            })),
+            readerDiskLimit: this.config?.readerDiskLimit,
+            syncing: [...this.syncActivities],
+        };
+    }
 
-    private blobName(n: number) {
-        return `blob_${String(n).padStart(6, "0")}.bin`;
+    /** Walks the whole index for exact totals - more expensive than getSyncProgress, but immune to
+     *  any drift in the maintained counters (and loads the index first, so it's never cold zeros). */
+    public async computeIndexTotals(): Promise<{
+        fileCount: number;
+        byteCount: number;
+        sources: { debugName: string; fileCount: number; byteCount: number }[];
+    }> {
+        await this.init();
+        let fileCount = 0;
+        let byteCount = 0;
+        let sources = this.sources.map(x => ({ debugName: x.source.getDebugName(), fileCount: 0, byteCount: 0 }));
+        for (let entry of this.mem.values()) {
+            if (entry.size === 0) continue;
+            fileCount++;
+            byteCount += entry.size;
+            let source = sources[entry.source];
+            if (source) {
+                source.fileCount++;
+                source.byteCount += entry.size;
+            }
+        }
+        return { fileCount, byteCount, sources };
     }
-    private blobPath(name: string) {
-        return path.join(this.blobsDir, name);
+
+    private async flushIndex(): Promise<void> {
+        if (!this.dirty.size) return;
+        let dirty = this.dirty;
+        this.dirty = new Map();
+        let writes: BlobIndexEntry[] = [];
+        let deletes: string[] = [];
+        for (let [key, entry] of dirty) {
+            if (entry) {
+                writes.push({ key, writeTime: entry.writeTime, size: entry.size, source: entry.source });
+            } else {
+                deletes.push(key);
+            }
+        }
+        if (writes.length) await this.index.writeBatch(writes);
+        if (deletes.length) await this.index.deleteBatch(deletes);
     }
 
-    private async getBlobHandle(name: string): Promise<fs.promises.FileHandle> {
-        let cached = this.openBlobs.get(name);
-        if (cached) {
-            // Re-insert for LRU ordering
-            this.openBlobs.delete(name);
-            this.openBlobs.set(name, cached);
-            return cached;
+    // ── synchronization ──
+
+    private async runSourceSync(sourceIndex: number): Promise<void> {
+        let { source, noFullSync } = this.sources[sourceIndex];
+        let state = this.sourceStates[sourceIndex];
+        let listing: Map<string, number> | undefined;
+        while (!this.stopped.stop) {
+            try {
+                let config = await source.getConfig();
+                state.supportsChangesAfter = !!(config.supportsChangesAfter && source.getChangesAfter);
+                listing = await this.scanSource(sourceIndex);
+                break;
+            } catch (e) {
+                console.error(`Initial scan of sync source ${source.getDebugName()} failed, retrying:`, e);
+                await delay(SCAN_RETRY_DELAY);
+            }
+        }
+        state.scanComplete = true;
+        state.initialScan.resolve(undefined);
+        if (this.stopped.stop) return;
+        if (listing) {
+            await this.reconcileSource(sourceIndex, listing);
+        }
+        if (!noFullSync) {
+            try {
+                await this.copySourceFiles(sourceIndex);
+            } catch (e) {
+                console.error(`Copying files from sync source ${source.getDebugName()} failed:`, e);
+            }
         }
-        let handle = fs.promises.open(this.blobPath(name), "r");
-        this.openBlobs.set(name, handle);
-        while (this.openBlobs.size > MAX_OPEN_BLOBS) {
-            let oldest = this.openBlobs.keys().next().value;
-            if (oldest === undefined) break;
-            let oldHandle = this.openBlobs.get(oldest);
-            this.openBlobs.delete(oldest);
-            void oldHandle?.then(h => h.close()).catch(() => { });
+        if (state.supportsChangesAfter) {
+            runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
+                await this.pollChanges(sourceIndex);
+                if (!noFullSync) await this.copySourceFiles(sourceIndex);
+            }, this.stopped);
+            // Change polls only show what the source HAS, never what it's missing, so pushes run on
+            // the full-rescan cadence (findInfo on an index-backed source is cheap)
+            runInfinitePoll(FULL_RESCAN_INTERVAL, async () => {
+                let files = await source.findInfo("");
+                await this.reconcileSource(sourceIndex, new Map(files.map(x => [x.path, x.createTime])));
+            }, this.stopped);
+        } else {
+            runInfinitePoll(FULL_RESCAN_INTERVAL, async () => {
+                let rescan = await this.scanSource(sourceIndex);
+                await this.reconcileSource(sourceIndex, rescan);
+                if (!noFullSync) await this.copySourceFiles(sourceIndex);
+            }, this.stopped);
         }
-        return handle;
     }
-    private async closeBlobHandle(name: string) {
-        let handle = this.openBlobs.get(name);
-        if (!handle) return;
-        this.openBlobs.delete(name);
-        await handle.then(h => h.close()).catch(() => { });
+
+    // Full metadata scan (size, writeTime, path) of one source, applied to the index. Returns the
+    // source's listing (path -> write time), which reconcileSource uses for the push direction.
+    private async scanSource(sourceIndex: number): Promise<Map<string, number>> {
+        let { source } = this.sources[sourceIndex];
+        let state = this.sourceStates[sourceIndex];
+        let scanStart = Date.now();
+        let scanLogged = false;
+        let activity: SyncActivity = { type: "metadataScan", sourceDebugName: source.getDebugName(), startTime: scanStart };
+        this.syncActivities.add(activity);
+        let progressTimer = setInterval(() => {
+            scanLogged = true;
+            console.log(`Metadata scan of ${source.getDebugName()} still running (${Math.round((Date.now() - scanStart) / 1000)}s, store ${this.folder})`);
+        }, SYNC_PROGRESS_LOG_INTERVAL);
+        (progressTimer as { unref?: () => void }).unref?.();
+        let files: ArchiveFileInfo[];
+        try {
+            files = await source.findInfo("");
+        } finally {
+            clearInterval(progressTimer);
+            this.syncActivities.delete(activity);
+        }
+        if (scanLogged) {
+            console.log(`Metadata scan of ${source.getDebugName()} finished: ${files.length} files in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.folder})`);
+        }
+        let seen = new Map<string, number>();
+        for (let file of files) {
+            seen.set(file.path, file.createTime);
+            this.applyScanned(sourceIndex, file);
+        }
+        state.scannedCount = files.length;
+        // Index entries this source was the holder of, but that vanished from it (e.g. deleted
+        // while we were offline), come out of the index. Entries changed after the scan started
+        // are kept — the scan listing may simply predate them. Tombstones have no physical file
+        // for a listing to vouch for, so they're exempt (cleanupTombstones expires them instead).
+        for (let [key, entry] of this.mem) {
+            if (entry.source !== sourceIndex) continue;
+            if (entry.size === 0) continue;
+            if (seen.has(key)) continue;
+            if (entry.changedAt >= scanStart) continue;
+            this.deleteIndexEntry(key);
+        }
+        state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
+        return seen;
     }
 
-    private async addDeadBytes(entry: IndexEntry) {
-        // Large files get a dedicated blob file, so on delete we can unlink it immediately
-        if (entry.f.startsWith("large_")) {
-            await this.closeBlobHandle(entry.f);
-            await fs.promises.unlink(this.blobPath(entry.f)).catch(() => { });
-            return;
+    // The push direction of synchronization: everything we know that the source is missing (or
+    // holds an older copy of) is written to it — including deletions, as tombstone writes. This is
+    // what heals a source whose background writes failed (e.g. it was down): the next scan sees
+    // what's missing and re-sends it.
+    private async reconcileSource(sourceIndex: number, listing: Map<string, number>): Promise<void> {
+        let { source, validWindow, route } = this.sources[sourceIndex];
+        let acceptsWrites = windowAcceptsWrites(validWindow);
+        try {
+            for (let [key, entry] of this.mem) {
+                if (this.stopped.stop) return;
+                if (entry.source === sourceIndex) continue;
+                // Past-window sources only receive the routing file (see writeToSources), and
+                // sharded sources only the keys routing into them
+                if (key !== ROUTING_FILE) {
+                    if (!acceptsWrites) continue;
+                    if (!routeContains(route, getRoute(key))) continue;
+                }
+                let theirTime = listing.get(key);
+                if (theirTime !== undefined && theirTime >= entry.writeTime) continue;
+                if (entry.size === 0) {
+                    // A deletion only needs pushing while the source still holds an older copy
+                    if (theirTime === undefined) continue;
+                    await source.set(key, Buffer.alloc(0), { lastModified: entry.writeTime });
+                    continue;
+                }
+                let result = await this.sources[entry.source].source.get2(key);
+                if (!result) continue;
+                await source.set(key, result.data, { lastModified: result.writeTime });
+            }
+        } catch (e) {
+            // Abort the pass instead of logging per file; the next scan cycle retries
+            console.error(`Reconciling sync source ${source.getDebugName()} failed: ${(e as Error).stack ?? e}`);
         }
-        let dead = await this.deadBytes.get(entry.f) || 0;
-        await this.deadBytes.set(entry.f, dead + entry.l);
     }
 
-    // Appends data to the current blob file, returning where it landed
-    private async appendData(data: Buffer): Promise<{ f: string; o: number; l: number }> {
-        let result: { f: string; o: number; l: number } | undefined;
-        await this.writeQueue(async () => {
-            if (!this.currentBlobFd || this.currentBlobOffset >= MAX_BLOB_SIZE) {
-                if (this.currentBlobFd) {
-                    await this.currentBlobFd.close();
-                    this.currentBlobFd = undefined;
+    private applyScanned(sourceIndex: number, file: ArchiveFileInfo): void {
+        let { validWindow, route } = this.sources[sourceIndex];
+        let [windowStart, windowEnd] = validWindow;
+        if (file.createTime < windowStart || file.createTime > windowEnd) return;
+        // A partially-overlapping shard's listing includes keys outside our route; ignore them
+        if (file.path !== ROUTING_FILE && !routeContains(route, getRoute(file.path))) return;
+        let existing = this.mem.get(file.path);
+        // The highest write time wins across all sources (ties keep the existing entry)
+        if (existing && file.createTime <= existing.writeTime) return;
+        this.setIndexEntry(file.path, { writeTime: file.createTime, size: file.size, source: sourceIndex });
+    }
+
+    private async pollChanges(sourceIndex: number): Promise<void> {
+        let { source } = this.sources[sourceIndex];
+        if (!source.getChangesAfter) return;
+        let state = this.sourceStates[sourceIndex];
+        let pollStart = Date.now();
+        let changes = await source.getChangesAfter(state.changesAfterTime);
+        for (let file of changes) {
+            this.applyScanned(sourceIndex, file);
+        }
+        state.scannedCount += changes.length;
+        state.changesAfterTime = pollStart - CHANGES_POLL_OVERLAP;
+    }
+
+    // Downloads the files a source currently holds onto our own base source (the local disk),
+    // preserving their modified times — so a newer local write always wins. Skipped for noFullSync
+    // sources (fronting a large database without copying it); reads still down-cache lazily.
+    private async copySourceFiles(sourceIndex: number): Promise<void> {
+        if (sourceIndex === 0) return;
+        let { source } = this.sources[sourceIndex];
+        let pending: { key: string; entry: IndexEntry }[] = [];
+        let totalBytes = 0;
+        for (let [key, entry] of this.mem) {
+            if (entry.source !== sourceIndex) continue;
+            if (entry.size === 0) continue;
+            pending.push({ key, entry });
+            totalBytes += entry.size;
+        }
+        if (!pending.length) return;
+        let activity: SyncActivity = {
+            type: "fullSync",
+            sourceDebugName: source.getDebugName(),
+            startTime: Date.now(),
+            doneFiles: 0,
+            totalFiles: pending.length,
+            doneBytes: 0,
+            totalBytes,
+        };
+        this.syncActivities.add(activity);
+        let progressLogged = false;
+        let logProgress = () => {
+            progressLogged = true;
+            console.log(`Full sync from ${source.getDebugName()} (store ${this.folder}): ${activity.doneFiles}/${pending.length} files (${((activity.doneFiles || 0) / pending.length * 100).toFixed(1)}%), ${formatNumber(activity.doneBytes || 0)}B/${formatNumber(totalBytes)}B (${(totalBytes && (activity.doneBytes || 0) / totalBytes * 100 || 100).toFixed(1)}%)`);
+        };
+        let progressTimer = setInterval(logProgress, SYNC_PROGRESS_LOG_INTERVAL);
+        (progressTimer as { unref?: () => void }).unref?.();
+        try {
+            let nextIndex = 0;
+            let failed = false;
+            let copyWorker = async () => {
+                while (!failed && !this.stopped.stop) {
+                    let index = nextIndex++;
+                    if (index >= pending.length) return;
+                    let { key, entry } = pending[index];
+                    let result = await source.get2(key);
+                    if (result) {
+                        await this.sources[0].source.set(key, result.data, { lastModified: result.writeTime });
+                        // Only move the entry's source if it wasn't changed while we copied
+                        if (this.mem.get(key) === entry) {
+                            this.setIndexEntry(key, { writeTime: result.writeTime, size: result.data.length, source: 0 });
+                        }
+                    }
+                    activity.doneFiles = (activity.doneFiles || 0) + 1;
+                    activity.doneBytes = (activity.doneBytes || 0) + entry.size;
                 }
-                this.currentBlobNumber++;
-                this.currentBlobOffset = 0;
+            };
+            let workers: Promise<void>[] = [];
+            for (let i = 0; i < Math.min(FULL_SYNC_PARALLEL, pending.length); i++) {
+                workers.push(copyWorker().catch((e: Error) => {
+                    // Stop the other workers pulling new files, then surface the error
+                    failed = true;
+                    throw e;
+                }));
             }
-            if (!this.currentBlobFd) {
-                this.currentBlobFd = await fs.promises.open(this.blobPath(this.blobName(this.currentBlobNumber)), "a");
-                this.currentBlobOffset = (await this.currentBlobFd.stat()).size;
+            await Promise.all(workers);
+        } finally {
+            clearInterval(progressTimer);
+            this.syncActivities.delete(activity);
+            // A sync slow enough to have logged progress also logs its completion
+            if (progressLogged) {
+                logProgress();
             }
-            let offset = this.currentBlobOffset;
-            await this.currentBlobFd.write(data, 0, data.length);
-            this.currentBlobOffset += data.length;
-            result = { f: this.blobName(this.currentBlobNumber), o: offset, l: data.length };
-        });
-        if (!result) throw new Error(`Append did not run, this should be impossible`);
-        return result;
+        }
+    }
+
+    // findInfo and getChangesAfter list from the index, so they must wait for our own base
+    // source's initial scan (which might lag minutes) before the listing is trustworthy. The base
+    // (local disk) is implicitly required - remote sources are not, they come and go.
+    private async waitForRequiredScans(): Promise<void> {
+        await this.sourceStates[0].initialScan.promise;
     }
 
-    private async setIndexEntry(key: string, entry: IndexEntry) {
-        let prev = await this.index.get(key);
-        await this.index.set(key, entry);
-        if (prev) {
-            await this.addDeadBytes(prev);
+    // A requested file isn't in the index: our own base source (implicitly required) is checked
+    // directly if its initial scan hasn't finished, and changes-after sources are re-polled (at
+    // most every 5 seconds)
+    private async checkMissingKey(key: string): Promise<void> {
+        for (let i = 0; i < this.sources.length; i++) {
+            let { source } = this.sources[i];
+            let state = this.sourceStates[i];
+            if (i === 0 && !state.scanComplete) {
+                let info = await source.getInfo(key);
+                if (info) {
+                    this.applyScanned(i, { path: key, createTime: info.writeTime, size: info.size });
+                }
+                continue;
+            }
+            if (state.supportsChangesAfter && Date.now() - state.lastMissCheck > MISS_CHECK_INTERVAL) {
+                state.lastMissCheck = Date.now();
+                await this.pollChanges(i);
+            }
         }
     }
 
-    public async set(key: string, data: Buffer, config?: WriteConfig): Promise<void> {
+    private async getIndexEntry(key: string): Promise<IndexEntry | undefined> {
+        let entry = this.mem.get(key);
+        if (entry && this.sources[entry.source]) return entry;
+        if (entry) {
+            // The source list changed and this entry's source no longer exists; treat as missing
+            this.deleteIndexEntry(key);
+        }
+        await this.checkMissingKey(key);
+        return this.mem.get(key);
+    }
+
+    // ── data operations ──
+
+    public async get(key: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
+        let result = await this.get2(key, config);
+        return result && result.data || undefined;
+    }
+
+    public async get2(key: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         await this.init();
-        this.memCache.set(key, data);
-        if (config?.fast) {
-            let writeDelay = config.writeDelay || DEFAULT_FAST_WRITE_DELAY;
-            this.overlay.set(key, { data, t: Date.now(), flushAt: Date.now() + writeDelay });
-            return;
+        let range = config?.range;
+        let overlayEntry = this.overlay.get(key);
+        if (overlayEntry) {
+            // An empty file IS a missing file (tombstone)
+            if (overlayEntry.data.length === 0) return undefined;
+            let data = overlayEntry.data;
+            let size = data.length;
+            if (range) {
+                data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
+            }
+            return { data, writeTime: overlayEntry.t, size };
         }
-        this.overlay.delete(key);
-        let location = await this.appendData(data);
-        await this.setIndexEntry(key, { ...location, t: Date.now() });
+        let entry = await this.getIndexEntry(key);
+        if (!entry) return undefined;
+        if (entry.size === 0) return undefined;
+        entry.lastAccess = Date.now();
+        let holder = entry.source;
+        let result: { data: Buffer; writeTime: number; size: number } | undefined;
+        let holderError: Error | undefined;
+        try {
+            result = await this.sources[holder].source.get2(key, { range });
+        } catch (e) {
+            holderError = e as Error;
+        }
+        if (result) {
+            // Ranged reads can't populate a cache (they're partial)
+            if (holder !== 0 && !range) {
+                await this.cacheRead(key, result);
+            }
+            return result;
+        }
+        // The holder is down or lost the file. ANY other source's copy beats no value - even an
+        // OLDER one - and it's copied onto our disk so the next read doesn't depend on luck.
+        for (let i = 0; i < this.sources.length; i++) {
+            if (i === holder) continue;
+            let fallback: typeof result;
+            try {
+                fallback = await this.sources[i].source.get2(key);
+            } catch {
+                continue;
+            }
+            if (!fallback) continue;
+            await this.cacheRead(key, fallback);
+            let data = fallback.data;
+            if (range) {
+                data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
+            }
+            return { data, writeTime: fallback.writeTime, size: fallback.size };
+        }
+        if (holderError) throw holderError;
+        // The holder answered "not there" and no other source has it either: the entry was stale
+        this.deleteIndexEntry(key);
+        return undefined;
     }
 
-    public async del(key: string, config?: WriteConfig): Promise<void> {
+    // The read's bytes came from a remote source, so write them onto our own base source (the
+    // local disk), which becomes the entry's new holder - reads only pay the remote fetch once
+    private async cacheRead(key: string, result: { data: Buffer; writeTime: number }): Promise<void> {
+        await this.sources[0].source.set(key, result.data, { lastModified: result.writeTime });
+        this.setIndexEntry(key, { writeTime: result.writeTime, size: result.data.length, source: 0 });
+    }
+
+    public async set(key: string, data: Buffer, config?: WriteConfig): Promise<void> {
         await this.init();
-        this.memCache.delete(key);
+        let lastModified = config?.lastModified;
+        if (lastModified) {
+            assertValidLastModified(lastModified);
+            let overlayEntry = this.overlay.get(key);
+            let entry = this.mem.get(key);
+            let currentTime = overlayEntry && overlayEntry.t || entry && entry.writeTime || 0;
+            // An older write never overwrites a newer one (see IArchives.set)
+            if (lastModified < currentTime) return;
+        }
+        let writeTime = lastModified || Date.now();
         if (config?.fast) {
             let writeDelay = config.writeDelay || DEFAULT_FAST_WRITE_DELAY;
-            this.overlay.set(key, { data: undefined, t: Date.now(), flushAt: Date.now() + writeDelay });
+            this.overlay.set(key, { data, t: writeTime, flushAt: Date.now() + writeDelay });
             return;
         }
         this.overlay.delete(key);
-        let prev = await this.index.get(key);
-        if (!prev) return;
-        await this.index.remove(key);
-        await this.addDeadBytes(prev);
+        await this.writeToSources(key, data, writeTime);
     }
 
-    public async get(key: string, range?: { start: number; end: number }): Promise<Buffer | undefined> {
-        await this.init();
-        let overlayEntry = this.overlay.get(key);
-        if (overlayEntry) {
-            if (!overlayEntry.data) return undefined;
-            if (!range) return overlayEntry.data;
-            return overlayEntry.data.subarray(Math.min(range.start, overlayEntry.data.length), Math.min(range.end, overlayEntry.data.length));
+    public async del(key: string, config?: WriteConfig): Promise<void> {
+        // Deletes are tombstone writes (an empty file IS a missing file): the size-0 index entry is
+        // ordered by write time like any other write, propagates through synchronization, and
+        // expires after TOMBSTONE_EXPIRY
+        await this.set(key, Buffer.alloc(0), config);
+    }
+
+    private getWritableSources(config?: { ignoreWindow?: boolean }): number[] {
+        let writable: number[] = [];
+        for (let i = 0; i < this.sources.length; i++) {
+            if (!config?.ignoreWindow && !windowAcceptsWrites(this.sources[i].validWindow)) continue;
+            writable.push(i);
         }
-        let cached = this.memCache.get(key);
-        if (cached) {
-            if (!range) return cached;
-            return cached.subarray(Math.min(range.start, cached.length), Math.min(range.end, cached.length));
+        return writable;
+    }
+
+    private async writeToSources(key: string, data: Buffer, writeTime: number): Promise<void> {
+        // The routing file is exempt from the valid-window and route write filters: the CONFIG
+        // must keep flowing to every source, or a client probing one of them first would adopt a
+        // stale config. Only data writes stop.
+        let isRouting = key === ROUTING_FILE;
+        let writable = this.getWritableSources({ ignoreWindow: isRouting });
+        let first = writable.shift();
+        if (first === undefined) {
+            throw new Error(`No source accepts writes (every source's valid window is in the past), so writes cannot be stored (store ${this.folder})`);
         }
-        let entry = await this.index.get(key);
-        if (!entry) return undefined;
-        let start = range && Math.min(range.start, entry.l) || 0;
-        let end = range && Math.min(range.end, entry.l) || entry.l;
-        if (end <= start) return Buffer.alloc(0);
-        let handle = await this.getBlobHandle(entry.f);
-        let buffer = Buffer.alloc(end - start);
-        let { bytesRead } = await handle.read(buffer, 0, buffer.length, entry.o + start);
-        if (bytesRead !== buffer.length) {
-            throw new Error(`Expected ${buffer.length} bytes at ${entry.f}:${entry.o + start} for ${key}, read ${bytesRead}`);
+        // Only our own (first) source blocks the write. Downstream sources are written in the
+        // background: a down downstream source must not fail or stall writes, and reconcileSource
+        // re-sends anything they missed once they come back.
+        if (data.length === 0) {
+            // A tombstone stores nothing on our own source - the index entry alone records it
+            await this.sources[first].source.del(key);
+        } else {
+            await this.sources[first].source.set(key, data, { lastModified: writeTime });
         }
-        if (!range && buffer.length <= MEMORY_CACHE_MAX_FILE) {
-            this.memCache.set(key, buffer);
+        this.setIndexEntry(key, { writeTime, size: data.length, source: first });
+        let route = !isRouting && getRoute(key) || 0;
+        for (let i of writable) {
+            if (!isRouting && !routeContains(this.sources[i].route, route)) continue;
+            // Downstream sources receive tombstones as actual empty writes, so their listings show
+            // the deletion (size 0) and other stores scan it in as a tombstone
+            void this.sources[i].source.set(key, data, { lastModified: writeTime }).catch((e: Error) => {
+                console.error(`Background write of ${key} to sync source ${this.sources[i].source.getDebugName()} failed: ${e.stack ?? e}`);
+            });
         }
-        return buffer;
     }
 
     public async getInfo(key: string): Promise<{ writeTime: number; size: number } | undefined> {
         await this.init();
         let overlayEntry = this.overlay.get(key);
         if (overlayEntry) {
-            if (!overlayEntry.data) return undefined;
             return { writeTime: overlayEntry.t, size: overlayEntry.data.length };
         }
-        let entry = await this.index.get(key);
+        let entry = await this.getIndexEntry(key);
         if (!entry) return undefined;
-        return { writeTime: entry.t, size: entry.l };
+        return { writeTime: entry.writeTime, size: entry.size };
     }
 
     public async findInfo(prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
         await this.init();
+        await this.waitForRequiredScans();
         let infos = new Map<string, ArchiveFileInfo>();
-        for (let key of await this.index.getKeys()) {
+        for (let [key, entry] of this.mem) {
             if (!key.startsWith(prefix)) continue;
-            if (this.overlay.has(key)) continue;
-            let entry = await this.index.get(key);
-            if (!entry) continue;
-            infos.set(key, { path: key, createTime: entry.t, size: entry.l });
+            // Tombstones are missing files, so listings hide them
+            if (entry.size === 0) continue;
+            infos.set(key, { path: key, createTime: entry.writeTime, size: entry.size });
         }
         for (let [key, overlayEntry] of this.overlay) {
             if (!key.startsWith(prefix)) continue;
-            if (!overlayEntry.data) continue;
+            if (overlayEntry.data.length === 0) {
+                infos.delete(key);
+                continue;
+            }
             infos.set(key, { path: key, createTime: overlayEntry.t, size: overlayEntry.data.length });
         }
-        let files = Array.from(infos.values());
-        if (config?.type === "folders") {
-            let folders = new Map<string, ArchiveFileInfo>();
-            for (let file of files) {
-                let rest = file.path.slice(prefix.length);
-                let restParts = rest.split("/");
-                if (restParts.length < 2) continue;
-                let folder: string;
-                if (config.shallow) {
-                    folder = prefix + restParts[0];
-                } else {
-                    folder = file.path.split("/").slice(0, -1).join("/");
-                }
-                folders.set(folder, { path: folder, createTime: file.createTime, size: file.size });
-            }
-            files = Array.from(folders.values());
-        } else if (config?.shallow) {
-            files = files.filter(file => !file.path.slice(prefix.length).includes("/"));
+        let files = applyFindInfoShape(Array.from(infos.values()), prefix, config);
+        sort(files, x => x.path);
+        return files;
+    }
+
+    // All files changed after the given time — fast, straight from the in-memory index. Filters on
+    // when WE learned of the change (changedAt), so files synchronized late (with old write times)
+    // are still reported. Deletions ARE reported, as size-0 tombstone entries — that's how they
+    // propagate to stores syncing from us.
+    public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
+        await this.init();
+        await this.waitForRequiredScans();
+        let files: ArchiveFileInfo[] = [];
+        for (let [key, entry] of this.mem) {
+            if (entry.changedAt <= time) continue;
+            if (this.overlay.has(key)) continue;
+            files.push({ path: key, createTime: entry.writeTime, size: entry.size });
+        }
+        for (let [key, overlayEntry] of this.overlay) {
+            if (overlayEntry.t <= time) continue;
+            files.push({ path: key, createTime: overlayEntry.t, size: overlayEntry.data.length });
         }
         sort(files, x => x.path);
         return files;
     }
 
-    // Large files stream into their own dedicated blob file, so concurrent small writes don't
-    // interleave into the middle of them.
+    public async getSyncStatus(): Promise<ArchivesSyncStatus> {
+        await this.init();
+        return {
+            allScansComplete: this.sourceStates.every(x => x.scanComplete),
+            indexSize: this.mem.size,
+            sources: this.sources.map((x, i) => ({
+                debugName: x.source.getDebugName(),
+                validWindow: x.validWindow,
+                route: x.route,
+                noFullSync: x.noFullSync,
+                supportsChangesAfter: this.sourceStates[i].supportsChangesAfter,
+                initialScanComplete: this.sourceStates[i].scanComplete,
+                scannedCount: this.sourceStates[i].scannedCount,
+            })),
+        };
+    }
+
+    // ── large uploads ──
+    // Large uploads stream onto the local disk source directly (they may not fit in memory)
+
+    private getDiskSource(): { disk: ArchivesDisk; sourceIndex: number } {
+        for (let i = 0; i < this.sources.length; i++) {
+            let source = this.sources[i].source;
+            if (source instanceof ArchivesDisk) return { disk: source, sourceIndex: i };
+        }
+        throw new Error(`Large uploads require an ArchivesDisk source, and this store has none (store ${this.folder})`);
+    }
     public async startLargeUpload(): Promise<string> {
         await this.init();
-        let id = `${Date.now()}_${this.nextLargeUploadId++}`;
-        let tmpPath = path.join(this.blobsDir, `upload_${id}.tmp`);
-        let fd = await fs.promises.open(tmpPath, "w");
-        this.largeUploads.set(id, { fd, tmpPath, size: 0 });
-        return id;
+        return await this.getDiskSource().disk.startLargeUpload();
     }
     public async appendLargeUpload(id: string, data: Buffer): Promise<void> {
-        let upload = this.largeUploads.get(id);
-        if (!upload) throw new Error(`Unknown large upload ${id}`);
-        await upload.fd.write(data, 0, data.length);
-        upload.size += data.length;
+        await this.getDiskSource().disk.appendLargeUpload(id, data);
     }
     public async finishLargeUpload(id: string, key: string): Promise<void> {
-        let upload = this.largeUploads.get(id);
-        if (!upload) throw new Error(`Unknown large upload ${id}`);
-        this.largeUploads.delete(id);
-        await upload.fd.close();
-        let blobName = `large_${id}.bin`;
-        await fs.promises.rename(upload.tmpPath, this.blobPath(blobName));
-        this.memCache.delete(key);
+        let { disk, sourceIndex } = this.getDiskSource();
+        await disk.finishLargeUpload(id, key);
         this.overlay.delete(key);
-        await this.setIndexEntry(key, { f: blobName, o: 0, l: upload.size, t: Date.now() });
+        let info = await disk.getInfo(key);
+        if (info) {
+            this.setIndexEntry(key, { writeTime: info.writeTime, size: info.size, source: sourceIndex });
+        }
     }
     public async cancelLargeUpload(id: string): Promise<void> {
-        let upload = this.largeUploads.get(id);
-        if (!upload) return;
-        this.largeUploads.delete(id);
-        await upload.fd.close();
-        await fs.promises.unlink(upload.tmpPath).catch(() => { });
+        await this.getDiskSource().disk.cancelLargeUpload(id);
     }
 
-    private async flushOverlay(): Promise<void> {
+    private async flushOverlay(force?: boolean): Promise<void> {
         let now = Date.now();
         for (let [key, entry] of this.overlay) {
-            if (entry.flushAt > now) continue;
-            if (entry.data) {
-                let location = await this.appendData(entry.data);
-                await this.setIndexEntry(key, { ...location, t: entry.t });
-            } else {
-                let prev = await this.index.get(key);
-                if (prev) {
-                    await this.index.remove(key);
-                    await this.addDeadBytes(prev);
-                }
-            }
+            if (!force && entry.flushAt > now) continue;
+            await this.writeToSources(key, entry.data, entry.t);
             // Only remove if it wasn't overwritten while we were flushing
             if (this.overlay.get(key) === entry) {
                 this.overlay.delete(key);
@@ -364,40 +804,67 @@ export class BlobStore {
         }
     }
 
-    private async compact(): Promise<void> {
-        let currentBlob = this.blobName(this.currentBlobNumber);
-        for (let blobName of await this.deadBytes.getKeys()) {
-            if (blobName === currentBlob) continue;
-            let dead = await this.deadBytes.get(blobName) || 0;
-            let blobPath = this.blobPath(blobName);
-            let size = 0;
-            try {
-                size = fs.statSync(blobPath).size;
-            } catch {
-                await this.deadBytes.remove(blobName);
-                continue;
+    // readerDiskLimit: the disk is only a bounded read cache, so once it exceeds the limit, the
+    // least recently used files are deleted from it - but ONLY when another source verifiably
+    // holds a same-or-newer copy (the only copy of a file is never deleted), and the index entry
+    // repoints to that source so reads keep working (re-caching on the next read).
+    private evicting = false;
+    private async enforceDiskLimit(): Promise<void> {
+        let limit = this.config?.readerDiskLimit;
+        if (!limit || this.evicting) return;
+        if (this.sourceByteCounts[0] <= limit) return;
+        this.evicting = true;
+        try {
+            let candidates: { key: string; entry: IndexEntry }[] = [];
+            for (let [key, entry] of this.mem) {
+                if (entry.source !== 0 || entry.size === 0 || key === ROUTING_FILE) continue;
+                candidates.push({ key, entry });
             }
-            if (dead < size * COMPACTION_DEAD_FRACTION) continue;
-
-            console.log(`Compacting blob ${blobName} (${formatNumber(dead)}B dead of ${formatNumber(size)}B)`);
-            for (let key of await this.index.getKeys()) {
-                let entry = await this.index.get(key);
-                if (!entry || entry.f !== blobName) continue;
-                let data = await this.get(key);
-                if (!data) continue;
-                let location = await this.appendData(data);
-                // Only move it if it wasn't rewritten while we were reading
-                let latest = await this.index.get(key);
-                if (latest && latest.f === entry.f && latest.o === entry.o) {
-                    await this.index.set(key, { ...location, t: latest.t });
-                } else {
-                    // Our copy is stale, so its bytes are immediately dead
-                    await this.deadBytes.set(location.f, (await this.deadBytes.get(location.f) || 0) + location.l);
+            sort(candidates, x => x.entry.lastAccess);
+            for (let { key, entry } of candidates) {
+                if (this.stopped.stop) return;
+                if (this.sourceByteCounts[0] <= limit) break;
+                if (this.mem.get(key) !== entry) continue;
+                let holder: number | undefined;
+                for (let i = 1; i < this.sources.length; i++) {
+                    try {
+                        let info = await this.sources[i].source.getInfo(key);
+                        if (info && info.writeTime >= entry.writeTime) {
+                            holder = i;
+                            break;
+                        }
+                    } catch {
+                        // A down source just can't vouch for this file right now
+                    }
                 }
+                if (holder === undefined) continue;
+                await this.sources[0].source.del(key);
+                this.setIndexEntry(key, { writeTime: entry.writeTime, size: entry.size, source: holder });
+            }
+        } finally {
+            this.evicting = false;
+        }
+    }
+
+    // Tombstones only need to exist long enough for every store to learn of the deletion; expired
+    // ones come out of the index. The physical empty file is removed only on backblaze sources:
+    // remote stores expire their own tombstones (a del there would just mint a fresh one), and our
+    // own disk never stored anything for it.
+    private async cleanupTombstones(): Promise<void> {
+        let cutoff = Date.now() - TOMBSTONE_EXPIRY;
+        for (let [key, entry] of this.mem) {
+            if (this.stopped.stop) return;
+            if (entry.size !== 0) continue;
+            if (entry.writeTime > cutoff) continue;
+            this.deleteIndexEntry(key);
+            for (let sourceEntry of this.sources) {
+                if (!windowAcceptsWrites(sourceEntry.validWindow)) continue;
+                let source = sourceEntry.source;
+                if (!(source instanceof ArchivesBackblaze)) continue;
+                void source.del(key).catch((e: Error) => {
+                    console.error(`Removing expired tombstone ${key} from ${source.getDebugName()} failed: ${e.stack ?? e}`);
+                });
             }
-            await this.closeBlobHandle(blobName);
-            await fs.promises.unlink(blobPath).catch(() => { });
-            await this.deadBytes.remove(blobName);
         }
     }
 }