sliftutils 1.7.9 → 1.7.10

package/storage/remoteStorage/blobStore.ts

@@ -1,47 +1,61 @@
-import fs from "fs";
 import path from "path";
 import { lazy } from "socket-function/src/caching";
-import { runInSerial, runInfinitePoll } from "socket-function/src/batching";
-import { timeInMinute, sort } from "socket-function/src/misc";
-import { formatNumber } from "socket-function/src/formatting/format";
-import { TransactionStorage } from "../TransactionStorage";
-import { JSONStorage } from "../JSONStorage";
-import { getFileStorageNested2 } from "../FileFolderAPI";
-import { ArchiveFileInfo } from "../IArchives";
-
-// Disk engine for the remote storage server. Files are packed into large append-only blob files
-// (instead of one file on disk per stored file), with a transaction-log index mapping
-// key -> (blob, offset, length). This keeps us efficient with many small files and scales to
-// terabytes: reads are single pread calls, writes are appends, and deleted space is reclaimed by
-// compacting blobs that are mostly dead.
-
-// Roll to a new blob file once the current one reaches this size
-const MAX_BLOB_SIZE = 4 * 1024 * 1024 * 1024;
-// Whole-file memory cache (only files up to MEMORY_CACHE_MAX_FILE are cached)
-const MEMORY_CACHE_BYTES = 256 * 1024 * 1024;
-const MEMORY_CACHE_MAX_FILE = 16 * 1024 * 1024;
+import { runInfinitePoll, delay } from "socket-function/src/batching";
+import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
+import {
+    IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, assertValidLastModified,
+} from "../IArchives";
+import { ArchivesDisk, applyFindInfoShape } from "../ArchivesDisk";
+import { BulkDatabaseBase, noopReactiveDeps } from "../BulkDatabase2/BulkDatabaseBase";
+import { wrapHandle, NodeJSDirectoryHandleWrapper, DirectoryWrapper } from "../FileFolderAPI";
+
+// The storage engine of the remote storage server. Data lives in synchronization sources (at
+// minimum an ArchivesDisk, the local disk); BlobStore keeps an index of every file (path, last
+// modified time, size, and which source currently holds the data) in a BulkDatabase2, and
+// synchronizes the index from all sources (see ArchivesSource / SyncOptions in IArchives.ts).
+// Every startup fully rescans each source's metadata, so the index self-heals; the file with the
+// highest write time wins across all sources, so multiple sources need no stacking order.
+
 export const DEFAULT_FAST_WRITE_DELAY = timeInMinute * 5;
 const FAST_FLUSH_POLL = 1000 * 15;
-const COMPACTION_INTERVAL = timeInMinute * 10;
-// Compact a blob once this fraction of its bytes are dead (deleted/overwritten)
-const COMPACTION_DEAD_FRACTION = 0.5;
-const MAX_OPEN_BLOBS = 64;
-
-type IndexEntry = {
-    // Blob file name
-    f: string;
-    // Offset + length within the blob
-    o: number;
-    l: number;
-    // Write time
-    t: number;
-};
+// Index changes are buffered in memory and written to the BulkDatabase2 in batches
+const INDEX_FLUSH_INTERVAL = 1000 * 30;
+// Sources that support getChangesAfter are polled this often
+const CHANGES_POLL_INTERVAL = 1000 * 60;
+// Sources that don't support getChangesAfter get a full metadata rescan this often
+const FULL_RESCAN_INTERVAL = 1000 * 60 * 60;
+// On a request for a file the index doesn't know, changes-after sources are re-polled, at most
+// this often
+const MISS_CHECK_INTERVAL = 1000 * 5;
+// Change polls re-request this much overlap, so clock skew between us and a source can't drop changes
+const CHANGES_POLL_OVERLAP = timeInMinute;
+const SCAN_RETRY_DELAY = 1000 * 30;
 
 export type WriteConfig = {
-    // Resolve once the write is in memory; flush to disk after writeDelay, coalescing writes to
-    // the same key (only the latest is written). Data is lost if the process crashes first.
+    // Resolve once the write is in memory; flush to the sources after writeDelay, coalescing
+    // writes to the same key (only the latest is written). Data is lost if the process crashes first.
     fast?: boolean;
     writeDelay?: number;
+    // Stamps the write with this last-write time instead of now. Older than the current file's
+    // time no-ops; more than 15 minutes in the future throws. See IArchives.set.
+    lastModified?: number;
+};
+
+// What the storage server needs from a bucket's store. BlobStore implements it fully; ArchivesDisk
+// also satisfies it (used directly for rawDisk buckets), minus the optional index-backed methods.
+export type IBucketStore = {
+    get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined>;
+    get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined>;
+    set(fileName: string, data: Buffer, config?: WriteConfig): Promise<void>;
+    del(fileName: string, config?: WriteConfig): Promise<void>;
+    getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined>;
+    findInfo(prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]>;
+    getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+    getSyncStatus?(): Promise<ArchivesSyncStatus>;
+    startLargeUpload(): Promise<string>;
+    appendLargeUpload(id: string, data: Buffer): Promise<void>;
+    finishLargeUpload(id: string, key: string): Promise<void>;
+    cancelLargeUpload(id: string): Promise<void>;
 };
 
 type OverlayEntry = {
@@ -51,211 +65,379 @@ type OverlayEntry = {
     flushAt: number;
 };
 
-class ByteLRU {
-    private map = new Map<string, Buffer>();
-    private bytes = 0;
-    constructor(private budget: number, private maxEntry: number) { }
-    public get(key: string): Buffer | undefined {
-        let value = this.map.get(key);
-        if (value) {
-            this.map.delete(key);
-            this.map.set(key, value);
-        }
-        return value;
-    }
-    public set(key: string, value: Buffer) {
-        this.delete(key);
-        if (value.length > this.maxEntry) return;
-        this.map.set(key, value);
-        this.bytes += value.length;
-        while (this.bytes > this.budget && this.map.size > 0) {
-            let oldest = this.map.keys().next().value;
-            if (oldest === undefined) break;
-            this.delete(oldest);
+// One row of the BulkDatabase2 index (key is the file path)
+type BlobIndexEntry = {
+    key: string;
+    writeTime: number;
+    size: number;
+    // Which synchronization source currently holds the data (an index into the sources array)
+    source: number;
+};
+type IndexEntry = {
+    writeTime: number;
+    size: number;
+    source: number;
+    // When WE last changed this entry (not the file's write time) — what getChangesAfter filters
+    // on, so late-arriving files with old write times are still reported as changes
+    changedAt: number;
+};
+
+type SourceState = {
+    supportsChangesAfter: boolean;
+    initialScan: ReturnType<typeof promiseObj>;
+    scanComplete: boolean;
+    // Files seen in this source's scans / change polls so far
+    scannedCount: number;
+    // Watermark for getChangesAfter polls
+    changesAfterTime: number;
+    lastMissCheck: number;
+};
+
+export class BlobStore implements IBucketStore {
+    constructor(
+        private folder: string,
+        private sources: ArchivesSource[],
+        private config?: {
+            // Called whenever a key's index entry changes (our own writes AND files pulled in via
+            // synchronization) — how the storage server notices routing config updates.
+            onIndexChanged?: (key: string) => void;
         }
-    }
-    public delete(key: string) {
-        let existing = this.map.get(key);
-        if (!existing) return;
-        this.bytes -= existing.length;
-        this.map.delete(key);
-    }
-}
+    ) { }
 
-export class BlobStore {
-    constructor(private folder: string) { }
+    private stopped = { stop: false };
 
-    private memCache = new ByteLRU(MEMORY_CACHE_BYTES, MEMORY_CACHE_MAX_FILE);
+    // The index's BulkDatabase2 files live under <folder>/index
+    private index = new BulkDatabaseBase<BlobIndexEntry>("blobIndex", noopReactiveDeps, async (p: string) => {
+        let base: DirectoryWrapper = new NodeJSDirectoryHandleWrapper(path.join(this.folder, "index"));
+        for (let part of p.split("/")) {
+            if (part) base = await base.getDirectoryHandle(part, { create: true });
+        }
+        return wrapHandle(base);
+    });
+    // The in-memory copy of the index. All reads are served from it; changes are buffered in
+    // dirty and flushed to the BulkDatabase2 in batches (undefined = delete).
+    private mem = new Map<string, IndexEntry>();
+    private dirty = new Map<string, IndexEntry | undefined>();
     private overlay = new Map<string, OverlayEntry>();
-    private writeQueue = runInSerial(async (fnc: () => Promise<void>) => fnc());
-    private openBlobs = new Map<string, Promise<fs.promises.FileHandle>>();
-    private largeUploads = new Map<string, { fd: fs.promises.FileHandle; tmpPath: string; size: number }>();
-    private nextLargeUploadId = 1;
+    private sourceStates = this.sources.map((): SourceState => ({
+        supportsChangesAfter: false,
+        initialScan: promiseObj(),
+        scanComplete: false,
+        scannedCount: 0,
+        changesAfterTime: 0,
+        lastMissCheck: 0,
+    }));
 
-    private currentBlobNumber = 0;
-    private currentBlobOffset = 0;
-    private currentBlobFd: fs.promises.FileHandle | undefined;
+    public init = lazy(async () => {
+        await this.loadIndex();
+        for (let i = 0; i < this.sources.length; i++) {
+            void this.runSourceSync(i);
+        }
+        runInfinitePoll(FAST_FLUSH_POLL, () => this.flushOverlay(), this.stopped);
+        runInfinitePoll(INDEX_FLUSH_INTERVAL, () => this.flushIndex(), this.stopped);
+    });
 
-    private index!: JSONStorage<IndexEntry>;
-    // Dead (deleted/overwritten) byte count per blob file, for compaction
-    private deadBytes!: JSONStorage<number>;
+    // Stops all synchronization scans/polls and flushes pending writes. Used when a bucket's
+    // routing config changes and the store is rebuilt with new sources.
+    public async dispose(): Promise<void> {
+        this.stopped.stop = true;
+        await this.flushOverlay(true);
+        await this.flushIndex();
+    }
 
-    private blobsDir = path.join(this.folder, "blobs");
+    private async loadIndex(): Promise<void> {
+        let [writeTimes, sizes, sources] = await Promise.all([
+            this.index.getColumn("writeTime"),
+            this.index.getColumn("size"),
+            this.index.getColumn("source"),
+        ]);
+        let sizeMap = new Map(sizes.map(x => [x.key, x.value]));
+        let sourceMap = new Map(sources.map(x => [x.key, x.value]));
+        for (let entry of writeTimes) {
+            let size = sizeMap.get(entry.key);
+            let source = sourceMap.get(entry.key);
+            // Explicit checks, as 0 is a valid size and a valid source number
+            if (size === undefined || source === undefined) continue;
+            this.mem.set(entry.key, { writeTime: entry.value, size, source, changedAt: entry.time });
+        }
+    }
 
-    public init = lazy(async () => {
-        fs.mkdirSync(this.blobsDir, { recursive: true });
-        let root = await getFileStorageNested2(path.resolve(this.folder));
-        let indexRaw = await root.folder.getStorage("index");
-        this.index = new JSONStorage<IndexEntry>(new TransactionStorage(indexRaw, "blobStoreIndex"));
-        let metaRaw = await root.folder.getStorage("meta");
-        this.deadBytes = new JSONStorage<number>(new TransactionStorage(metaRaw, "blobStoreDeadBytes"));
+    private setIndexEntry(key: string, entry: { writeTime: number; size: number; source: number }): void {
+        let full: IndexEntry = { ...entry, changedAt: Date.now() };
+        this.mem.set(key, full);
+        this.dirty.set(key, full);
+        this.config?.onIndexChanged?.(key);
+    }
+    private deleteIndexEntry(key: string): void {
+        if (!this.mem.has(key)) return;
+        this.mem.delete(key);
+        this.dirty.set(key, undefined);
+    }
 
-        for (let file of fs.readdirSync(this.blobsDir)) {
-            let match = /^blob_(\d+)\.bin$/.exec(file);
-            if (!match) continue;
-            this.currentBlobNumber = Math.max(this.currentBlobNumber, +match[1]);
-        }
-        if (this.currentBlobNumber > 0) {
-            this.currentBlobOffset = fs.statSync(path.join(this.blobsDir, this.blobName(this.currentBlobNumber))).size;
+    private async flushIndex(): Promise<void> {
+        if (!this.dirty.size) return;
+        let dirty = this.dirty;
+        this.dirty = new Map();
+        let writes: BlobIndexEntry[] = [];
+        let deletes: string[] = [];
+        for (let [key, entry] of dirty) {
+            if (entry) {
+                writes.push({ key, writeTime: entry.writeTime, size: entry.size, source: entry.source });
+            } else {
+                deletes.push(key);
+            }
         }
+        if (writes.length) await this.index.writeBatch(writes);
+        if (deletes.length) await this.index.deleteBatch(deletes);
+    }
 
-        runInfinitePoll(FAST_FLUSH_POLL, () => this.flushOverlay());
-        runInfinitePoll(COMPACTION_INTERVAL, () => this.compact());
-    });
+    // ── synchronization ──
+
+    private async runSourceSync(sourceIndex: number): Promise<void> {
+        let { source, options } = this.sources[sourceIndex];
+        let state = this.sourceStates[sourceIndex];
+        while (!this.stopped.stop) {
+            try {
+                let config = await source.getConfig();
+                state.supportsChangesAfter = !!(config.supportsChangesAfter && source.getChangesAfter);
+                await this.scanSource(sourceIndex);
+                break;
+            } catch (e) {
+                console.error(`Initial scan of sync source ${source.getDebugName()} failed, retrying:`, e);
+                await delay(SCAN_RETRY_DELAY);
+            }
+        }
+        state.scanComplete = true;
+        state.initialScan.resolve(undefined);
+        if (this.stopped.stop) return;
+        if (options.copyFiles) {
+            try {
+                await this.copySourceFiles(sourceIndex);
+            } catch (e) {
+                console.error(`Copying files from sync source ${source.getDebugName()} failed:`, e);
+            }
+        }
+        if (state.supportsChangesAfter) {
+            runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
+                await this.pollChanges(sourceIndex);
+                if (options.copyFiles) await this.copySourceFiles(sourceIndex);
+            }, this.stopped);
+        } else {
+            runInfinitePoll(FULL_RESCAN_INTERVAL, async () => {
+                await this.scanSource(sourceIndex);
+                if (options.copyFiles) await this.copySourceFiles(sourceIndex);
+            }, this.stopped);
+        }
+    }
 
-    private blobName(n: number) {
-        return `blob_${String(n).padStart(6, "0")}.bin`;
+    // Full metadata scan (size, writeTime, path) of one source, applied to the index
+    private async scanSource(sourceIndex: number): Promise<void> {
+        let { source } = this.sources[sourceIndex];
+        let state = this.sourceStates[sourceIndex];
+        let scanStart = Date.now();
+        let files = await source.findInfo("");
+        let seen = new Set<string>();
+        for (let file of files) {
+            seen.add(file.path);
+            this.applyScanned(sourceIndex, file);
+        }
+        state.scannedCount = files.length;
+        // Index entries this source was the holder of, but that vanished from it (e.g. deleted
+        // while we were offline), come out of the index. Entries changed after the scan started
+        // are kept — the scan listing may simply predate them.
+        for (let [key, entry] of this.mem) {
+            if (entry.source !== sourceIndex) continue;
+            if (seen.has(key)) continue;
+            if (entry.changedAt >= scanStart) continue;
+            this.deleteIndexEntry(key);
+        }
+        state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
     }
-    private blobPath(name: string) {
-        return path.join(this.blobsDir, name);
+
+    private applyScanned(sourceIndex: number, file: ArchiveFileInfo): void {
+        let [windowStart, windowEnd] = this.sources[sourceIndex].options.validWindow;
+        if (file.createTime < windowStart || file.createTime > windowEnd) return;
+        let existing = this.mem.get(file.path);
+        // The highest write time wins across all sources (ties keep the existing entry)
+        if (existing && file.createTime <= existing.writeTime) return;
+        this.setIndexEntry(file.path, { writeTime: file.createTime, size: file.size, source: sourceIndex });
     }
 
-    private async getBlobHandle(name: string): Promise<fs.promises.FileHandle> {
-        let cached = this.openBlobs.get(name);
-        if (cached) {
-            // Re-insert for LRU ordering
-            this.openBlobs.delete(name);
-            this.openBlobs.set(name, cached);
-            return cached;
-        }
-        let handle = fs.promises.open(this.blobPath(name), "r");
-        this.openBlobs.set(name, handle);
-        while (this.openBlobs.size > MAX_OPEN_BLOBS) {
-            let oldest = this.openBlobs.keys().next().value;
-            if (oldest === undefined) break;
-            let oldHandle = this.openBlobs.get(oldest);
-            this.openBlobs.delete(oldest);
-            void oldHandle?.then(h => h.close()).catch(() => { });
-        }
-        return handle;
+    private async pollChanges(sourceIndex: number): Promise<void> {
+        let { source } = this.sources[sourceIndex];
+        if (!source.getChangesAfter) return;
+        let state = this.sourceStates[sourceIndex];
+        let pollStart = Date.now();
+        let changes = await source.getChangesAfter(state.changesAfterTime);
+        for (let file of changes) {
+            this.applyScanned(sourceIndex, file);
+        }
+        state.scannedCount += changes.length;
+        state.changesAfterTime = pollStart - CHANGES_POLL_OVERLAP;
     }
-    private async closeBlobHandle(name: string) {
-        let handle = this.openBlobs.get(name);
-        if (!handle) return;
-        this.openBlobs.delete(name);
-        await handle.then(h => h.close()).catch(() => { });
+
+    // Downloads the files a copyFiles source currently holds into the cacheReads sources (the
+    // local cache), preserving their modified times — so a newer local write always wins
+    private async copySourceFiles(sourceIndex: number): Promise<void> {
+        let { source } = this.sources[sourceIndex];
+        let targets: number[] = [];
+        for (let i = 0; i < this.sources.length; i++) {
+            if (i !== sourceIndex && this.sources[i].options.cacheReads) targets.push(i);
+        }
+        if (!targets.length) return;
+        for (let [key, entry] of this.mem) {
+            if (entry.source !== sourceIndex) continue;
+            let result = await source.get2(key);
+            if (!result) continue;
+            for (let target of targets) {
+                await this.sources[target].source.set(key, result.data, { lastModified: result.writeTime });
+            }
+            // Only move the entry's source if it wasn't changed while we copied
+            if (this.mem.get(key) === entry) {
+                this.setIndexEntry(key, { writeTime: result.writeTime, size: result.data.length, source: targets[0] });
+            }
+        }
     }
 
-    private async addDeadBytes(entry: IndexEntry) {
-        // Large files get a dedicated blob file, so on delete we can unlink it immediately
-        if (entry.f.startsWith("large_")) {
-            await this.closeBlobHandle(entry.f);
-            await fs.promises.unlink(this.blobPath(entry.f)).catch(() => { });
-            return;
+    // findInfo and getChangesAfter list from the index, so they must wait for the required
+    // sources' initial scans (which might lag minutes) before the listing is trustworthy
+    private async waitForRequiredScans(): Promise<void> {
+        for (let i = 0; i < this.sources.length; i++) {
+            if (!this.sources[i].options.required) continue;
+            await this.sourceStates[i].initialScan.promise;
         }
-        let dead = await this.deadBytes.get(entry.f) || 0;
-        await this.deadBytes.set(entry.f, dead + entry.l);
     }
 
-    // Appends data to the current blob file, returning where it landed
-    private async appendData(data: Buffer): Promise<{ f: string; o: number; l: number }> {
-        let result: { f: string; o: number; l: number } | undefined;
-        await this.writeQueue(async () => {
-            if (!this.currentBlobFd || this.currentBlobOffset >= MAX_BLOB_SIZE) {
-                if (this.currentBlobFd) {
-                    await this.currentBlobFd.close();
-                    this.currentBlobFd = undefined;
+    // A requested file isn't in the index: required sources that haven't finished their initial
+    // scan are checked directly, and changes-after sources are re-polled (at most every 5 seconds)
+    private async checkMissingKey(key: string): Promise<void> {
+        for (let i = 0; i < this.sources.length; i++) {
+            let { source, options } = this.sources[i];
+            let state = this.sourceStates[i];
+            if (options.required && !state.scanComplete) {
+                let info = await source.getInfo(key);
+                if (info) {
+                    this.applyScanned(i, { path: key, createTime: info.writeTime, size: info.size });
                 }
-                this.currentBlobNumber++;
-                this.currentBlobOffset = 0;
+                continue;
             }
-            if (!this.currentBlobFd) {
-                this.currentBlobFd = await fs.promises.open(this.blobPath(this.blobName(this.currentBlobNumber)), "a");
-                this.currentBlobOffset = (await this.currentBlobFd.stat()).size;
+            if (state.supportsChangesAfter && Date.now() - state.lastMissCheck > MISS_CHECK_INTERVAL) {
+                state.lastMissCheck = Date.now();
+                await this.pollChanges(i);
+            }
+        }
+    }
+
+    private async getIndexEntry(key: string): Promise<IndexEntry | undefined> {
+        let entry = this.mem.get(key);
+        if (entry && this.sources[entry.source]) return entry;
+        if (entry) {
+            // The source list changed and this entry's source no longer exists; treat as missing
+            this.deleteIndexEntry(key);
+        }
+        await this.checkMissingKey(key);
+        return this.mem.get(key);
+    }
+
+    // ── data operations ──
+
+    public async get(key: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
+        let result = await this.get2(key, config);
+        return result && result.data || undefined;
+    }
+
+    public async get2(key: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
+        await this.init();
+        let range = config?.range;
+        let overlayEntry = this.overlay.get(key);
+        if (overlayEntry) {
+            if (!overlayEntry.data) return undefined;
+            let data = overlayEntry.data;
+            if (range) {
+                data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
             }
-            let offset = this.currentBlobOffset;
-            await this.currentBlobFd.write(data, 0, data.length);
-            this.currentBlobOffset += data.length;
-            result = { f: this.blobName(this.currentBlobNumber), o: offset, l: data.length };
-        });
-        if (!result) throw new Error(`Append did not run, this should be impossible`);
+            return { data, writeTime: overlayEntry.t };
+        }
+        let entry = await this.getIndexEntry(key);
+        if (!entry) return undefined;
+        let { source, options } = this.sources[entry.source];
+        let result = await source.get2(key, { range });
+        if (!result) {
+            // The source no longer has it, so our index entry was stale
+            this.deleteIndexEntry(key);
+            return undefined;
+        }
+        // Ranged reads can't populate a cache (they're partial)
+        if (!options.cacheReads && !range) {
+            await this.cacheRead(key, result);
+        }
         return result;
     }
 
-    private async setIndexEntry(key: string, entry: IndexEntry) {
-        let prev = await this.index.get(key);
-        await this.index.set(key, entry);
-        if (prev) {
-            await this.addDeadBytes(prev);
+    // The read didn't come from a cacheReads source, so write it into all of them (using them as
+    // caches), and the first one becomes the entry's new source
+    private async cacheRead(key: string, result: { data: Buffer; writeTime: number }): Promise<void> {
+        let first: number | undefined;
+        for (let i = 0; i < this.sources.length; i++) {
+            if (!this.sources[i].options.cacheReads) continue;
+            await this.sources[i].source.set(key, result.data, { lastModified: result.writeTime });
+            if (first === undefined) first = i;
         }
+        if (first === undefined) return;
+        this.setIndexEntry(key, { writeTime: result.writeTime, size: result.data.length, source: first });
     }
 
     public async set(key: string, data: Buffer, config?: WriteConfig): Promise<void> {
         await this.init();
-        this.memCache.set(key, data);
+        let lastModified = config?.lastModified;
+        if (lastModified) {
+            assertValidLastModified(lastModified);
+            let overlayEntry = this.overlay.get(key);
+            let entry = this.mem.get(key);
+            let currentTime = overlayEntry && overlayEntry.t || entry && entry.writeTime || 0;
+            // An older write never overwrites a newer one (see IArchives.set)
+            if (lastModified < currentTime) return;
+        }
+        let writeTime = lastModified || Date.now();
         if (config?.fast) {
             let writeDelay = config.writeDelay || DEFAULT_FAST_WRITE_DELAY;
-            this.overlay.set(key, { data, t: Date.now(), flushAt: Date.now() + writeDelay });
+            this.overlay.set(key, { data, t: writeTime, flushAt: Date.now() + writeDelay });
             return;
         }
         this.overlay.delete(key);
-        let location = await this.appendData(data);
-        await this.setIndexEntry(key, { ...location, t: Date.now() });
+        await this.writeToSources(key, data, writeTime);
+    }
+
+    private async writeToSources(key: string, data: Buffer, writeTime: number): Promise<void> {
+        let first: number | undefined;
+        for (let i = 0; i < this.sources.length; i++) {
+            if (this.sources[i].options.noWriteBack) continue;
+            await this.sources[i].source.set(key, data, { lastModified: writeTime });
+            if (first === undefined) first = i;
+        }
+        if (first === undefined) {
+            throw new Error(`Every source is noWriteBack, so writes cannot be stored (store ${this.folder})`);
+        }
+        this.setIndexEntry(key, { writeTime, size: data.length, source: first });
     }
 
     public async del(key: string, config?: WriteConfig): Promise<void> {
         await this.init();
-        this.memCache.delete(key);
         if (config?.fast) {
             let writeDelay = config.writeDelay || DEFAULT_FAST_WRITE_DELAY;
             this.overlay.set(key, { data: undefined, t: Date.now(), flushAt: Date.now() + writeDelay });
             return;
         }
         this.overlay.delete(key);
-        let prev = await this.index.get(key);
-        if (!prev) return;
-        await this.index.remove(key);
-        await this.addDeadBytes(prev);
+        await this.deleteFromSources(key);
     }
 
-    public async get(key: string, range?: { start: number; end: number }): Promise<Buffer | undefined> {
-        await this.init();
-        let overlayEntry = this.overlay.get(key);
-        if (overlayEntry) {
-            if (!overlayEntry.data) return undefined;
-            if (!range) return overlayEntry.data;
-            return overlayEntry.data.subarray(Math.min(range.start, overlayEntry.data.length), Math.min(range.end, overlayEntry.data.length));
+    private async deleteFromSources(key: string): Promise<void> {
+        for (let i = 0; i < this.sources.length; i++) {
+            if (this.sources[i].options.noWriteBack) continue;
+            await this.sources[i].source.del(key);
         }
-        let cached = this.memCache.get(key);
-        if (cached) {
-            if (!range) return cached;
-            return cached.subarray(Math.min(range.start, cached.length), Math.min(range.end, cached.length));
-        }
-        let entry = await this.index.get(key);
-        if (!entry) return undefined;
-        let start = range && Math.min(range.start, entry.l) || 0;
-        let end = range && Math.min(range.end, entry.l) || entry.l;
-        if (end <= start) return Buffer.alloc(0);
-        let handle = await this.getBlobHandle(entry.f);
-        let buffer = Buffer.alloc(end - start);
-        let { bytesRead } = await handle.read(buffer, 0, buffer.length, entry.o + start);
-        if (bytesRead !== buffer.length) {
-            throw new Error(`Expected ${buffer.length} bytes at ${entry.f}:${entry.o + start} for ${key}, read ${bytesRead}`);
-        }
-        if (!range && buffer.length <= MEMORY_CACHE_MAX_FILE) {
-            this.memCache.set(key, buffer);
-        }
-        return buffer;
+        this.deleteIndexEntry(key);
     }
 
     public async getInfo(key: string): Promise<{ writeTime: number; size: number } | undefined> {
@@ -265,97 +447,106 @@ export class BlobStore {
             if (!overlayEntry.data) return undefined;
             return { writeTime: overlayEntry.t, size: overlayEntry.data.length };
         }
-        let entry = await this.index.get(key);
+        let entry = await this.getIndexEntry(key);
         if (!entry) return undefined;
-        return { writeTime: entry.t, size: entry.l };
+        return { writeTime: entry.writeTime, size: entry.size };
     }
 
     public async findInfo(prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
         await this.init();
+        await this.waitForRequiredScans();
         let infos = new Map<string, ArchiveFileInfo>();
-        for (let key of await this.index.getKeys()) {
+        for (let [key, entry] of this.mem) {
             if (!key.startsWith(prefix)) continue;
-            if (this.overlay.has(key)) continue;
-            let entry = await this.index.get(key);
-            if (!entry) continue;
-            infos.set(key, { path: key, createTime: entry.t, size: entry.l });
+            infos.set(key, { path: key, createTime: entry.writeTime, size: entry.size });
         }
         for (let [key, overlayEntry] of this.overlay) {
             if (!key.startsWith(prefix)) continue;
-            if (!overlayEntry.data) continue;
+            if (!overlayEntry.data) {
+                infos.delete(key);
+                continue;
+            }
             infos.set(key, { path: key, createTime: overlayEntry.t, size: overlayEntry.data.length });
         }
-        let files = Array.from(infos.values());
-        if (config?.type === "folders") {
-            let folders = new Map<string, ArchiveFileInfo>();
-            for (let file of files) {
-                let rest = file.path.slice(prefix.length);
-                let restParts = rest.split("/");
-                if (restParts.length < 2) continue;
-                let folder: string;
-                if (config.shallow) {
-                    folder = prefix + restParts[0];
-                } else {
-                    folder = file.path.split("/").slice(0, -1).join("/");
-                }
-                folders.set(folder, { path: folder, createTime: file.createTime, size: file.size });
-            }
-            files = Array.from(folders.values());
-        } else if (config?.shallow) {
-            files = files.filter(file => !file.path.slice(prefix.length).includes("/"));
+        let files = applyFindInfoShape(Array.from(infos.values()), prefix, config);
+        sort(files, x => x.path);
+        return files;
+    }
+
+    // All files changed after the given time — fast, straight from the in-memory index. Filters on
+    // when WE learned of the change (changedAt), so files synchronized late (with old write times)
+    // are still reported. Deletions are not reported.
+    public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
+        await this.init();
+        await this.waitForRequiredScans();
+        let files: ArchiveFileInfo[] = [];
+        for (let [key, entry] of this.mem) {
+            if (entry.changedAt <= time) continue;
+            if (this.overlay.has(key)) continue;
+            files.push({ path: key, createTime: entry.writeTime, size: entry.size });
+        }
+        for (let [key, overlayEntry] of this.overlay) {
+            if (!overlayEntry.data) continue;
+            if (overlayEntry.t <= time) continue;
+            files.push({ path: key, createTime: overlayEntry.t, size: overlayEntry.data.length });
         }
         sort(files, x => x.path);
         return files;
     }
 
-    // Large files stream into their own dedicated blob file, so concurrent small writes don't
-    // interleave into the middle of them.
+    public async getSyncStatus(): Promise<ArchivesSyncStatus> {
+        await this.init();
+        return {
+            allScansComplete: this.sourceStates.every(x => x.scanComplete),
+            indexSize: this.mem.size,
+            sources: this.sources.map((x, i) => ({
+                debugName: x.source.getDebugName(),
+                options: x.options,
+                supportsChangesAfter: this.sourceStates[i].supportsChangesAfter,
+                initialScanComplete: this.sourceStates[i].scanComplete,
+                scannedCount: this.sourceStates[i].scannedCount,
+            })),
+        };
+    }
+
+    // ── large uploads ──
+    // Large uploads stream onto the local disk source directly (they may not fit in memory)
+
+    private getDiskSource(): { disk: ArchivesDisk; sourceIndex: number } {
+        for (let i = 0; i < this.sources.length; i++) {
+            let source = this.sources[i].source;
+            if (source instanceof ArchivesDisk) return { disk: source, sourceIndex: i };
+        }
+        throw new Error(`Large uploads require an ArchivesDisk source, and this store has none (store ${this.folder})`);
+    }
     public async startLargeUpload(): Promise<string> {
         await this.init();
-        let id = `${Date.now()}_${this.nextLargeUploadId++}`;
-        let tmpPath = path.join(this.blobsDir, `upload_${id}.tmp`);
-        let fd = await fs.promises.open(tmpPath, "w");
-        this.largeUploads.set(id, { fd, tmpPath, size: 0 });
-        return id;
+        return await this.getDiskSource().disk.startLargeUpload();
     }
     public async appendLargeUpload(id: string, data: Buffer): Promise<void> {
-        let upload = this.largeUploads.get(id);
-        if (!upload) throw new Error(`Unknown large upload ${id}`);
-        await upload.fd.write(data, 0, data.length);
-        upload.size += data.length;
+        await this.getDiskSource().disk.appendLargeUpload(id, data);
     }
     public async finishLargeUpload(id: string, key: string): Promise<void> {
-        let upload = this.largeUploads.get(id);
-        if (!upload) throw new Error(`Unknown large upload ${id}`);
-        this.largeUploads.delete(id);
-        await upload.fd.close();
-        let blobName = `large_${id}.bin`;
-        await fs.promises.rename(upload.tmpPath, this.blobPath(blobName));
-        this.memCache.delete(key);
+        let { disk, sourceIndex } = this.getDiskSource();
+        await disk.finishLargeUpload(id, key);
         this.overlay.delete(key);
-        await this.setIndexEntry(key, { f: blobName, o: 0, l: upload.size, t: Date.now() });
+        let info = await disk.getInfo(key);
+        if (info) {
+            this.setIndexEntry(key, { writeTime: info.writeTime, size: info.size, source: sourceIndex });
+        }
     }
     public async cancelLargeUpload(id: string): Promise<void> {
-        let upload = this.largeUploads.get(id);
-        if (!upload) return;
-        this.largeUploads.delete(id);
-        await upload.fd.close();
-        await fs.promises.unlink(upload.tmpPath).catch(() => { });
+        await this.getDiskSource().disk.cancelLargeUpload(id);
     }
 
-    private async flushOverlay(): Promise<void> {
+    private async flushOverlay(force?: boolean): Promise<void> {
         let now = Date.now();
         for (let [key, entry] of this.overlay) {
-            if (entry.flushAt > now) continue;
+            if (!force && entry.flushAt > now) continue;
             if (entry.data) {
-                let location = await this.appendData(entry.data);
-                await this.setIndexEntry(key, { ...location, t: entry.t });
+                await this.writeToSources(key, entry.data, entry.t);
             } else {
-                let prev = await this.index.get(key);
-                if (prev) {
-                    await this.index.remove(key);
-                    await this.addDeadBytes(prev);
-                }
+                await this.deleteFromSources(key);
             }
             // Only remove if it wasn't overwritten while we were flushing
             if (this.overlay.get(key) === entry) {
@@ -363,41 +554,4 @@ export class BlobStore {
             }
         }
     }
-
-    private async compact(): Promise<void> {
-        let currentBlob = this.blobName(this.currentBlobNumber);
-        for (let blobName of await this.deadBytes.getKeys()) {
-            if (blobName === currentBlob) continue;
-            let dead = await this.deadBytes.get(blobName) || 0;
-            let blobPath = this.blobPath(blobName);
-            let size = 0;
-            try {
-                size = fs.statSync(blobPath).size;
-            } catch {
-                await this.deadBytes.remove(blobName);
-                continue;
-            }
-            if (dead < size * COMPACTION_DEAD_FRACTION) continue;
-
-            console.log(`Compacting blob ${blobName} (${formatNumber(dead)}B dead of ${formatNumber(size)}B)`);
-            for (let key of await this.index.getKeys()) {
-                let entry = await this.index.get(key);
-                if (!entry || entry.f !== blobName) continue;
-                let data = await this.get(key);
-                if (!data) continue;
-                let location = await this.appendData(data);
-                // Only move it if it wasn't rewritten while we were reading
-                let latest = await this.index.get(key);
-                if (latest && latest.f === entry.f && latest.o === entry.o) {
-                    await this.index.set(key, { ...location, t: latest.t });
-                } else {
-                    // Our copy is stale, so its bytes are immediately dead
-                    await this.deadBytes.set(location.f, (await this.deadBytes.get(location.f) || 0) + location.l);
-                }
-            }
-            await this.closeBlobHandle(blobName);
-            await fs.promises.unlink(blobPath).catch(() => { });
-            await this.deadBytes.remove(blobName);
-        }
-    }
 }