sliftutils 1.7.8 → 1.7.10

package/storage/ArchivesDisk.ts

@@ -0,0 +1,365 @@
+import fs from "fs";
+import path from "path";
+import { lazy } from "socket-function/src/caching";
+import { runInfinitePoll } from "socket-function/src/batching";
+import { sort, binarySearchBasic } from "socket-function/src/misc";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, assertValidLastModified } from "./IArchives";
+
+// The base file-system IArchives: storage is one-to-one with the file system, every key is exactly
+// one real file under <folder>/files, so the file system itself is the index. File handles are
+// cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in
+// serial, so they can't collide with each other or with handle closing. Used as the disk
+// synchronization source of BlobStore (see remoteStorage/blobStore.ts).
+
+const HANDLE_IDLE_TIMEOUT = 1000 * 60;
+const HANDLE_SWEEP_INTERVAL = 1000 * 15;
+
+type HandleEntry = {
+    filePath: string;
+    handle: fs.promises.FileHandle;
+    lastUse: number;
+};
+
+// Caches open file handles, closing them once idle for HANDLE_IDLE_TIMEOUT. Instead of one
+// setTimeout per handle, a list sorted by last use is swept periodically (entries are moved via
+// binary search on access, and lastUse values only increase, so touched entries append at the
+// end). Also serializes operations per file: each operation only starts once the previous one on
+// the same file finished, and a handle is never closed while an operation is pending on it.
+class FileHandleCache {
+    private entries = new Map<string, HandleEntry>();
+    // Sorted by lastUse ascending (least recently used first)
+    private lru: HandleEntry[] = [];
+    private pending = new Map<string, Promise<void>>();
+
+    constructor() {
+        runInfinitePoll(HANDLE_SWEEP_INTERVAL, () => this.sweep());
+    }
+
+    // Runs fnc after every previously scheduled operation on filePath has finished
+    public run<T>(filePath: string, fnc: () => Promise<T>): Promise<T> {
+        let prev = this.pending.get(filePath) || Promise.resolve();
+        let result = prev.then(fnc);
+        // The pending chain must never reject, or one failed operation would poison all later ones
+        let last = result.then(() => { }, () => { });
+        this.pending.set(filePath, last);
+        void last.then(() => {
+            // If we're still the last pending operation on this file, clear ourselves
+            if (this.pending.get(filePath) === last) {
+                this.pending.delete(filePath);
+            }
+        });
+        return result;
+    }
+
+    // Only call inside run() for the same filePath (so opens can't collide with closes)
+    public async getHandle(filePath: string, flags: number): Promise<fs.promises.FileHandle> {
+        let entry = this.entries.get(filePath);
+        if (entry) {
+            this.removeFromLRU(entry);
+            entry.lastUse = Date.now();
+            this.lru.push(entry);
+            return entry.handle;
+        }
+        let handle = await fs.promises.open(filePath, flags);
+        entry = { filePath, handle, lastUse: Date.now() };
+        this.entries.set(filePath, entry);
+        this.lru.push(entry);
+        return handle;
+    }
+
+    // Only call inside run() for the same filePath
+    public async closeNow(filePath: string): Promise<void> {
+        let entry = this.entries.get(filePath);
+        if (!entry) return;
+        this.entries.delete(filePath);
+        this.removeFromLRU(entry);
+        await entry.handle.close();
+    }
+
+    private removeFromLRU(entry: HandleEntry) {
+        let index = binarySearchBasic(this.lru, x => x.lastUse, entry.lastUse);
+        if (index < 0) return;
+        // Multiple entries can share a lastUse, so scan for the exact one
+        while (index < this.lru.length && this.lru[index].lastUse === entry.lastUse) {
+            if (this.lru[index] === entry) {
+                this.lru.splice(index, 1);
+                return;
+            }
+            index++;
+        }
+    }
+
+    private async sweep(): Promise<void> {
+        let cutoff = Date.now() - HANDLE_IDLE_TIMEOUT;
+        let index = 0;
+        while (index < this.lru.length && this.lru[index].lastUse <= cutoff) {
+            let entry = this.lru[index];
+            // Can't close a handle with a pending operation; it'll be swept after it finishes
+            if (this.pending.has(entry.filePath)) {
+                index++;
+                continue;
+            }
+            this.lru.splice(index, 1);
+            this.entries.delete(entry.filePath);
+            await entry.handle.close();
+        }
+    }
+}
+
+export class ArchivesDisk implements IArchives {
+    constructor(private folder: string) { }
+
+    private filesDir = path.join(this.folder, "files");
+    private uploadsDir = path.join(this.folder, "uploads");
+    private handles = new FileHandleCache();
+    private largeUploads = new Map<string, { tmpPath: string }>();
+    private nextLargeUploadId = 1;
+
+    public init = lazy(async () => {
+        await fs.promises.mkdir(this.filesDir, { recursive: true });
+        await fs.promises.mkdir(this.uploadsDir, { recursive: true });
+        // Uploads don't survive restarts (the uploader streams into them), so old ones are garbage
+        for (let file of await fs.promises.readdir(this.uploadsDir)) {
+            await fs.promises.unlink(path.join(this.uploadsDir, file));
+        }
+    });
+
+    public getDebugName() {
+        return `disk/${this.folder}`;
+    }
+
+    public async getConfig(): Promise<ArchivesConfig> {
+        return {};
+    }
+
+    private filePath(key: string): string {
+        let result = path.join(this.filesDir, key);
+        if (!result.startsWith(this.filesDir + path.sep)) {
+            throw new Error(`Invalid key ${JSON.stringify(key.slice(0, 200))}, it escapes the store folder`);
+        }
+        return result;
+    }
+
+    public async set(key: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
+        await this.init();
+        let lastModified = config?.lastModified;
+        if (lastModified) {
+            assertValidLastModified(lastModified);
+        }
+        let filePath = this.filePath(key);
+        await this.handles.run(filePath, async () => {
+            if (lastModified) {
+                let existing = await statOrUndefined(filePath);
+                // An older write never overwrites a newer one (see IArchives.set)
+                if (existing && lastModified < existing.mtimeMs) return;
+            }
+            await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+            let handle = await this.handles.getHandle(filePath, fs.constants.O_RDWR | fs.constants.O_CREAT);
+            await handle.truncate(0);
+            await handle.write(data, 0, data.length, 0);
+            if (lastModified) {
+                await handle.utimes(new Date(lastModified), new Date(lastModified));
+            }
+        });
+    }
+
+    public async del(key: string): Promise<void> {
+        await this.init();
+        let filePath = this.filePath(key);
+        await this.handles.run(filePath, async () => {
+            await this.handles.closeNow(filePath);
+            try {
+                await fs.promises.unlink(filePath);
+            } catch (e: any) {
+                if (e.code !== "ENOENT") throw e;
+            }
+        });
+    }
+
+    public async get(key: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
+        let result = await this.get2(key, config);
+        return result && result.data || undefined;
+    }
+
+    public async get2(key: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
+        await this.init();
+        let range = config?.range;
+        let filePath = this.filePath(key);
+        return await this.handles.run(filePath, async () => {
+            let handle: fs.promises.FileHandle;
+            try {
+                handle = await this.handles.getHandle(filePath, fs.constants.O_RDWR);
+            } catch (e: any) {
+                if (e.code === "ENOENT") return undefined;
+                throw e;
+            }
+            let stats = await handle.stat();
+            let size = stats.size;
+            let start = range && Math.min(range.start, size) || 0;
+            let end = range && Math.min(range.end, size) || size;
+            if (end <= start) return { data: Buffer.alloc(0), writeTime: stats.mtimeMs };
+            let buffer = Buffer.alloc(end - start);
+            let { bytesRead } = await handle.read(buffer, 0, buffer.length, start);
+            if (bytesRead !== buffer.length) {
+                throw new Error(`Expected ${buffer.length} bytes at ${filePath}:${start}, read ${bytesRead}`);
+            }
+            return { data: buffer, writeTime: stats.mtimeMs };
+        });
+    }
+
+    public async getInfo(key: string): Promise<{ writeTime: number; size: number } | undefined> {
+        await this.init();
+        let filePath = this.filePath(key);
+        return await this.handles.run(filePath, async () => {
+            let stats = await statOrUndefined(filePath);
+            if (!stats || !stats.isFile()) return undefined;
+            return { writeTime: stats.mtimeMs, size: stats.size };
+        });
+    }
+
+    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
+        return (await this.findInfo(prefix, config)).map(x => x.path);
+    }
+
+    public async findInfo(prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
+        await this.init();
+        let infos = new Map<string, ArchiveFileInfo>();
+        await this.collectFiles("", prefix, infos);
+        let files = Array.from(infos.values());
+        files = applyFindInfoShape(files, prefix, config);
+        sort(files, x => x.path);
+        return files;
+    }
+
+    // relDir is "" or ends with "/". Only descends into directories that can still match the prefix.
+    private async collectFiles(relDir: string, prefix: string, infos: Map<string, ArchiveFileInfo>): Promise<void> {
+        let entries: fs.Dirent[];
+        try {
+            entries = await fs.promises.readdir(path.join(this.filesDir, relDir), { withFileTypes: true });
+        } catch (e: any) {
+            if (e.code === "ENOENT") return;
+            throw e;
+        }
+        for (let entry of entries) {
+            let relPath = relDir + entry.name;
+            if (entry.isDirectory()) {
+                let dirPath = relPath + "/";
+                if (dirPath.startsWith(prefix) || prefix.startsWith(dirPath)) {
+                    await this.collectFiles(dirPath, prefix, infos);
+                }
+                continue;
+            }
+            if (!entry.isFile()) continue;
+            if (!relPath.startsWith(prefix)) continue;
+            let stats = await statOrUndefined(path.join(this.filesDir, relPath));
+            // Deleted while we were walking
+            if (!stats) continue;
+            infos.set(relPath, { path: relPath, createTime: stats.mtimeMs, size: stats.size });
+        }
+    }
+
+    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
+        let id = await this.startLargeUpload();
+        try {
+            while (true) {
+                let data = await config.getNextData();
+                if (!data) break;
+                await this.appendLargeUpload(id, data);
+            }
+            await this.finishLargeUpload(id, config.path);
+        } catch (e) {
+            await this.cancelLargeUpload(id);
+            throw e;
+        }
+    }
+
+    // Large files stream into their own file under uploads/, then move into place on finish. No
+    // handle is opened until the first append actually happens.
+    public async startLargeUpload(): Promise<string> {
+        await this.init();
+        let id = `${Date.now()}_${this.nextLargeUploadId++}`;
+        this.largeUploads.set(id, { tmpPath: path.join(this.uploadsDir, `upload_${id}.tmp`) });
+        return id;
+    }
+    public async appendLargeUpload(id: string, data: Buffer): Promise<void> {
+        let upload = this.largeUploads.get(id);
+        if (!upload) throw new Error(`Unknown large upload ${id}`);
+        const tmpPath = upload.tmpPath;
+        await this.handles.run(tmpPath, async () => {
+            let handle = await this.handles.getHandle(tmpPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_APPEND);
+            await handle.write(data, 0, data.length);
+        });
+    }
+    public async finishLargeUpload(id: string, key: string): Promise<void> {
+        let upload = this.largeUploads.get(id);
+        if (!upload) throw new Error(`Unknown large upload ${id}`);
+        const tmpPath = upload.tmpPath;
+        this.largeUploads.delete(id);
+        let filePath = this.filePath(key);
+        await this.handles.run(tmpPath, () => this.handles.closeNow(tmpPath));
+        await this.handles.run(filePath, async () => {
+            // Close any cached handle to the file we're replacing, so later reads reopen the new file
+            await this.handles.closeNow(filePath);
+            await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+            try {
+                await fs.promises.rename(tmpPath, filePath);
+            } catch (e: any) {
+                if (e.code !== "ENOENT") throw e;
+                // Nothing was ever appended, so the upload file was never created
+                await fs.promises.writeFile(filePath, Buffer.alloc(0));
+            }
+        });
+    }
+    public async cancelLargeUpload(id: string): Promise<void> {
+        let upload = this.largeUploads.get(id);
+        if (!upload) return;
+        const tmpPath = upload.tmpPath;
+        this.largeUploads.delete(id);
+        await this.handles.run(tmpPath, async () => {
+            await this.handles.closeNow(tmpPath);
+            try {
+                await fs.promises.unlink(tmpPath);
+            } catch (e: any) {
+                if (e.code !== "ENOENT") throw e;
+            }
+        });
+    }
+
+    public async getURL(path: string): Promise<string> {
+        throw new Error(`getURL is not supported for disk archives (${this.getDebugName()}, path ${path})`);
+    }
+}
+
+async function statOrUndefined(filePath: string): Promise<fs.Stats | undefined> {
+    try {
+        return await fs.promises.stat(filePath);
+    } catch (e: any) {
+        if (e.code === "ENOENT") return undefined;
+        throw e;
+    }
+}
+
+// The folders/shallow post-processing shared by findInfo implementations that list flat files
+// (used by ArchivesDisk on the raw disk walk, and by BlobStore on its index).
+export function applyFindInfoShape(files: ArchiveFileInfo[], prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): ArchiveFileInfo[] {
+    if (config?.type === "folders") {
+        let folders = new Map<string, ArchiveFileInfo>();
+        for (let file of files) {
+            let rest = file.path.slice(prefix.length);
+            let restParts = rest.split("/");
+            if (restParts.length < 2) continue;
+            let folder: string;
+            if (config.shallow) {
+                folder = prefix + restParts[0];
+            } else {
+                folder = file.path.split("/").slice(0, -1).join("/");
+            }
+            folders.set(folder, { path: folder, createTime: file.createTime, size: file.size });
+        }
+        return Array.from(folders.values());
+    }
+    if (config?.shallow) {
+        return files.filter(file => !file.path.slice(prefix.length).includes("/"));
+    }
+    return files;
+}

package/storage/IArchives.d.ts

@@ -1,10 +1,77 @@
 /// <reference types="node" />
 /// <reference types="node" />
+export declare const MAX_LAST_MODIFIED_FUTURE: number;
+export declare function assertValidLastModified(lastModified: number): void;
+export type RemoteConfig = {
+    version?: number;
+    sources: RemoteConfigBase[];
+};
+/**
+    string arguments will be a url, looking like:
+        https://storage2.vidgridweb.com:4445/file/exampleaccount/examplebucket/storage/storagerouting.json
+        https://f002.backblazeb2.com/file/querysubtest-com-public-immutable/storage/storagerouting.json
+        - These map to { url }, with the type inferred from the url
+        - Hosted urls are /file/<account>/<bucketName>/..., backblaze urls are /file/<bucketName>/...
+
+    NOTE: If we do not have right access to these, then it becomes a read-only IArchives, where we solely read using the url form (which might throw due to not having access as well). UNLESS Our configuration explicitly has public: false, in which case, we don't even hit the URL and we throw on access.
+
+    NOTE: If we're in the browser, we should allow downloading the files via the URL form (if it's a public bucket), however, we won't allow writing, because their servers do not allow secure browser writes.
+*/
+export type RemoteConfigBase = string | HostedConfig | BackblazeConfig;
+export type CommonConfig = {
+    /** The default options for the first config in a list is DEFAULT_BASE_SYNC_OPTIONS. The rest default to DEFAULT_SYNC_OPTIONS. */
+    syncOptions?: SyncOptions;
+};
+export type HostedConfig = CommonConfig & {
+    type: "remote";
+    url: string;
+    accountName?: string;
+    public?: boolean;
+    fast?: boolean;
+    writeDelay?: number;
+    rawDisk?: boolean;
+    immutable?: boolean;
+};
+export type BackblazeConfig = CommonConfig & {
+    type: "backblaze";
+    url: string;
+    bucketName: string;
+    public?: boolean;
+    immutable?: boolean;
+};
+export declare const DEFAULT_BASE_SYNC_OPTIONS: SyncOptions;
+export declare const DEFAULT_SYNC_OPTIONS: SyncOptions;
 export type ArchiveFileInfo = {
     path: string;
     createTime: number;
     size: number;
 };
+export type ArchivesConfig = {
+    supportsChangesAfter?: boolean;
+};
+export type SyncOptions = {
+    copyFiles?: boolean;
+    noWriteBack?: boolean;
+    cacheReads?: boolean;
+    validWindow: [number, number];
+    required?: boolean;
+};
+export type ArchivesSource = {
+    source: IArchives;
+    options: SyncOptions;
+};
+export type ArchivesSyncSourceStatus = {
+    debugName: string;
+    options: SyncOptions;
+    supportsChangesAfter: boolean;
+    initialScanComplete: boolean;
+    scannedCount: number;
+};
+export type ArchivesSyncStatus = {
+    allScansComplete: boolean;
+    indexSize: number;
+    sources: ArchivesSyncSourceStatus[];
+};
 export interface IArchives {
     getDebugName(): string;
     get(fileName: string, config?: {
@@ -13,13 +80,31 @@ export interface IArchives {
             end: number;
         };
     }): Promise<Buffer | undefined>;
-    set(fileName: string, data: Buffer): Promise<void>;
+    /** Like get, but also returns the last-write time of the file. get just calls get2. */
+    get2(fileName: string, config?: {
+        range?: {
+            start: number;
+            end: number;
+        };
+    }): Promise<{
+        data: Buffer;
+        writeTime: number;
+    } | undefined>;
+    /**
+     * lastModified stamps the write with that last-write time instead of now. If it is OLDER than
+     * the file's current last-write time the write no-ops (so delayed / synchronized writes can
+     * never clobber newer data). Times more than 15 minutes in the future are rejected.
+     */
+    set(fileName: string, data: Buffer, config?: {
+        lastModified?: number;
+    }): Promise<void>;
     del(fileName: string): Promise<void>;
     /** Streams a file too large to hold in memory. getNextData returns undefined when done. */
     setLargeFile(config: {
         path: string;
         getNextData(): Promise<Buffer | undefined>;
     }): Promise<void>;
+    /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. */
     getInfo(fileName: string): Promise<{
         writeTime: number;
         size: number;
@@ -34,4 +119,13 @@ export interface IArchives {
     }): Promise<ArchiveFileInfo[]>;
     /** Only works for public buckets (private buckets are API-access only). */
     getURL(path: string): Promise<string>;
+    /** The bucket's configuration, which tells whether the optional functions are supported. */
+    getConfig(): Promise<ArchivesConfig>;
+    /**
+     * All files changed after the given time. Only exists when getConfig().supportsChangesAfter;
+     * backed by an index, so it is fast (unlike a full findInfo scan). Deletions are not reported.
+     */
+    getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+    /** Synchronization introspection, for backends that synchronize from sources (see BlobStore). */
+    getSyncStatus?(): Promise<ArchivesSyncStatus>;
 }

package/storage/IArchives.ts

@@ -3,18 +3,163 @@ module.allowclient = true;
 // The important operations of an archive bucket (extracted from ArchivesBackblaze), so other
 // backends (e.g. our own remote storage server) can be used interchangeably.
 
+// A write may not be stamped more than this far in the future, or clock skew between machines
+// would let a bad timestamp block writes for a long time.
+export const MAX_LAST_MODIFIED_FUTURE = 15 * 60 * 1000;
+export function assertValidLastModified(lastModified: number): void {
+    let max = Date.now() + MAX_LAST_MODIFIED_FUTURE;
+    if (lastModified > max) {
+        throw new Error(`lastModified is too far in the future: ${lastModified} > ${max} (now + 15 minutes)`);
+    }
+}
+
+
+export type RemoteConfig = {
+    // NOTE: Version is used when updating the configuration. The newer version is always taken. A missing version counts as version -1.
+    version?: number;
+    sources: RemoteConfigBase[];
+};
+
+/**
+    string arguments will be a url, looking like:
+        https://storage2.vidgridweb.com:4445/file/exampleaccount/examplebucket/storage/storagerouting.json
+        https://f002.backblazeb2.com/file/querysubtest-com-public-immutable/storage/storagerouting.json
+        - These map to { url }, with the type inferred from the url
+        - Hosted urls are /file/<account>/<bucketName>/..., backblaze urls are /file/<bucketName>/...
+
+    NOTE: If we do not have right access to these, then it becomes a read-only IArchives, where we solely read using the url form (which might throw due to not having access as well). UNLESS Our configuration explicitly has public: false, in which case, we don't even hit the URL and we throw on access.
+
+    NOTE: If we're in the browser, we should allow downloading the files via the URL form (if it's a public bucket), however, we won't allow writing, because their servers do not allow secure browser writes.
+*/
+export type RemoteConfigBase = string | HostedConfig | BackblazeConfig;
+
+export type CommonConfig = {
+    /** The default options for the first config in a list is DEFAULT_BASE_SYNC_OPTIONS. The rest default to DEFAULT_SYNC_OPTIONS. */
+    syncOptions?: SyncOptions;
+};
+
+export type HostedConfig = CommonConfig & {
+    type: "remote";
+
+    // Ex: https://storage2.vidgridweb.com:4445/file/exampleaccount/examplebucket/storage/storagerouting.json
+    // NOTE: The account and bucket name are obtained from the URL.
+    url: string;
+
+    // NOTE: Authentication is handled by cert.ts, via having your machine trusted to access this account. 
+    accountName?: string;
+
+    public?: boolean;
+    // Fast mode: the server acknowledges writes once they are in memory, flushing to disk after
+    // writeDelay (default 5 minutes) and coalescing writes to the same file. A server crash loses
+    // writes that haven't flushed yet.
+    fast?: boolean;
+    writeDelay?: number;
+    // The bucket is served straight from the server's disk, with no index — so no fast writes and
+    // no getChangesAfter/getSyncStatus.
+    rawDisk?: boolean;
+    // Writes to paths that already exist are disallowed (deletes still work).
+    immutable?: boolean;
+};
+
+export type BackblazeConfig = CommonConfig & {
+    type: "backblaze";
+    // Ex: https://f002.backblazeb2.com/file/querysubtest-com-public-immutable/storage/storagerouting.json
+    // NOTE: The bucket name is obtained from the URL.
+    url: string;
+    // Public buckets are served over plain HTTPS GETs (getURL). Private buckets are API-access only.
+    bucketName: string;
+    public?: boolean;
+    // NOTE: This isn't enforced on the backblaze level, so this is just a client-side guarantee. This can change how we cache files.
+    //  - Backblaze does support immutability. However, apparently, once we enable it on a bucket, we can't disable it, which is really bad, as it means if our code could ever enable it and we accidentally enable it on an important bucket, we essentially just bricked that bucket. So we should never write any code that ever tries to use backblaze to make things immutable. 
+    immutable?: boolean;
+
+    // NOTE: We will access the api key from getSecret, see backblaze.ts for the specific keys.
+};
+
+
+export const DEFAULT_BASE_SYNC_OPTIONS: SyncOptions = {
+    cacheReads: true,
+    required: true,
+    validWindow: [0, Number.MAX_SAFE_INTEGER],
+};
+export const DEFAULT_SYNC_OPTIONS: SyncOptions = {
+    validWindow: [0, Number.MAX_SAFE_INTEGER],
+};
+
+
+
+// createTime is a misnomer kept for compatibility — it is really the LAST-WRITE time, same as
+// getInfo's writeTime. Neither Backblaze nor our remote storage tracks a distinct creation date:
+// each write stamps a fresh timestamp on the current version, so both fields are just "when the
+// bytes served by get() were most recently written".
 export type ArchiveFileInfo = { path: string; createTime: number; size: number };
 
+export type ArchivesConfig = {
+    // Whether getChangesAfter is implemented (fast change polling, instead of full rescans)
+    supportsChangesAfter?: boolean;
+};
+
+// How a synchronization source behaves (see BlobStore, which synchronizes an index + local cache
+// from a list of { source, options }).
+export type SyncOptions = {
+    // After the source's metadata is copied into the index, also copy the file contents over (into
+    // the cacheReads sources), preserving their modified times.
+    copyFiles?: boolean;
+    // Writes are NOT written to this source (by default every source receives writes).
+    noWriteBack?: boolean;
+    // If a read wasn't served by this source, write the data back to it (using it as a cache), and
+    // update the index so it becomes the new source. Set for the local disk source.
+    cacheReads?: boolean;
+    // Ignore values with write times outside [start, end].
+    validWindow: [number, number];
+    // If a file isn't in the index and this source hasn't finished its initial scan, check the
+    // source directly before declaring the file nonexistent. Set for the disk source.
+    required?: boolean;
+};
+export type ArchivesSource = { source: IArchives; options: SyncOptions };
+
+export type ArchivesSyncSourceStatus = {
+    debugName: string;
+    options: SyncOptions;
+    supportsChangesAfter: boolean;
+    initialScanComplete: boolean;
+    // Files seen in this source's scans / change polls so far
+    scannedCount: number;
+};
+export type ArchivesSyncStatus = {
+    allScansComplete: boolean;
+    // Number of files in the index
+    indexSize: number;
+    sources: ArchivesSyncSourceStatus[];
+};
+
 export interface IArchives {
     getDebugName(): string;
     get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined>;
-    set(fileName: string, data: Buffer): Promise<void>;
+    /** Like get, but also returns the last-write time of the file. get just calls get2. */
+    get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined>;
+    /**
+     * lastModified stamps the write with that last-write time instead of now. If it is OLDER than
+     * the file's current last-write time the write no-ops (so delayed / synchronized writes can
+     * never clobber newer data). Times more than 15 minutes in the future are rejected.
+     */
+    set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void>;
     del(fileName: string): Promise<void>;
     /** Streams a file too large to hold in memory. getNextData returns undefined when done. */
     setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void>;
+    /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. */
     getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined>;
     find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]>;
     findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]>;
     /** Only works for public buckets (private buckets are API-access only). */
     getURL(path: string): Promise<string>;
+    /** The bucket's configuration, which tells whether the optional functions are supported. */
+    getConfig(): Promise<ArchivesConfig>;
+    /**
+     * All files changed after the given time. Only exists when getConfig().supportsChangesAfter;
+     * backed by an index, so it is fast (unlike a full findInfo scan). Deletions are not reported.
+     */
+    getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+    /** Synchronization introspection, for backends that synchronize from sources (see BlobStore). */
+    getSyncStatus?(): Promise<ArchivesSyncStatus>;
 }

package/storage/backblaze.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives } from "./IArchives";
+import { IArchives, ArchivesConfig } from "./IArchives";
 export declare class ArchivesBackblaze implements IArchives {
     private config;
     constructor(config: {
@@ -25,7 +25,19 @@ export declare class ArchivesBackblaze implements IArchives {
         };
         retryCount?: number;
     }): Promise<Buffer | undefined>;
-    set(fileName: string, data: Buffer): Promise<void>;
+    get2(fileName: string, config?: {
+        range?: {
+            start: number;
+            end: number;
+        };
+    }): Promise<{
+        data: Buffer;
+        writeTime: number;
+    } | undefined>;
+    getConfig(): Promise<ArchivesConfig>;
+    set(fileName: string, data: Buffer, config?: {
+        lastModified?: number;
+    }): Promise<void>;
     del(fileName: string): Promise<void>;
     setLargeFile(config: {
         path: string;