sliftutils 1.7.9 → 1.7.11

package/index.d.ts

@@ -136,25 +136,81 @@ declare module "sliftutils/misc/https/certs" {
 
 }
 
+declare module "sliftutils/misc/https/cloudflareHelpers" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    export type CloudflareCreds = {
+        key: string;
+        /** Set for legacy global API keys, which auth via X-Auth-Email/X-Auth-Key. Absent for API tokens, which auth via Authorization: Bearer. */
+        email?: string;
+    };
+    export declare const getCloudflareCreds: {
+        (): Promise<CloudflareCreds>;
+        reset(): void;
+        set(newValue: Promise<CloudflareCreds>): void;
+    };
+    export declare function cloudflareGETCall<T>(path: string, params?: {
+        [key: string]: string;
+    }): Promise<T>;
+    export declare function cloudflarePOSTCall<T>(path: string, params: {
+        [key: string]: unknown;
+    }): Promise<T>;
+    export declare function cloudflareCall<T>(path: string, payload: Buffer, method: string): Promise<T>;
+
+}
+
 declare module "sliftutils/misc/https/dns" {
+    /** Parses our tag back out; 0 (i.e. always stale) when it's absent or unparseable. */
+    export declare function freshnessTime(comment?: string): number;
+    export declare const hasDNSWritePermissions: {
+        (): Promise<boolean>;
+        reset(): void;
+        set(newValue: Promise<boolean>): void;
+    };
+    export declare const getZoneId: {
+        (key: string): Promise<string>;
+        clear(key: string): void;
+        clearAll(): void;
+        forceSet(key: string, value: Promise<string>): void;
+        getAllKeys(): string[];
+        get(key: string): Promise<string> | undefined;
+    };
     export declare function getRecordsRaw(type: string, key: string): Promise<{
         id: string;
         type: string;
         name: string;
         content: string;
         proxied: boolean;
+        modified_on: string;
+        comment?: string | undefined;
     }[]>;
+    /** Cloudflare's batch endpoint applies deletes, then patches, then posts in a single database
+     *   transaction. We route edits (patches) through here because the standalone PATCH/PUT verbs
+     *   aren't usable in our setup, and because it lets "remove others + assert target" happen
+     *   without a window where the name resolves to nothing. */
+    export declare function batchRecords(zoneId: string, batch: {
+        deletes?: {
+            id: string;
+        }[];
+        patches?: {
+            id: string;
+            comment?: string;
+        }[];
+        posts?: {
+            type: string;
+            name: string;
+            content: string;
+            ttl: number;
+            proxied: boolean;
+            comment?: string;
+        }[];
+    }): Promise<void>;
     export declare function getRecords(type: string, key: string): Promise<string[]>;
     export declare function deleteRecord(type: string, key: string, value: string): Promise<void>;
-    /** Removes all existing records (unless the record is already present) */
-    export declare function setRecord(type: string, key: string, value: string, proxied?: "proxied"): Promise<void>;
+    /** Removes all existing records (unless the record is already present and fresh) */
+    export declare function setRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter?: number): Promise<void>;
     /** Keeps existing records */
-    export declare function addRecord(type: string, key: string, value: string, proxied?: "proxied"): Promise<void>;
-    /** Provide Cloudflare credentials directly (an API token, or a path to a file containing one), instead of relying on ./cloudflare.json */
-    export declare function setCloudflareCredentials(config: {
-        key?: string;
-        path?: string;
-    }): void;
+    export declare function addRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter?: number): Promise<void>;
 
 }
 
@@ -163,9 +219,6 @@ declare module "sliftutils/misc/https/hostServer" {
         /** Full domain to host on (e.g. "testsite.example.com"). The HTTPS cert is created for this domain and *.domain, so using a subdomain never touches the root domain (beyond its _acme-challenge TXT record). */
         domain: string;
         port: number;
-        /** Cloudflare API token (or a path to a file containing one). If neither is given, ./cloudflare.json is used. */
-        cloudflareApiToken?: string;
-        cloudflareApiTokenPath?: string;
         /** Creates an unproxied A record pointing domain at this machine (publicIp, or our detected external IP) */
         setDNSRecord?: boolean;
         publicIp?: string;
@@ -796,6 +849,77 @@ declare module "sliftutils/render-utils/observer" {
 
 }
 
+declare module "sliftutils/storage/ArchivesDisk" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    import { IArchives, ArchiveFileInfo, ArchivesConfig } from "./IArchives";
+    export declare class ArchivesDisk implements IArchives {
+        private folder;
+        constructor(folder: string);
+        private filesDir;
+        private uploadsDir;
+        private handles;
+        private largeUploads;
+        private nextLargeUploadId;
+        init: {
+            (): Promise<void>;
+            reset(): void;
+            set(newValue: Promise<void>): void;
+        };
+        getDebugName(): string;
+        getConfig(): Promise<ArchivesConfig>;
+        hasWriteAccess(): Promise<boolean>;
+        private filePath;
+        set(key: string, data: Buffer, config?: {
+            lastModified?: number;
+        }): Promise<void>;
+        del(key: string): Promise<void>;
+        get(key: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<Buffer | undefined>;
+        get2(key: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<{
+            data: Buffer;
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        getInfo(key: string): Promise<{
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        find(prefix: string, config?: {
+            shallow?: boolean;
+            type: "files" | "folders";
+        }): Promise<string[]>;
+        findInfo(prefix: string, config?: {
+            shallow?: boolean;
+            type?: "files" | "folders";
+        }): Promise<ArchiveFileInfo[]>;
+        private collectFiles;
+        setLargeFile(config: {
+            path: string;
+            getNextData(): Promise<Buffer | undefined>;
+        }): Promise<void>;
+        startLargeUpload(): Promise<string>;
+        appendLargeUpload(id: string, data: Buffer): Promise<void>;
+        finishLargeUpload(id: string, key: string): Promise<void>;
+        cancelLargeUpload(id: string): Promise<void>;
+        getURL(path: string): Promise<string>;
+    }
+    export declare function applyFindInfoShape(files: ArchiveFileInfo[], prefix: string, config?: {
+        shallow?: boolean;
+        type?: "files" | "folders";
+    }): ArchiveFileInfo[];
+
+}
+
 declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
     import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing, MergeAttemptResult } from "./BulkDatabaseBase";
     export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
@@ -1972,26 +2096,143 @@ declare module "sliftutils/storage/FileFolderAPI" {
 declare module "sliftutils/storage/IArchives" {
     /// <reference types="node" />
     /// <reference types="node" />
+    export declare const MAX_LAST_MODIFIED_FUTURE: number;
+    export declare const IMMUTABLE_CACHE_TIME: number;
+    export declare function assertValidLastModified(lastModified: number): void;
+    export type RemoteConfig = {
+        version?: number;
+        sources: RemoteConfigBase[];
+    };
+    /**
+        string arguments will be a url, looking like:
+            https://storage2.vidgridweb.com:4445/file/exampleaccount/examplebucket/storage/storagerouting.json
+            https://f002.backblazeb2.com/file/querysubtest-com-public-immutable/storage/storagerouting.json
+            - These map to { url }, with the type inferred from the url
+            - Hosted urls are /file/<account>/<bucketName>/..., backblaze urls are /file/<bucketName>/...
+
+        NOTE: If we do not have right access to these, then it becomes a read-only IArchives, where we solely read using the url form (which might throw due to not having access as well). UNLESS Our configuration explicitly has public: false, in which case, we don't even hit the URL and we throw on access.
+
+        NOTE: If we're in the browser, we should allow downloading the files via the URL form (if it's a public bucket), however, we won't allow writing, because their servers do not allow secure browser writes.
+    */
+    export type RemoteConfigBase = string | HostedConfig | BackblazeConfig;
+    export type CommonConfig = {
+        /** By default a server hosting this bucket eagerly copies this source's full contents onto its own disk (on top of the lazy read-through caching). Set this to be a front end for a very large database without copying the full database - reads still down-cache individual files on demand. */
+        noFullSync?: boolean;
+        /** Bytes of read-cache this server's disk may hold; least-recently-used files are deleted from disk to stay under it (only ever when another source verifiably holds the file - the only copy is never deleted). Requires noFullSync (a full copy can't be bounded). */
+        readerDiskLimit?: number;
+        /** The write times ([startMs, endMs]) this source is valid for (see ArchivesSource.validWindow for the synchronization semantics). Required on object configs: configuration changes must be SCHEDULED (a new source becomes valid at a future time while the old one's window ends), not flipped instantly. Plain URL-string sources default to FULL_VALID_WINDOW - once you're writing object configs, you're doing something complicated enough to think about when things change. */
+        validWindow: [number, number];
+        /** Sharding: the fraction of the key space this source handles, as [start, end) over [0, 1) (keys are routed by getRoute in remoteConfig.ts). Defaults to FULL_ROUTE (unsharded). At every point in time the sources' routes must fully cover [0, 1), or some keys could never be read. */
+        route?: [number, number];
+    };
+    export type HostedConfig = CommonConfig & {
+        type: "remote";
+        url: string;
+        accountName?: string;
+        public?: boolean;
+        fast?: boolean;
+        writeDelay?: number;
+        rawDisk?: boolean;
+        immutable?: boolean;
+    };
+    export type BackblazeConfig = CommonConfig & {
+        type: "backblaze";
+        url: string;
+        public?: boolean;
+        immutable?: boolean;
+    };
+    export declare const FULL_VALID_WINDOW: [number, number];
     export type ArchiveFileInfo = {
         path: string;
         createTime: number;
         size: number;
     };
+    export type SyncActivity = {
+        type: "metadataScan" | "fullSync";
+        sourceDebugName: string;
+        startTime: number;
+        doneFiles?: number;
+        totalFiles?: number;
+        doneBytes?: number;
+        totalBytes?: number;
+    };
+    export type ArchivesConfig = {
+        supportsChangesAfter?: boolean;
+        remoteConfig?: RemoteConfig;
+        index?: {
+            fileCount: number;
+            byteCount: number;
+        };
+        indexSources?: {
+            debugName: string;
+            fileCount: number;
+            byteCount: number;
+        }[];
+        readerDiskLimit?: number;
+        syncing?: SyncActivity[];
+    };
+    export type ArchivesSource = {
+        source: IArchives;
+        validWindow: [number, number];
+        route?: [number, number];
+        noFullSync?: boolean;
+    };
+    export declare const WRITE_PAST_WINDOW_GRACE: number;
+    export declare const STORAGE_WRONG_VALID_WINDOW = "REMOTE_STORAGE_WRONG_VALID_WINDOW_a7c1f04e";
+    export declare const STORAGE_WRONG_ROUTE = "REMOTE_STORAGE_WRONG_ROUTE_c94d2e17";
+    export declare const FULL_ROUTE: [number, number];
+    export declare const VARIABLE_SHARD = "VARIABLE_SHARD_f0234jfah08fgyhfgyssdds83nmp";
+    export declare function windowAcceptsWrites(validWindow: [number, number] | undefined): boolean;
+    export type ArchivesSyncSourceStatus = {
+        debugName: string;
+        validWindow: [number, number];
+        route?: [number, number];
+        noFullSync?: boolean;
+        supportsChangesAfter: boolean;
+        initialScanComplete: boolean;
+        scannedCount: number;
+    };
+    export type ArchivesSyncStatus = {
+        allScansComplete: boolean;
+        indexSize: number;
+        sources: ArchivesSyncSourceStatus[];
+    };
     export interface IArchives {
         getDebugName(): string;
+        /** Whether writes would be accepted (credentials exist, the account trusts this machine, etc).
+         *  Checked without writing anything. */
+        hasWriteAccess(): Promise<boolean>;
         get(fileName: string, config?: {
             range?: {
                 start: number;
                 end: number;
             };
         }): Promise<Buffer | undefined>;
-        set(fileName: string, data: Buffer): Promise<void>;
+        get2(fileName: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<{
+            data: Buffer;
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        /**
+         * lastModified stamps the write with that last-write time instead of now. If it is OLDER than
+         * the file's current last-write time the write no-ops (so delayed / synchronized writes can
+         * never clobber newer data). Times more than 15 minutes in the future are rejected.
+         */
+        set(fileName: string, data: Buffer, config?: {
+            lastModified?: number;
+        }): Promise<void>;
         del(fileName: string): Promise<void>;
         /** Streams a file too large to hold in memory. getNextData returns undefined when done. */
         setLargeFile(config: {
             path: string;
             getNextData(): Promise<Buffer | undefined>;
         }): Promise<void>;
+        /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. */
         getInfo(fileName: string): Promise<{
             writeTime: number;
             size: number;
@@ -2006,6 +2247,15 @@ declare module "sliftutils/storage/IArchives" {
         }): Promise<ArchiveFileInfo[]>;
         /** Only works for public buckets (private buckets are API-access only). */
         getURL(path: string): Promise<string>;
+        /** The bucket's configuration, which tells whether the optional functions are supported. */
+        getConfig(): Promise<ArchivesConfig>;
+        /**
+         * All files changed after the given time. Only exists when getConfig().supportsChangesAfter;
+         * backed by an index, so it is fast (unlike a full findInfo scan). Deletions are not reported.
+         */
+        getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+        /** Synchronization introspection, for backends that synchronize from sources (see BlobStore). */
+        getSyncStatus?(): Promise<ArchivesSyncStatus>;
     }
 
 }
@@ -2266,7 +2516,7 @@ declare module "sliftutils/storage/TransactionStorage" {
 declare module "sliftutils/storage/backblaze" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives } from "./IArchives";
+    import { IArchives, ArchivesConfig } from "./IArchives";
     export declare class ArchivesBackblaze implements IArchives {
         private config;
         constructor(config: {
@@ -2274,6 +2524,7 @@ declare module "sliftutils/storage/backblaze" {
             public?: boolean;
             immutable?: boolean;
             cacheTime?: number;
+            allowedOrigins?: string[];
         });
         private bucketName;
         private bucketId;
@@ -2282,6 +2533,7 @@ declare module "sliftutils/storage/backblaze" {
         private log;
         getDebugName(): string;
         private getBucketAPI;
+        private currentReset;
         private last503Reset;
         private apiRetryLogic;
         get(fileName: string, config?: {
@@ -2291,7 +2543,21 @@ declare module "sliftutils/storage/backblaze" {
             };
             retryCount?: number;
         }): Promise<Buffer | undefined>;
-        set(fileName: string, data: Buffer): Promise<void>;
+        get2(fileName: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<{
+            data: Buffer;
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        getConfig(): Promise<ArchivesConfig>;
+        hasWriteAccess(): Promise<boolean>;
+        set(fileName: string, data: Buffer, config?: {
+            lastModified?: number;
+        }): Promise<void>;
         del(fileName: string): Promise<void>;
         setLargeFile(config: {
             path: string;
@@ -2314,6 +2580,17 @@ declare module "sliftutils/storage/backblaze" {
             size: number;
         }[]>;
         assertPathValid(path: string): Promise<void>;
+        move(config: {
+            path: string;
+            target: IArchives;
+            targetPath: string;
+            copyInstead?: boolean;
+        }): Promise<void>;
+        copy(config: {
+            path: string;
+            target: IArchives;
+            targetPath: string;
+        }): Promise<void>;
         getURL(path: string): Promise<string>;
         getDownloadAuthorization(config: {
             fileNamePrefix?: string;
@@ -2574,23 +2851,16 @@ declare module "sliftutils/storage/remoteFileStorage" {
 declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
     export type ArchivesRemoteConfig = {
-        address: string;
-        port: number;
-        account: string;
-        bucketName: string;
-        public?: boolean;
-        fast?: boolean;
-        writeDelay?: number;
+        url: string;
+        accountName?: string;
+        waitForAccess?: boolean;
     };
-    export declare function buildPublicFileURL(config: {
+    export declare function parseStorageUrl(url: string): {
         address: string;
         port: number;
-        account: string;
-        bucketName: string;
-        path: string;
-    }): string;
+    };
     export declare function authenticateStorage(config: {
         address: string;
         port: number;
@@ -2602,16 +2872,24 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
     export declare class ArchivesRemote implements IArchives {
         private config;
         constructor(config: ArchivesRemoteConfig);
+        private parsed;
+        private account;
+        private bucketName;
         private nodeId;
         private controller;
-        private setupDone;
         private lastDeniedLog;
         getDebugName(): string;
+        isConnected(): boolean;
+        ping(): Promise<void>;
         private authenticate;
         private callAuthed;
-        waitingForAccess(): Promise<string | undefined>;
-        private onAccessDenied;
-        private ensureSetup;
+        waitingForAccess(): Promise<{
+            link: string;
+            machineId: string;
+            ip: string;
+        } | undefined>;
+        hasWriteAccess(): Promise<boolean>;
+        private registerAccessRequest;
         private call;
         get(fileName: string, config?: {
             range?: {
@@ -2619,7 +2897,19 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
                 end: number;
             };
         }): Promise<Buffer | undefined>;
-        set(fileName: string, data: Buffer): Promise<void>;
+        get2(fileName: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<{
+            data: Buffer;
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        set(fileName: string, data: Buffer, config?: {
+            lastModified?: number;
+        }): Promise<void>;
         del(fileName: string): Promise<void>;
         getInfo(fileName: string): Promise<{
             writeTime: number;
@@ -2633,6 +2923,9 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
             shallow?: boolean;
             type: "files" | "folders";
         }): Promise<string[]>;
+        getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+        getConfig(): Promise<ArchivesConfig>;
+        getSyncStatus(): Promise<ArchivesSyncStatus>;
         setLargeFile(config: {
             path: string;
             getNextData(): Promise<Buffer | undefined>;
@@ -2642,6 +2935,58 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
 
 }
 
+declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
+    export declare class ArchivesUrl implements IArchives {
+        private base;
+        constructor(base: string);
+        getDebugName(): string;
+        private readOnlyError;
+        get(fileName: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<Buffer | undefined>;
+        get2(fileName: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<{
+            data: Buffer;
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        getInfo(fileName: string): Promise<{
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        set(fileName: string, data: Buffer, config?: {
+            lastModified?: number;
+        }): Promise<void>;
+        del(fileName: string): Promise<void>;
+        setLargeFile(config: {
+            path: string;
+            getNextData(): Promise<Buffer | undefined>;
+        }): Promise<void>;
+        find(prefix: string, config?: {
+            shallow?: boolean;
+            type: "files" | "folders";
+        }): Promise<string[]>;
+        findInfo(prefix: string, config?: {
+            shallow?: boolean;
+            type: "files" | "folders";
+        }): Promise<ArchiveFileInfo[]>;
+        getURL(path: string): Promise<string>;
+        getConfig(): Promise<ArchivesConfig>;
+        hasWriteAccess(): Promise<boolean>;
+    }
+
+}
+
 declare module "sliftutils/storage/remoteStorage/accessPage" {
     export {};
 
@@ -2650,45 +2995,143 @@ declare module "sliftutils/storage/remoteStorage/accessPage" {
 declare module "sliftutils/storage/remoteStorage/blobStore" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { ArchiveFileInfo } from "../IArchives";
+    import { ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, SyncActivity } from "../IArchives";
     export declare const DEFAULT_FAST_WRITE_DELAY: number;
     export type WriteConfig = {
         fast?: boolean;
         writeDelay?: number;
+        lastModified?: number;
+    };
+    export type IBucketStore = {
+        get(fileName: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<Buffer | undefined>;
+        get2(fileName: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<{
+            data: Buffer;
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        set(fileName: string, data: Buffer, config?: WriteConfig): Promise<void>;
+        del(fileName: string, config?: WriteConfig): Promise<void>;
+        getInfo(fileName: string): Promise<{
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        findInfo(prefix: string, config?: {
+            shallow?: boolean;
+            type?: "files" | "folders";
+        }): Promise<ArchiveFileInfo[]>;
+        getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+        getSyncStatus?(): Promise<ArchivesSyncStatus>;
+        getSyncProgress?(): {
+            index: {
+                fileCount: number;
+                byteCount: number;
+            };
+            syncing: SyncActivity[];
+        };
+        computeIndexTotals?(): Promise<{
+            fileCount: number;
+            byteCount: number;
+        }>;
+        startLargeUpload(): Promise<string>;
+        appendLargeUpload(id: string, data: Buffer): Promise<void>;
+        finishLargeUpload(id: string, key: string): Promise<void>;
+        cancelLargeUpload(id: string): Promise<void>;
     };
-    export declare class BlobStore {
+    export declare class BlobStore implements IBucketStore {
         private folder;
-        constructor(folder: string);
-        private memCache;
-        private overlay;
-        private writeQueue;
-        private openBlobs;
-        private largeUploads;
-        private nextLargeUploadId;
-        private currentBlobNumber;
-        private currentBlobOffset;
-        private currentBlobFd;
+        private sources;
+        private config?;
+        constructor(folder: string, sources: ArchivesSource[], config?: {
+            onIndexChanged?: ((key: string) => void) | undefined;
+            readerDiskLimit?: number | undefined;
+        } | undefined);
+        private stopped;
         private index;
-        private deadBytes;
-        private blobsDir;
+        private mem;
+        private indexFileCount;
+        private indexByteCount;
+        private sourceFileCounts;
+        private sourceByteCounts;
+        private syncActivities;
+        private dirty;
+        private overlay;
+        private sourceStates;
         init: {
             (): Promise<void>;
             reset(): void;
             set(newValue: Promise<void>): void;
         };
-        private blobName;
-        private blobPath;
-        private getBlobHandle;
-        private closeBlobHandle;
-        private addDeadBytes;
-        private appendData;
+        dispose(): Promise<void>;
+        private loadIndex;
+        private countEntry;
         private setIndexEntry;
+        private deleteIndexEntry;
+        /** The cheap always-current totals plus any in-progress background synchronization. */
+        getSyncProgress(): {
+            index: {
+                fileCount: number;
+                byteCount: number;
+            };
+            sources: {
+                debugName: string;
+                fileCount: number;
+                byteCount: number;
+            }[];
+            readerDiskLimit?: number;
+            syncing: SyncActivity[];
+        };
+        /** Walks the whole index for exact totals - more expensive than getSyncProgress, but immune to
+         *  any drift in the maintained counters (and loads the index first, so it's never cold zeros). */
+        computeIndexTotals(): Promise<{
+            fileCount: number;
+            byteCount: number;
+            sources: {
+                debugName: string;
+                fileCount: number;
+                byteCount: number;
+            }[];
+        }>;
+        private flushIndex;
+        private runSourceSync;
+        private scanSource;
+        private reconcileSource;
+        private applyScanned;
+        private pollChanges;
+        private copySourceFiles;
+        private waitForRequiredScans;
+        private checkMissingKey;
+        private getIndexEntry;
+        get(key: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<Buffer | undefined>;
+        get2(key: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<{
+            data: Buffer;
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        private cacheRead;
         set(key: string, data: Buffer, config?: WriteConfig): Promise<void>;
         del(key: string, config?: WriteConfig): Promise<void>;
-        get(key: string, range?: {
-            start: number;
-            end: number;
-        }): Promise<Buffer | undefined>;
+        private getWritableSources;
+        private writeToSources;
         getInfo(key: string): Promise<{
             writeTime: number;
             size: number;
@@ -2697,12 +3140,194 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
             shallow?: boolean;
             type?: "files" | "folders";
         }): Promise<ArchiveFileInfo[]>;
+        getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+        getSyncStatus(): Promise<ArchivesSyncStatus>;
+        private getDiskSource;
         startLargeUpload(): Promise<string>;
         appendLargeUpload(id: string, data: Buffer): Promise<void>;
         finishLargeUpload(id: string, key: string): Promise<void>;
         cancelLargeUpload(id: string): Promise<void>;
         private flushOverlay;
-        private compact;
+        private evicting;
+        private enforceDiskLimit;
+        private cleanupTombstones;
+    }
+
+}
+
+declare module "sliftutils/storage/remoteStorage/cliArgs" {
+    export declare function getArg(name: string): string | undefined;
+
+}
+
+declare module "sliftutils/storage/remoteStorage/createArchives" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+    export declare function createApiArchives(source: HostedConfig | BackblazeConfig): IArchives;
+    export declare class ArchivesChain implements IArchives {
+        private configured;
+        private activeConfig;
+        private statePromise;
+        private initRetryDelay;
+        private initRetryTimer;
+        private pollTimer;
+        private disposed;
+        constructor(config: RemoteConfig | RemoteConfigBase);
+        getDebugName(): string;
+        private getState;
+        private init;
+        private buildSources;
+        private startConfigPoll;
+        private checkForNewConfig;
+        private run;
+        private runWrite;
+        private request;
+        waitingForAccess(): Promise<{
+            link: string;
+            machineId: string;
+            ip: string;
+        } | undefined>;
+        get(fileName: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<Buffer | undefined>;
+        get2(fileName: string, config?: {
+            range?: {
+                start: number;
+                end: number;
+            };
+        }): Promise<{
+            data: Buffer;
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        getInfo(fileName: string): Promise<{
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        private selectCoveringSources;
+        private runOnApi;
+        find(prefix: string, config?: {
+            shallow?: boolean;
+            type: "files" | "folders";
+        }): Promise<string[]>;
+        findInfo(prefix: string, config?: {
+            shallow?: boolean;
+            type: "files" | "folders";
+        }): Promise<ArchiveFileInfo[]>;
+        getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+        getSyncStatus(): Promise<ArchivesSyncStatus>;
+        getConfig(): Promise<ArchivesConfig>;
+        /** True only when EVERY write-receiving source would accept our writes (partial write access
+         *  desynchronizes sources, so it counts as no access). */
+        hasWriteAccess(): Promise<boolean>;
+        private assertNotBareVariableShard;
+        set(fileName: string, data: Buffer, config?: {
+            lastModified?: number;
+        }): Promise<void>;
+        del(fileName: string): Promise<void>;
+        /** Writes a key containing the VARIABLE_SHARD sentinel: picks the lowest-latency up write
+         *  shard, materializes the key with a random value inside that shard's route, writes it, and
+         *  returns the FULL key actually written (the caller needs it to ever read the value back).
+         *  Unlike normal writes this CAN move to another shard when the preferred one is down (error +
+         *  socket down, same rule as reads) - each shard receives a different key, so write
+         *  consistency is preserved. */
+        setVariableShard(key: string, data: Buffer, config?: {
+            lastModified?: number;
+        }): Promise<string>;
+        setLargeFile(config: {
+            path: string;
+            getNextData(): Promise<Buffer | undefined>;
+        }): Promise<void>;
+        getURL(path: string): Promise<string>;
+        dispose(): void;
+    }
+    export declare function createArchives(config: RemoteConfig | RemoteConfigBase): ArchivesChain;
+
+}
+
+declare module "sliftutils/storage/remoteStorage/grantAccessCli" {
+    export {};
+
+}
+
+declare module "sliftutils/storage/remoteStorage/remoteConfig" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } from "../IArchives";
+    export declare const ROUTING_FILE = "storage/storagerouting.json";
+    /** The variable-shard route override embedded in the key ("<sentinel>_<value>", see VARIABLE_SHARD), or undefined when the key has no sentinel or the sentinel has no value yet. */
+    export declare function parseVariableRoute(key: string): number | undefined;
+    /** Where a key routes in [0, 1). A materialized variable-shard suffix completely overrides the hash. */
+    export declare function getRoute(key: string): number;
+    export declare function routeContains(route: [number, number] | undefined, value: number): boolean;
+    export declare function routesOverlap(a: [number, number] | undefined, b: [number, number] | undefined): boolean;
+    /** The overlap of two route ranges, or undefined when they don't overlap. */
+    export declare function routeIntersection(a: [number, number] | undefined, b: [number, number] | undefined): [number, number] | undefined;
+    export declare function getConfigVersion(config: RemoteConfig): number;
+    /** Strips the routing-file suffix, leaving the bucket's public base URL (file paths append to it). */
+    export declare function getBucketBaseUrl(url: string): string;
+    export declare function buildFileUrl(baseUrl: string, filePath: string): string;
+    export declare function parseHostedUrl(url: string): {
+        address: string;
+        port: number;
+        account: string;
+        bucketName: string;
+    };
+    export declare function parseBackblazeUrl(url: string): {
+        bucketName: string;
+    };
+    export declare function normalizeSource(source: RemoteConfigBase): HostedConfig | BackblazeConfig;
+    export declare function normalizeRemoteConfig(config: RemoteConfig | RemoteConfigBase): RemoteConfig;
+    export declare function parseRoutingData(data: Buffer): RemoteConfig;
+    export declare function serializeRemoteConfig(config: RemoteConfig): Buffer;
+
+}
+
+declare module "sliftutils/storage/remoteStorage/sourceWrapper" {
+    import { IArchives, HostedConfig, BackblazeConfig } from "../IArchives";
+    import { ArchivesUrl } from "./ArchivesUrl";
+    export declare const RETRY_START_DELAY: number;
+    export declare const RETRY_MAX_DELAY: number;
+    export declare const RETRY_GROWTH = 1.5;
+    export declare class SourceWrapper {
+        config: HostedConfig | BackblazeConfig;
+        private background;
+        api?: IArchives;
+        url?: ArchivesUrl;
+        writeBlocked?: string;
+        private remote?;
+        private disposed;
+        private reconnectRunning;
+        private accessCache?;
+        private constructor();
+        static create(config: HostedConfig | BackblazeConfig, options?: {
+            background?: boolean;
+        }): Promise<SourceWrapper>;
+        getDebugName(): string;
+        isConnected(): boolean;
+        /** Call after a request failed while isConnected() was false: starts (if not already running)
+         *  the background reconnect loop. Never blocks - the failed request still throws. */
+        noteFailure(): void;
+        private reconnectLoop;
+        private checkAccess;
+        read<T>(run: (archives: IArchives) => Promise<T>): Promise<T>;
+        hasWriteAccess(): Promise<boolean>;
+        private pings;
+        private pingTimer;
+        /** Starts measuring this source's latency (for variable-shard target preference). Only hosted
+         *  remotes are pinged; our own local server counts as 0, everything else as Infinity. */
+        startPinging(): void;
+        /** Median of the recent pings. Sources that can't be pinged sort last (Infinity), except our
+         *  own in-process server, which is the best possible target (0). */
+        getLatency(): number;
+        /** Writes always go through the API, so a permission error throws to the caller on every write
+         *  (and access granted in the meantime is picked up automatically). */
+        write<T>(run: (archives: IArchives) => Promise<T>): Promise<T>;
+        dispose(): void;
     }
 
 }
@@ -2710,9 +3335,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
 declare module "sliftutils/storage/remoteStorage/storageController" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { ArchiveFileInfo } from "../IArchives";
-    import type { BlobStore } from "./blobStore";
-    import type { IStorage } from "../IStorage";
+    import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
     export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
     export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
     export declare const STORAGE_NOT_AUTHENTICATED = "REMOTE_STORAGE_NOT_AUTHENTICATED_cf2f7b1e";
@@ -2735,34 +3358,15 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
         ip: string;
         time: number;
     };
-    export type BucketConfig = {
-        public?: boolean;
-        fast?: boolean;
-        writeDelay?: number;
-    };
     export type AccessState = {
         machineId: string;
         ip: string;
         hasAccess: boolean;
-        listAccessCommand: string;
         grantAccessCommand?: string;
-        machines?: (AccessRequest & {
-            trusted: boolean;
-        })[];
-    };
-    export type StorageServerState = {
-        domain: string;
-        port: number;
-        rootDomain: string;
-        sshTarget: string;
-        serverCommand: string;
-        blobStore: BlobStore;
-        trust: IStorage<TrustRecord>;
-        requests: IStorage<AccessRequest[]>;
-        buckets: IStorage<BucketConfig>;
+        trustedMachines?: TrustRecord[];
     };
-    export declare function setStorageServerState(state: StorageServerState): void;
     export declare const RemoteStorageController: import("socket-function/SocketFunctionTypes").SocketRegistered<{
+        ping: () => Promise<void>;
         authenticate: (token: AuthToken) => Promise<{
             machineId: string;
             ip: string;
@@ -2774,14 +3378,23 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
             grantAccessCommand: string;
         }>;
         getAccessState: (account: string) => Promise<AccessState>;
+        listRequestsForIP: (account: string, ip: string) => Promise<AccessRequest[]>;
+        grantAccess: (requestId: string) => Promise<TrustRecord>;
         adminListRequests: (ip: string) => Promise<AccessRequest[]>;
         adminGrantAccess: (requestId: string) => Promise<TrustRecord>;
-        ensureBucket: (account: string, bucketName: string, config: BucketConfig) => Promise<void>;
         get: (account: string, bucketName: string, path: string, range?: {
             start: number;
             end: number;
         }) => Promise<Buffer | undefined>;
-        set: (account: string, bucketName: string, path: string, data: Buffer) => Promise<void>;
+        get2: (account: string, bucketName: string, path: string, range?: {
+            start: number;
+            end: number;
+        }) => Promise<{
+            data: Buffer;
+            writeTime: number;
+            size: number;
+        } | undefined>;
+        set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number) => Promise<void>;
         del: (account: string, bucketName: string, path: string) => Promise<void>;
         getInfo: (account: string, bucketName: string, path: string) => Promise<{
             writeTime: number;
@@ -2791,17 +3404,90 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
             shallow?: boolean;
             type?: "files" | "folders";
         }) => Promise<ArchiveFileInfo[]>;
+        getChangesAfter: (account: string, bucketName: string, time: number) => Promise<ArchiveFileInfo[]>;
+        getArchivesConfig: (account: string, bucketName: string) => Promise<ArchivesConfig>;
+        getIndexInfo: (account: string, bucketName: string) => Promise<{
+            fileCount: number;
+            byteCount: number;
+            sources: {
+                debugName: string;
+                fileCount: number;
+                byteCount: number;
+            }[];
+        } | undefined>;
+        getSyncStatus: (account: string, bucketName: string) => Promise<ArchivesSyncStatus>;
         startLargeFile: (account: string, bucketName: string, path: string) => Promise<string>;
         uploadPart: (uploadId: string, data: Buffer) => Promise<void>;
         finishLargeFile: (uploadId: string) => Promise<void>;
         cancelLargeFile: (uploadId: string) => Promise<void>;
-        getPublicFile: (account: string, bucketName: string, path: string) => Promise<Buffer>;
+        httpEntry: (config?: {
+            requireCalls?: string[];
+            cacheTime?: number;
+        }) => Promise<Buffer>;
     }>;
 
 }
 
 declare module "sliftutils/storage/remoteStorage/storageServer" {
     import "./accessPage";
+    export type HostStorageServerConfig = {
+        url: string;
+        folder: string;
+        lowSpaceThresholdBytes?: number;
+    };
+    export declare function hostStorageServer(config: HostStorageServerConfig): Promise<void>;
+
+}
+
+declare module "sliftutils/storage/remoteStorage/storageServerCli" {
+    export {};
+
+}
+
+declare module "sliftutils/storage/remoteStorage/storageServerState" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    import { IBucketStore } from "./blobStore";
+    import { RemoteConfig, HostedConfig, IArchives } from "../IArchives";
+    import type { IStorage } from "../IStorage";
+    import type { AccessRequest, TrustRecord } from "./storageController";
+    export type StorageServerConfig = {
+        domain: string;
+        port: number;
+        rootDomain: string;
+        sshTarget: string;
+        serverCommand: string;
+        folder: string;
+    };
+    export declare function setStorageServerConfig(value: StorageServerConfig): void;
+    export declare function getStorageServerConfig(): StorageServerConfig;
+    export declare function getStorageServerConfigOptional(): StorageServerConfig | undefined;
+    export declare function setWritesRejectedReason(reason: string | undefined): void;
+    export declare function getWritesRejectedReason(): string | undefined;
+    export declare function assertWritesAllowed(): void;
+    export declare function getTrust(): Promise<IStorage<TrustRecord>>;
+    export declare function getRequests(): Promise<IStorage<AccessRequest[]>>;
+    /** Makes machineIds the complete trust list for the account: machines not in the list lose access, machines already trusted keep their existing record, and missing ones are added. */
+    export declare function setTrustedMachines(config: {
+        account: string;
+        machineIds: string[];
+    }): Promise<void>;
+    export type LoadedBucket = {
+        account: string;
+        bucketName: string;
+        routing: RemoteConfig;
+        routingJSON: string;
+        selfEntries: HostedConfig[];
+        self: HostedConfig | undefined;
+        store: IBucketStore;
+    };
+    export declare function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined>;
+    export declare function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void>;
+    export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
+        lastModified?: number;
+    }): Promise<void>;
+    export declare function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void>;
+    export declare function getLocalArchives(account: string, bucketName: string): IArchives;
 
 }