sliftutils 1.7.9 → 1.7.11

package/storage/remoteStorage/grantAccess.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+"use strict";
+require("typenode");
+require("./grantAccessCli");

package/storage/remoteStorage/grantAccessCli.d.ts

@@ -0,0 +1 @@
+export {};

package/storage/remoteStorage/grantAccessCli.ts

@@ -0,0 +1,27 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { RemoteStorageController } from "./storageController";
+import { authenticateStorage, parseStorageUrl } from "./ArchivesRemote";
+import { getArg } from "./cliArgs";
+
+// The grantAccess CLI. Invoked via the sibling grantAccess.js bootstrap (which loads typenode and
+// then requires this file). Grants a specific access request by its requestId; must be run on the
+// storage machine itself (its certs.ts identity is what the server trusts as admin).
+
+async function main() {
+    let url = getArg("url");
+    if (!url) throw new Error(`--url is required (ex: --url https://storage.example.com:4444)`);
+    let requestId = getArg("requestId");
+    if (!requestId) throw new Error(`--requestId is required (the request id shown on the access page)`);
+
+    let { address, port } = parseStorageUrl(url);
+    let nodeId = SocketFunction.connect({ address, port });
+    await authenticateStorage({ address, port, nodeId });
+    let record = await RemoteStorageController.nodes[nodeId].adminGrantAccess(requestId);
+    console.log(`Granted machine ${record.machineId} access to account ${JSON.stringify(record.account)}`);
+    process.exit(0);
+}
+
+main().catch(e => {
+    console.error(e);
+    process.exit(1);
+});

package/storage/remoteStorage/remoteConfig.d.ts

@@ -0,0 +1,29 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } from "../IArchives";
+export declare const ROUTING_FILE = "storage/storagerouting.json";
+/** The variable-shard route override embedded in the key ("<sentinel>_<value>", see VARIABLE_SHARD), or undefined when the key has no sentinel or the sentinel has no value yet. */
+export declare function parseVariableRoute(key: string): number | undefined;
+/** Where a key routes in [0, 1). A materialized variable-shard suffix completely overrides the hash. */
+export declare function getRoute(key: string): number;
+export declare function routeContains(route: [number, number] | undefined, value: number): boolean;
+export declare function routesOverlap(a: [number, number] | undefined, b: [number, number] | undefined): boolean;
+/** The overlap of two route ranges, or undefined when they don't overlap. */
+export declare function routeIntersection(a: [number, number] | undefined, b: [number, number] | undefined): [number, number] | undefined;
+export declare function getConfigVersion(config: RemoteConfig): number;
+/** Strips the routing-file suffix, leaving the bucket's public base URL (file paths append to it). */
+export declare function getBucketBaseUrl(url: string): string;
+export declare function buildFileUrl(baseUrl: string, filePath: string): string;
+export declare function parseHostedUrl(url: string): {
+    address: string;
+    port: number;
+    account: string;
+    bucketName: string;
+};
+export declare function parseBackblazeUrl(url: string): {
+    bucketName: string;
+};
+export declare function normalizeSource(source: RemoteConfigBase): HostedConfig | BackblazeConfig;
+export declare function normalizeRemoteConfig(config: RemoteConfig | RemoteConfigBase): RemoteConfig;
+export declare function parseRoutingData(data: Buffer): RemoteConfig;
+export declare function serializeRemoteConfig(config: RemoteConfig): Buffer;

package/storage/remoteStorage/remoteConfig.ts

@@ -0,0 +1,194 @@
+module.allowclient = true;
+
+import { sort, sha256HashBuffer } from "socket-function/src/misc";
+import { getBufferInt } from "socket-function/src/bits";
+import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, FULL_VALID_WINDOW, FULL_ROUTE, VARIABLE_SHARD } from "../IArchives";
+
+// Parsing / normalization of RemoteConfig (see IArchives.ts). Every bucket stores its own
+// configuration (a RemoteConfig) inside itself, at ROUTING_FILE. Writing that file creates the
+// bucket / reconfigures it (see storageServerState.ts); clients reconcile it by version (see
+// createArchives.ts).
+
+export const ROUTING_FILE = "storage/storagerouting.json";
+const ROUTING_SUFFIX = "/" + ROUTING_FILE;
+
+const ROUTE_PRECISION = 1000 * 1000 * 1000;
+
+/** The variable-shard route override embedded in the key ("<sentinel>_<value>", see VARIABLE_SHARD), or undefined when the key has no sentinel or the sentinel has no value yet. */
+export function parseVariableRoute(key: string): number | undefined {
+    let index = key.indexOf(VARIABLE_SHARD);
+    if (index === -1) return undefined;
+    let match = /^_(\d+(?:\.\d+)?)/.exec(key.slice(index + VARIABLE_SHARD.length));
+    if (!match) return undefined;
+    return +match[1];
+}
+
+/** Where a key routes in [0, 1). A materialized variable-shard suffix completely overrides the hash. */
+export function getRoute(key: string): number {
+    let override = parseVariableRoute(key);
+    if (override !== undefined) return override;
+    let hash = getBufferInt(sha256HashBuffer(key));
+    return hash % ROUTE_PRECISION / ROUTE_PRECISION;
+}
+
+// Route ranges are [start, end) - inclusive start, exclusive end
+export function routeContains(route: [number, number] | undefined, value: number): boolean {
+    if (!route) return true;
+    return route[0] <= value && value < route[1];
+}
+export function routesOverlap(a: [number, number] | undefined, b: [number, number] | undefined): boolean {
+    let [aStart, aEnd] = a || FULL_ROUTE;
+    let [bStart, bEnd] = b || FULL_ROUTE;
+    return aStart < bEnd && bStart < aEnd;
+}
+/** The overlap of two route ranges, or undefined when they don't overlap. */
+export function routeIntersection(a: [number, number] | undefined, b: [number, number] | undefined): [number, number] | undefined {
+    let [aStart, aEnd] = a || FULL_ROUTE;
+    let [bStart, bEnd] = b || FULL_ROUTE;
+    let start = Math.max(aStart, bStart);
+    let end = Math.min(aEnd, bEnd);
+    if (start >= end) return undefined;
+    return [start, end];
+}
+
+// A missing version counts as -1, so any explicitly versioned config beats an unversioned one
+export function getConfigVersion(config: RemoteConfig): number {
+    return config.version ?? -1;
+}
+
+/** Strips the routing-file suffix, leaving the bucket's public base URL (file paths append to it). */
+export function getBucketBaseUrl(url: string): string {
+    if (!url.endsWith(ROUTING_SUFFIX)) {
+        throw new Error(`Expected a bucket routing URL ending with ${JSON.stringify(ROUTING_SUFFIX)}, was ${JSON.stringify(url)}`);
+    }
+    return url.slice(0, -ROUTING_SUFFIX.length);
+}
+
+export function buildFileUrl(baseUrl: string, filePath: string): string {
+    return baseUrl + "/" + filePath.split("/").map(encodeURIComponent).join("/");
+}
+
+// Ex: https://storage2.vidgridweb.com:4445/file/exampleaccount/examplebucket/storage/storagerouting.json
+export function parseHostedUrl(url: string): { address: string; port: number; account: string; bucketName: string } {
+    let base = getBucketBaseUrl(url);
+    let u = new URL(base);
+    if (u.protocol !== "https:") {
+        throw new Error(`Storage URL must use https, got ${JSON.stringify(u.protocol)} in ${JSON.stringify(url)}`);
+    }
+    let parts = u.pathname.split("/").filter(x => x);
+    if (parts.length !== 3 || parts[0] !== "file") {
+        throw new Error(`Expected a hosted bucket URL like https://host:port/file/<account>/<bucketName>${ROUTING_SUFFIX}, was ${JSON.stringify(url)}`);
+    }
+    return { address: u.hostname, port: +u.port || 443, account: decodeURIComponent(parts[1]), bucketName: decodeURIComponent(parts[2]) };
+}
+
+// Ex: https://f002.backblazeb2.com/file/querysubtest-com-public-immutable/storage/storagerouting.json
+export function parseBackblazeUrl(url: string): { bucketName: string } {
+    let base = getBucketBaseUrl(url);
+    let u = new URL(base);
+    let parts = u.pathname.split("/").filter(x => x);
+    if (parts.length !== 2 || parts[0] !== "file") {
+        throw new Error(`Expected a backblaze bucket URL like https://f002.backblazeb2.com/file/<bucketName>${ROUTING_SUFFIX}, was ${JSON.stringify(url)}`);
+    }
+    return { bucketName: decodeURIComponent(parts[1]) };
+}
+
+export function normalizeSource(source: RemoteConfigBase): HostedConfig | BackblazeConfig {
+    if (typeof source !== "string") {
+        let window = source.validWindow;
+        if (!Array.isArray(window) || window.length !== 2 || !window.every(x => typeof x === "number")) {
+            throw new Error(`Object source configs must specify validWindow ([startMs, endMs] of the write times the source is valid for) - configuration changes must be scheduled, not flipped instantly. Use FULL_VALID_WINDOW when the source is always valid. Was ${JSON.stringify(window)} on ${JSON.stringify(source).slice(0, 500)}`);
+        }
+        let route = source.route;
+        if (route !== undefined && (!Array.isArray(route) || route.length !== 2 || !route.every(x => typeof x === "number") || route[0] < 0 || route[1] > 1 || route[0] >= route[1])) {
+            throw new Error(`Source route must be a [start, end) fraction range within [0, 1] with start < end, was ${JSON.stringify(route)} on ${JSON.stringify(source).slice(0, 500)}`);
+        }
+        if (source.readerDiskLimit !== undefined) {
+            if (typeof source.readerDiskLimit !== "number" || source.readerDiskLimit <= 0) {
+                throw new Error(`readerDiskLimit must be a positive byte count, was ${JSON.stringify(source.readerDiskLimit)} on ${JSON.stringify(source).slice(0, 500)}`);
+            }
+            if (!source.noFullSync) {
+                throw new Error(`readerDiskLimit requires noFullSync: a full copy cannot be bounded, only a read cache can. On ${JSON.stringify(source).slice(0, 500)}`);
+            }
+        }
+        if (source.type === "remote") {
+            // Throws if the URL is malformed, so bad configs are rejected before they're stored
+            parseHostedUrl(source.url);
+        }
+        return source;
+    }
+    let hostname = new URL(source).hostname;
+    if (hostname.endsWith(".backblazeb2.com")) {
+        // Validates the URL (throws on malformed) before it's stored; the bucket name is read back
+        // out of the URL at use sites, never stored on the config.
+        parseBackblazeUrl(source);
+        return { type: "backblaze", url: source, validWindow: FULL_VALID_WINDOW };
+    }
+    parseHostedUrl(source);
+    return { type: "remote", url: source, validWindow: FULL_VALID_WINDOW };
+}
+
+export function normalizeRemoteConfig(config: RemoteConfig | RemoteConfigBase): RemoteConfig {
+    let result: RemoteConfig;
+    if (typeof config !== "string" && "sources" in config) {
+        result = { version: config.version, sources: config.sources.map(normalizeSource) };
+    } else {
+        result = { sources: [normalizeSource(config)] };
+    }
+    // Mixed immutability makes no sense AMONG sources valid at the same time: a mutable source
+    // would accept overwrites that its immutable peers refuse to synchronize, forking their
+    // contents. Sources are grouped by transitively overlapping valid windows (absorb everything
+    // overlapping the group, extend the group's end, repeat) - a soft check that is correct as
+    // long as windows have clean breaks (a group ends exactly where the next begins).
+    let sources = result.sources.map(normalizeSource);
+    let sorted = [...sources];
+    sort(sorted, x => x.validWindow[0]);
+    let group: typeof sources = [];
+    let groupEnd = 0;
+    function checkGroup() {
+        let immutableCount = group.filter(x => x.immutable).length;
+        if (immutableCount && immutableCount !== group.length) {
+            throw new Error(`Sources with overlapping valid windows must agree on immutability: ${immutableCount} of ${group.length} are immutable. Sources: ${JSON.stringify(group.map(x => ({ url: x.url, validWindow: x.validWindow, immutable: !!x.immutable })))}`);
+        }
+        // The sources valid at any instant must cover the whole key space, or keys routing into a
+        // gap could never be read
+        let routes = group.map(x => x.route || FULL_ROUTE);
+        sort(routes, x => x[0]);
+        let covered = 0;
+        for (let route of routes) {
+            if (route[0] > covered) break;
+            covered = Math.max(covered, route[1]);
+        }
+        if (covered < 1) {
+            throw new Error(`Sources with overlapping valid windows must cover the full route space [0, 1); coverage stops at ${covered}. Sources: ${JSON.stringify(group.map(x => ({ url: x.url, validWindow: x.validWindow, route: x.route || FULL_ROUTE })))}`);
+        }
+    }
+    for (let source of sorted) {
+        if (group.length && source.validWindow[0] >= groupEnd) {
+            checkGroup();
+            group = [];
+        }
+        group.push(source);
+        groupEnd = Math.max(groupEnd, source.validWindow[1]);
+    }
+    checkGroup();
+    return result;
+}
+
+export function parseRoutingData(data: Buffer): RemoteConfig {
+    let text = data.toString();
+    let parsed: RemoteConfig;
+    try {
+        parsed = JSON.parse(text) as RemoteConfig;
+    } catch (e) {
+        throw new Error(`Routing config is not valid JSON (${String(e)}). Data: ${text.slice(0, 500)}`);
+    }
+    if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.sources)) {
+        throw new Error(`Routing config must be { version?, sources: [...] }, was ${text.slice(0, 500)}`);
+    }
+    return normalizeRemoteConfig(parsed);
+}
+
+export function serializeRemoteConfig(config: RemoteConfig): Buffer {
+    return Buffer.from(JSON.stringify(config, undefined, 4));
+}

package/storage/remoteStorage/sourceWrapper.d.ts

@@ -0,0 +1,41 @@
+import { IArchives, HostedConfig, BackblazeConfig } from "../IArchives";
+import { ArchivesUrl } from "./ArchivesUrl";
+export declare const RETRY_START_DELAY: number;
+export declare const RETRY_MAX_DELAY: number;
+export declare const RETRY_GROWTH = 1.5;
+export declare class SourceWrapper {
+    config: HostedConfig | BackblazeConfig;
+    private background;
+    api?: IArchives;
+    url?: ArchivesUrl;
+    writeBlocked?: string;
+    private remote?;
+    private disposed;
+    private reconnectRunning;
+    private accessCache?;
+    private constructor();
+    static create(config: HostedConfig | BackblazeConfig, options?: {
+        background?: boolean;
+    }): Promise<SourceWrapper>;
+    getDebugName(): string;
+    isConnected(): boolean;
+    /** Call after a request failed while isConnected() was false: starts (if not already running)
+     *  the background reconnect loop. Never blocks - the failed request still throws. */
+    noteFailure(): void;
+    private reconnectLoop;
+    private checkAccess;
+    read<T>(run: (archives: IArchives) => Promise<T>): Promise<T>;
+    hasWriteAccess(): Promise<boolean>;
+    private pings;
+    private pingTimer;
+    /** Starts measuring this source's latency (for variable-shard target preference). Only hosted
+     *  remotes are pinged; our own local server counts as 0, everything else as Infinity. */
+    startPinging(): void;
+    /** Median of the recent pings. Sources that can't be pinged sort last (Infinity), except our
+     *  own in-process server, which is the best possible target (0). */
+    getLatency(): number;
+    /** Writes always go through the API, so a permission error throws to the caller on every write
+     *  (and access granted in the meantime is picked up automatically). */
+    write<T>(run: (archives: IArchives) => Promise<T>): Promise<T>;
+    dispose(): void;
+}

package/storage/remoteStorage/sourceWrapper.ts

@@ -0,0 +1,223 @@
+module.allowclient = true;
+
+import { isNode, sort } from "socket-function/src/misc";
+import { delay } from "socket-function/src/batching";
+import { getSecret } from "../../misc/getSecret";
+import { IArchives, HostedConfig, BackblazeConfig } from "../IArchives";
+import { ROUTING_FILE, getBucketBaseUrl, parseHostedUrl, parseBackblazeUrl } from "./remoteConfig";
+import { ArchivesRemote } from "./ArchivesRemote";
+import { ArchivesUrl } from "./ArchivesUrl";
+import { ArchivesBackblaze } from "../backblaze";
+import { getStorageServerConfigOptional, getLocalArchives } from "./storageServerState";
+
+export const RETRY_START_DELAY = 2 * 1000;
+export const RETRY_MAX_DELAY = 5 * 60 * 1000;
+export const RETRY_GROWTH = 1.5;
+const ACCESS_RECHECK_INTERVAL = 60 * 1000;
+const PING_INTERVAL = 60 * 1000;
+const PING_HISTORY = 10;
+
+async function hasBackblazeCreds(): Promise<boolean> {
+    if (!isNode()) return false;
+    try {
+        await getSecret("backblaze.json.applicationKeyId");
+        await getSecret("backblaze.json.applicationKey");
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+// One source of a RemoteConfig, as a usable IArchives. Reads fall back to the public-URL form when
+// we don't have API access; writes always attempt the API (so access granted later is picked up on
+// the next write). When a call fails while the WebSocket is down, noteFailure starts a background
+// reconnect loop with a ramping delay - it never blocks callers, it just keeps trying so the
+// connection eventually comes back. Sources without a WebSocket (backblaze, URL-form, our own
+// local server) report always-connected: they're last-resort sources where throwing is fine.
+export class SourceWrapper {
+    public api?: IArchives;
+    public url?: ArchivesUrl;
+    public writeBlocked?: string;
+    private remote?: ArchivesRemote;
+    private disposed = false;
+    private reconnectRunning = false;
+    private accessCache?: { hasAccess: boolean; time: number };
+
+    private constructor(
+        public config: HostedConfig | BackblazeConfig,
+        // false for short-lived probe instances, which must never leave a retry loop behind
+        private background: boolean,
+    ) { }
+
+    public static async create(config: HostedConfig | BackblazeConfig, options?: { background?: boolean }): Promise<SourceWrapper> {
+        let wrapper = new SourceWrapper(config, options?.background !== false);
+        if (config.type === "backblaze") {
+            if (await hasBackblazeCreds()) {
+                wrapper.api = new ArchivesBackblaze({ bucketName: parseBackblazeUrl(config.url).bucketName, public: config.public, immutable: config.immutable });
+            } else if (isNode()) {
+                wrapper.writeBlocked = `No backblaze credentials for ${config.url}. Provide backblaze.json.applicationKeyId and backblaze.json.applicationKey via getSecret to enable API access.`;
+            } else {
+                wrapper.writeBlocked = `Browsers cannot write to backblaze (bucket ${config.url})`;
+            }
+            if (!wrapper.api && config.public !== false) {
+                wrapper.url = new ArchivesUrl(getBucketBaseUrl(config.url));
+            }
+            return wrapper;
+        }
+        let parsed = parseHostedUrl(config.url);
+        let server = isNode() && getStorageServerConfigOptional() || undefined;
+        if (server && parsed.address === server.domain && parsed.port === server.port) {
+            // A bucket hosted by our own process - use it directly instead of calling ourselves
+            wrapper.api = getLocalArchives(parsed.account, parsed.bucketName);
+            return wrapper;
+        }
+        wrapper.remote = new ArchivesRemote({ url: config.url, accountName: config.accountName, waitForAccess: false });
+        wrapper.api = wrapper.remote;
+        if (config.public !== false) {
+            wrapper.url = new ArchivesUrl(getBucketBaseUrl(config.url));
+        }
+        return wrapper;
+    }
+
+    public getDebugName(): string {
+        return `source/${this.config.url}`;
+    }
+
+    public isConnected(): boolean {
+        if (!this.remote) return true;
+        return this.remote.isConnected();
+    }
+
+    /** Call after a request failed while isConnected() was false: starts (if not already running)
+     *  the background reconnect loop. Never blocks - the failed request still throws. */
+    public noteFailure(): void {
+        if (!this.background || this.disposed || this.reconnectRunning) return;
+        if (this.isConnected()) return;
+        this.reconnectRunning = true;
+        void this.reconnectLoop();
+    }
+
+    private async reconnectLoop(): Promise<void> {
+        let retryDelay = RETRY_START_DELAY;
+        while (true) {
+            await delay(retryDelay);
+            if (this.disposed) {
+                this.reconnectRunning = false;
+                return;
+            }
+            if (this.isConnected()) break;
+            try {
+                // Any call re-establishes the WebSocket; the result doesn't matter
+                await this.remote?.getInfo(ROUTING_FILE);
+                break;
+            } catch (e) {
+                // Even a failing call (e.g. access denied) proves the connection is back
+                if (this.isConnected()) break;
+                console.warn(`Cannot connect to storage source ${this.config.url}, retrying in ${Math.round(retryDelay / 1000)}s. ${(e as Error).stack ?? e}`);
+            }
+            retryDelay = Math.min(RETRY_MAX_DELAY, retryDelay * RETRY_GROWTH);
+        }
+        this.reconnectRunning = false;
+        console.log(`Reconnected to storage source ${this.config.url}`);
+    }
+
+    // For hosted sources: cached access check, so reads know whether to use the API or the public
+    // URL form. Access, once seen, is assumed to stick; no-access is re-checked periodically (the
+    // check also registers our access request server-side, which logs the grant link).
+    private async checkAccess(): Promise<boolean> {
+        let remote = this.remote;
+        if (!remote) return !!this.api;
+        let cached = this.accessCache;
+        if (cached && (cached.hasAccess || Date.now() - cached.time < ACCESS_RECHECK_INTERVAL)) {
+            return cached.hasAccess;
+        }
+        let waiting = await remote.waitingForAccess();
+        this.accessCache = { hasAccess: !waiting, time: Date.now() };
+        if (waiting) {
+            console.warn(`No access to storage account for ${this.config.url} (our machine ${waiting.machineId}, ip ${waiting.ip}). Reading via the public URL form where possible. Grant access at ${waiting.link}`);
+        }
+        return !waiting;
+    }
+
+    public async read<T>(run: (archives: IArchives) => Promise<T>): Promise<T> {
+        if (this.api && await this.checkAccess()) {
+            return await run(this.api);
+        }
+        if (this.url) {
+            return await run(this.url);
+        }
+        if (this.api) {
+            // No public URL form - let the API call throw its own error (which includes the access
+            // instructions for hosted sources)
+            return await run(this.api);
+        }
+        throw new Error(`Source ${this.config.url} has no API access and no public URL form (public: false)`);
+    }
+
+    public async hasWriteAccess(): Promise<boolean> {
+        if (this.writeBlocked || !this.api) return false;
+        return await this.api.hasWriteAccess();
+    }
+
+    private pings: number[] = [];
+    private pingTimer: ReturnType<typeof setInterval> | undefined;
+
+    /** Starts measuring this source's latency (for variable-shard target preference). Only hosted
+     *  remotes are pinged; our own local server counts as 0, everything else as Infinity. */
+    public startPinging(): void {
+        let remote = this.remote;
+        if (!remote || this.pingTimer || this.disposed) return;
+        let measure = async () => {
+            let start = Date.now();
+            try {
+                await remote.ping();
+            } catch {
+                return;
+            }
+            this.pings.push(Date.now() - start);
+            if (this.pings.length > PING_HISTORY) {
+                this.pings.shift();
+            }
+        };
+        void measure();
+        this.pingTimer = setInterval(() => {
+            void measure();
+        }, PING_INTERVAL);
+        (this.pingTimer as { unref?: () => void }).unref?.();
+    }
+
+    /** Median of the recent pings. Sources that can't be pinged sort last (Infinity), except our
+     *  own in-process server, which is the best possible target (0). */
+    public getLatency(): number {
+        if (!this.remote) {
+            if (this.config.type === "remote" && this.api) return 0;
+            return Infinity;
+        }
+        if (!this.pings.length) return Infinity;
+        let sorted = [...this.pings];
+        sort(sorted, x => x);
+        return sorted[Math.floor(sorted.length / 2)];
+    }
+
+    /** Writes always go through the API, so a permission error throws to the caller on every write
+     *  (and access granted in the meantime is picked up automatically). */
+    public async write<T>(run: (archives: IArchives) => Promise<T>): Promise<T> {
+        if (this.writeBlocked) {
+            throw new Error(this.writeBlocked);
+        }
+        let api = this.api;
+        if (!api) {
+            throw new Error(`Source ${this.config.url} has no API access, so it cannot accept writes`);
+        }
+        let result = await run(api);
+        this.accessCache = { hasAccess: true, time: Date.now() };
+        return result;
+    }
+
+    public dispose(): void {
+        this.disposed = true;
+        if (this.pingTimer) {
+            clearInterval(this.pingTimer);
+        }
+    }
+}

package/storage/remoteStorage/spec.md

@@ -0,0 +1,31 @@
+# Remote storage — core design ideas
+
+This is the reasoning behind the remote storage system (createArchives / storageServer / BlobStore). The code documents the mechanics; this documents the ideas that must survive refactors.
+
+## Configuration updates happen only on startup
+
+A client adopts a routing config once, during initialization, and then runs on it. Configs never automatically upgrade mid-run: the 5-minute poll picks up whatever the sources currently hold (so a developer's change spreads), but the version-gated *upgrade* — writing a newer configured version over an older stored one — happens only at startup. This is deliberate: config changes are made by a developer running code, and that developer is present at startup. It also prevents reversion loops — if a server briefly held a newer version and then went away, running clients don't cling to that phantom version; they follow what the surviving sources actually hold, and only a fresh startup with an explicitly newer configured version moves things forward again.
+
+## Scanning + tombstones + versions make any two sources mergeable
+
+Every store fully rescans its sources' metadata (on startup and periodically), and every write — including deletions — is ordered by last-write time. Deletions are tombstones: an empty file IS a missing file, kept as a size-0 index entry (for a week) so the deletion itself propagates and reconciles like any other write. Because everything is a timestamped write and scans are bidirectional (pull what they have, push what they're missing), any two sources can be merged in any order and converge to the same state: newest write time wins, per file. There is no operation whose loss corrupts the system — a failed background write, a missed delete, a source that was down for a day — the next scan reconciles it.
+
+## The full source list is duplicated into every source
+
+Each bucket stores its complete routing config (the full redundancy list) inside itself, at storage/storagerouting.json, on every source. Discovery is therefore trivial: as long as ONE node is up, a client reading it gets the full overview of the intended sources. And because clients re-discover on every startup (and re-read every 5 minutes), a developer can change the configuration in one place and clients rapidly accept it — no redeploy, no coordinated restart of the fleet.
+
+## BulkDatabase2 index + our own disk as just another source
+
+Each bucket's store keeps a BulkDatabase2 index of every file (path, write time, size, holding source), served from memory — existence checks and listings are extremely fast, never touching a source. The trick that keeps the index accurate: our own disk is not special-cased, it is simply the first synchronization source. The same scan/reconcile code that synchronizes remote sources also synchronizes the disk with the index, so the index self-heals from the same machinery instead of needing separate consistency logic.
+
+## Metadata first, data second
+
+Synchronization starts with metadata: every source's full listing (path, write time, size) is scanned into the index up front. This is fast and small, so a server very quickly knows exactly what exists everywhere — and can act as a consistent authority on the bucket (serving correct reads, listings, and existence checks by fetching bytes from whichever source holds them) long before it has downloaded the actual data. The data then follows: eagerly by default (the full sync), or — when there is simply too much data to copy — in noFullSync mode, where the disk is only a lazy read cache. Either way consistency is unchanged: the index is complete, writes still go to all the servers, and only where the bytes rest differs.
+
+## Client writes are consistent; client reads are redundant
+
+Clients always write to the same node — the first source whose valid window is current — and if that node is down, the write FAILS rather than going to another node. A client having a network hiccup and wrongly deciding nodes are down must never scatter its writes across the chain; that would desynchronize the sources based on one client's flaky view of the network. Reads, by contrast, fail over freely across every redundant source: sources are synchronized copies, so reading from any of them is safe. Maximum read uptime, strictly consistent writes. (Write redundancy still exists — it just lives server-side: the receiving node fans writes out downstream and reconciliation heals anything missed, all ordered by the once-stamped write time.)
+
+## Trust instead of API keys
+
+Machines authenticate with their certs.ts identity (proving ownership of their machine key with a signed, server-bound token), and access is granted per account to specific machineIds. No API keys are minted, copied into configs, or passed around — granting a machine access is one command on the storage machine, and revoking it is removing the trust record. The only API keys left in the system are the ones third parties force on us: backblaze and cloudflare, both resolved through getSecret.

package/storage/remoteStorage/storageController.d.ts

@@ -1,8 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { ArchiveFileInfo } from "../IArchives";
-import type { BlobStore } from "./blobStore";
-import type { IStorage } from "../IStorage";
+import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
 export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
 export declare const STORAGE_NOT_AUTHENTICATED = "REMOTE_STORAGE_NOT_AUTHENTICATED_cf2f7b1e";
@@ -25,34 +23,15 @@ export type TrustRecord = {
     ip: string;
     time: number;
 };
-export type BucketConfig = {
-    public?: boolean;
-    fast?: boolean;
-    writeDelay?: number;
-};
 export type AccessState = {
     machineId: string;
     ip: string;
     hasAccess: boolean;
-    listAccessCommand: string;
     grantAccessCommand?: string;
-    machines?: (AccessRequest & {
-        trusted: boolean;
-    })[];
-};
-export type StorageServerState = {
-    domain: string;
-    port: number;
-    rootDomain: string;
-    sshTarget: string;
-    serverCommand: string;
-    blobStore: BlobStore;
-    trust: IStorage<TrustRecord>;
-    requests: IStorage<AccessRequest[]>;
-    buckets: IStorage<BucketConfig>;
+    trustedMachines?: TrustRecord[];
 };
-export declare function setStorageServerState(state: StorageServerState): void;
 export declare const RemoteStorageController: import("socket-function/SocketFunctionTypes").SocketRegistered<{
+    ping: () => Promise<void>;
     authenticate: (token: AuthToken) => Promise<{
         machineId: string;
         ip: string;
@@ -64,14 +43,23 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         grantAccessCommand: string;
     }>;
     getAccessState: (account: string) => Promise<AccessState>;
+    listRequestsForIP: (account: string, ip: string) => Promise<AccessRequest[]>;
+    grantAccess: (requestId: string) => Promise<TrustRecord>;
     adminListRequests: (ip: string) => Promise<AccessRequest[]>;
     adminGrantAccess: (requestId: string) => Promise<TrustRecord>;
-    ensureBucket: (account: string, bucketName: string, config: BucketConfig) => Promise<void>;
     get: (account: string, bucketName: string, path: string, range?: {
         start: number;
         end: number;
     }) => Promise<Buffer | undefined>;
-    set: (account: string, bucketName: string, path: string, data: Buffer) => Promise<void>;
+    get2: (account: string, bucketName: string, path: string, range?: {
+        start: number;
+        end: number;
+    }) => Promise<{
+        data: Buffer;
+        writeTime: number;
+        size: number;
+    } | undefined>;
+    set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number) => Promise<void>;
     del: (account: string, bucketName: string, path: string) => Promise<void>;
     getInfo: (account: string, bucketName: string, path: string) => Promise<{
         writeTime: number;
@@ -81,9 +69,24 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         shallow?: boolean;
         type?: "files" | "folders";
     }) => Promise<ArchiveFileInfo[]>;
+    getChangesAfter: (account: string, bucketName: string, time: number) => Promise<ArchiveFileInfo[]>;
+    getArchivesConfig: (account: string, bucketName: string) => Promise<ArchivesConfig>;
+    getIndexInfo: (account: string, bucketName: string) => Promise<{
+        fileCount: number;
+        byteCount: number;
+        sources: {
+            debugName: string;
+            fileCount: number;
+            byteCount: number;
+        }[];
+    } | undefined>;
+    getSyncStatus: (account: string, bucketName: string) => Promise<ArchivesSyncStatus>;
     startLargeFile: (account: string, bucketName: string, path: string) => Promise<string>;
     uploadPart: (uploadId: string, data: Buffer) => Promise<void>;
     finishLargeFile: (uploadId: string) => Promise<void>;
     cancelLargeFile: (uploadId: string) => Promise<void>;
-    getPublicFile: (account: string, bucketName: string, path: string) => Promise<Buffer>;
+    httpEntry: (config?: {
+        requireCalls?: string[];
+        cacheTime?: number;
+    }) => Promise<Buffer>;
 }>;