sliftutils 1.7.95 → 1.7.96

package/storage/remoteStorage/createArchives.ts

@@ -1,4 +1,4 @@
-import { isNode, sort, timeoutToUndefinedSilent, watchSlowPromise } from "socket-function/src/misc";
+import { sort, watchSlowPromise } from "socket-function/src/misc";
 import { delay } from "socket-function/src/batching";
 import {
     IArchives, RemoteConfig, RemoteConfigBase, SourceConfig,
@@ -7,27 +7,22 @@ import {
 } from "../IArchives";
 import { copyArchiveFile } from "../archiveHelpers";
 import {
-    ROUTING_FILE, getConfigVersion, parseHostedUrl, parseBackblazeUrl, parseRoutingData, assertValidRemoteConfig,
-    normalizeRemoteConfig, normalizeSource, serializeRemoteConfig,
+    ROUTING_FILE, parseHostedUrl, parseBackblazeUrl, parseRoutingData, assertValidRemoteConfig,
+    normalizeRemoteConfig, normalizeSource,
     getRoute, routeContains, parseVariableRoute, getBucketBaseUrl, buildFileUrl,
 } from "./remoteConfig";
 import { ArchivesUrl } from "./ArchivesUrl";
-import { resolveIntermediateSources } from "./intermediateSources";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { ArchivesRemote, parseStorageUrl, authenticateStorage } from "./ArchivesRemote";
-import { onServerRoutingChanged } from "./storageClientController";
-import { ArchivesBackblaze } from "../backblaze";
-import { ArchivesDisk } from "../ArchivesDisk";
 import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
 import { RemoteStorageController, STORAGE_NOT_AUTHENTICATED } from "./storageController";
-import { SourceWrapper, RETRY_START_DELAY, RETRY_MAX_DELAY, RETRY_GROWTH } from "./sourceWrapper";
+import { SourceWrapper } from "./sourceWrapper";
+import { ChainState, ChainStateManager } from "./chainStartup";
 import { formatTime } from "socket-function/src/formatting/format";
 
-const CONFIG_POLL_INTERVAL = 5 * 60 * 1000;
 const WRONG_TARGET_BOUNDARY_WINDOW = 30 * 1000;
 const WRONG_TARGET_BOUNDARY_RETRY_DELAY = 15 * 1000;
 const CONFIG_REFRESH_THROTTLE = 30 * 1000;
-const AVAILABILITY_RECHECK_THROTTLE = 5 * 1000;
 const PRIMARY_RETRY_TIMEOUT = 30 * 1000;
 const PRIMARY_RETRY_DELAY = 2 * 1000;
 const COVERING_RETRY_TIMEOUT = 30 * 1000;
@@ -54,11 +49,6 @@ export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteCon
 /** A client for ONE source - see storeSources.ts. Re-exported here because a chain is built out of them. */
 export { createApiArchives } from "./storeSources";
 
-type ChainState = {
-    config: RemoteConfig;
-    sources: SourceWrapper[];
-};
-
 export type ArchivesChainOptions = {
     /** Outside of node we default to read-only downloads over the public URLs (no API connection) when the config has public sources. Set this to connect to the API anyway - needed for writing, listing, and any other operation the plain URL form cannot serve. */
     directConnect?: boolean;
@@ -96,306 +86,22 @@ function preferUsable(sources: SourceWrapper[]): SourceWrapper[] {
 }
 
 export class ArchivesChain implements IArchives {
-    private configured: RemoteConfig;
-    private activeConfig: RemoteConfig;
-    private statePromise: Promise<ChainState> | undefined;
-    // The resolved state, for synchronous access (getGetURLs) - always the newest adopted config, where statePromise may briefly lag during a rebuild
-    private latestState: ChainState | undefined;
-    private initRetryDelay = RETRY_START_DELAY;
-    private initRetryTimer: ReturnType<typeof setTimeout> | undefined;
-    private pollTimer: ReturnType<typeof setInterval> | undefined;
-    private disposed = false;
-    private unsubscribeRoutingPush: (() => void) | undefined;
-
-    constructor(config: RemoteConfig | RemoteConfigBase, private options?: ArchivesChainOptions) {
-        this.configured = normalizeRemoteConfig(config);
-        this.activeConfig = this.configured;
-        this.unsubscribeRoutingPush = onServerRoutingChanged(() => {
-            void this.refreshActiveConfig().catch((e: Error) => console.error(`Config refresh failed for ${this.getDebugName()}: ${e.stack ?? e}`));
+    // The chain's state, and everything about having one (init + its retry, config polling/adoption, availability rechecks, the config rewrite loop), lives in the manager - what stays in this class is dispatch over that state
+    private state: ChainStateManager;
+
+    constructor(config: RemoteConfig | RemoteConfigBase, options?: ArchivesChainOptions) {
+        this.state = new ChainStateManager({
+            configured: normalizeRemoteConfig(config),
+            debugName: () => this.getDebugName(),
+            directConnect: options?.directConnect,
         });
     }
 
     public getDebugName() {
-        let urls = this.activeConfig.sources.map(x => typeof x === "string" && x || (x as SourceConfig).url);
+        let urls = this.state.activeConfig.sources.map(x => typeof x === "string" && x || (x as SourceConfig).url);
         return `chain ${urls.join(", ")}`;
     }
 
-    private getState(): Promise<ChainState> {
-        if (this.disposed) {
-            return Promise.reject(new Error(`ArchivesChain ${this.getDebugName()} has been disposed`));
-        }
-        if (!this.statePromise) {
-            let promise = this.init();
-            this.statePromise = promise;
-            promise.then(() => {
-                this.initRetryDelay = RETRY_START_DELAY;
-            }, (e: Error) => {
-                if (this.disposed || this.initRetryTimer) return;
-                console.error(`Storage init failed for ${this.getDebugName()}, retrying in ${Math.round(this.initRetryDelay / 1000)}s. ${e.stack ?? e}`);
-                this.initRetryTimer = setTimeout(() => {
-                    this.initRetryTimer = undefined;
-                    if (this.disposed) return;
-                    if (this.statePromise === promise) {
-                        this.statePromise = undefined;
-                    }
-                    this.getState().catch(() => { });
-                }, this.initRetryDelay);
-                (this.initRetryTimer as { unref?: () => void }).unref?.();
-                this.initRetryDelay = Math.min(RETRY_MAX_DELAY, this.initRetryDelay * RETRY_GROWTH);
-            });
-        }
-        return this.statePromise;
-    }
-
-    private async init(): Promise<ChainState> {
-        let configs = this.configured.sources.map(normalizeSource);
-        let readOnly = this.isReadOnly(this.configured);
-        let fetches = await Promise.all(configs.map(async sourceConfig => {
-            let probe = await SourceWrapper.create(sourceConfig, { background: false, readOnly });
-            let start = Date.now();
-            try {
-                let existing = await probe.readRoutingConfig();
-                return { probe, sourceConfig, responded: true, latency: Date.now() - start, existing, error: "" };
-            } catch (e) {
-                return { probe, sourceConfig, responded: false, latency: 0, existing: undefined, error: `${sourceConfig.url}: ${(e as Error).stack ?? e}` };
-            }
-        }));
-        try {
-            let found = fetches.find(x => x.responded);
-            if (!found) {
-                throw new Error(`Cannot initialize storage for ${this.getDebugName()}: no source answered. ${fetches.map(x => x.error).join(" | ")}`);
-            }
-            let existing = found.existing;
-            let active = this.configured;
-            let needsWrite = true;
-            if (existing && getConfigVersion(existing) >= getConfigVersion(this.configured)) {
-                if (getConfigVersion(existing) === getConfigVersion(this.configured) && JSON.stringify(existing) !== JSON.stringify(this.configured)) {
-                    console.error(`Archives configuration updated without updating the version, for ${found.sourceConfig.url}. Updates will be ignored until you increase the version. Using: ${JSON.stringify(existing)}, ignoring: ${JSON.stringify(this.configured)}`);
-                }
-                active = existing;
-                needsWrite = false;
-            }
-            if (needsWrite) {
-                let best: RemoteConfig | undefined;
-                let conflictUrl: string | undefined;
-                for (let fetch of fetches) {
-                    let stored = fetch.existing;
-                    if (!stored) continue;
-                    if (!best || getConfigVersion(stored) > getConfigVersion(best)) {
-                        best = stored;
-                    }
-                    if (getConfigVersion(stored) === getConfigVersion(this.configured) && JSON.stringify(stored) !== JSON.stringify(this.configured)) {
-                        conflictUrl = fetch.sourceConfig.url;
-                    }
-                }
-                if (best && getConfigVersion(best) >= getConfigVersion(this.configured)) {
-                    if (conflictUrl && getConfigVersion(best) === getConfigVersion(this.configured)) {
-                        console.error(`Archives configuration updated without updating the version, for ${conflictUrl}. Updates will be ignored until you increase the version. Using: ${JSON.stringify(best)}, ignoring: ${JSON.stringify(this.configured)}`);
-                    }
-                    active = best;
-                    needsWrite = false;
-                }
-            }
-            let sources = await this.buildSources(active);
-            if (needsWrite) {
-                let routingData = serializeRemoteConfig(this.configured);
-                let routingWriteTime = Date.now();
-                let targets: SourceWrapper[] = [];
-                let seen = new Set<string>();
-                for (let source of sources) {
-                    if (seen.has(source.config.url)) continue;
-                    seen.add(source.config.url);
-                    targets.push(source);
-                }
-                console.log(`Storage routing config for ${this.getDebugName()} is out of date (stored version ${existing && getConfigVersion(existing) || "none"}, ours ${getConfigVersion(this.configured)}): writing ours to all ${targets.length} nodes (write time ${new Date(routingWriteTime).toISOString()}): ${targets.map(x => x.config.url).join(", ")}`);
-                // All in parallel, every failure tolerated (no-write-access included - the attempt classifies it, nothing pre-checks it). A down node must not stop the others from getting the config - they would then reject writes as unconfigured precisely BECAUSE it never arrived - and it must never stop the chain from starting. A node that missed it pulls it off its peers, and the next startup writes it again.
-                let failures: string[] = [];
-                await Promise.all(targets.map(async source => {
-                    try {
-                        await source.write(archives => archives.set(ROUTING_FILE, routingData, { lastModified: routingWriteTime }));
-                        console.log(`Wrote storage routing config version ${getConfigVersion(this.configured)} to ${source.config.url}`);
-                    } catch (e) {
-                        failures.push(`${source.config.url}: ${(e as Error).stack ?? e}`);
-                    }
-                }));
-                if (failures.length === targets.length) {
-                    // Nobody has our config, so running on it would just disagree with every server. The stored one (when there is one) is what the servers actually run.
-                    console.error(`The storage routing config for ${this.getDebugName()} (version ${getConfigVersion(this.configured)}) could not be written to ANY of the ${targets.length} nodes. ${existing && `Running on the stored config (version ${getConfigVersion(existing)}) until a write succeeds.` || `No stored config exists either, so servers will reject writes until this succeeds.`} Failures: ${failures.join(" | ")}`);
-                    if (existing) {
-                        active = existing;
-                        for (let source of sources) {
-                            source.dispose();
-                        }
-                        sources = await this.buildSources(active);
-                    }
-                } else if (failures.length) {
-                    console.error(`The storage routing config for ${this.getDebugName()} (version ${getConfigVersion(this.configured)}) was written to ${targets.length - failures.length} of ${targets.length} nodes; the rest pull it from the ones that have it. Failures: ${failures.join(" | ")}`);
-                }
-            }
-            for (let source of sources) {
-                let fetch = fetches.find(x => x.responded && x.sourceConfig.url === source.config.url);
-                if (fetch) {
-                    source.seedLatency(fetch.latency);
-                }
-            }
-            this.activeConfig = active;
-            this.startConfigPoll();
-            let state: ChainState = { config: active, sources };
-            this.latestState = state;
-            return state;
-        } finally {
-            for (let fetch of fetches) {
-                fetch.probe.dispose();
-            }
-        }
-    }
-
-    /** Clientside, a config with public sources is served entirely over plain URL downloads - no API connection, no access grant, and no writing. directConnect opts out of that. */
-    private isReadOnly(config: RemoteConfig): boolean {
-        if (this.options?.directConnect || isNode()) return false;
-        return config.sources.map(normalizeSource).some(x => x.public);
-    }
-
-    private async createChainSource(sourceConfig: SourceConfig, readOnly: boolean): Promise<SourceWrapper> {
-        let source = await SourceWrapper.create(sourceConfig, { readOnly });
-        source.startPinging();
-        return source;
-    }
-
-    private async buildSources(config: RemoteConfig): Promise<SourceWrapper[]> {
-        let readOnly = this.isReadOnly(config);
-        let sources: SourceWrapper[] = [];
-        for (let sourceConfig of config.sources.map(normalizeSource)) {
-            sources.push(await this.createChainSource(sourceConfig, readOnly));
-        }
-        return sources;
-    }
-
-    private startConfigPoll(): void {
-        if (this.pollTimer || this.disposed) return;
-        this.pollTimer = setInterval(() => {
-            void this.refreshActiveConfig().catch((e: Error) => {
-                console.error(`Checking for a new storage routing config failed for ${this.getDebugName()}: ${e.stack ?? e}`);
-            });
-        }, CONFIG_POLL_INTERVAL);
-        (this.pollTimer as { unref?: () => void }).unref?.();
-    }
-
-    // Deduplicates concurrent refreshes (the poll timer and wrong-target write retries share this)
-    private configRefreshInFlight: Promise<void> | undefined;
-    private refreshActiveConfig(): Promise<void> {
-        if (!this.configRefreshInFlight) {
-            this.configRefreshInFlight = this.checkForNewConfig().finally(() => {
-                this.configRefreshInFlight = undefined;
-            });
-        }
-        return this.configRefreshInFlight;
-    }
-
-    // The latest config, as the server INTERPRETS it: getConfig carries in-memory overlays (deploy
-    private async fetchLatestConfig(state: ChainState): Promise<RemoteConfig | undefined> {
-        let errors: string[] = [];
-        for (let source of state.sources) {
-            try {
-                let latest = await source.readRoutingConfig();
-                if (latest) return normalizeRemoteConfig(latest);
-            } catch (e) {
-                errors.push(`${source.config.url}: ${(e as Error).stack ?? e}`);
-            }
-        }
-        if (errors.length === state.sources.length) {
-            throw new Error(`No storage source could give us the routing config for ${this.getDebugName()}: ${errors.join(" | ")}`);
-        }
-        return undefined;
-    }
-
-    private async checkForNewConfig(): Promise<void> {
-        if (this.disposed || !this.statePromise) return;
-        let state: ChainState;
-        try {
-            state = await this.statePromise;
-        } catch {
-            return;
-        }
-        let latest = await this.fetchLatestConfig(state);
-        if (!latest) return;
-        await this.adoptNewConfig(state, latest);
-    }
-
-    private async adoptNewConfig(state: ChainState, latest: RemoteConfig): Promise<void> {
-        if (JSON.stringify(latest) === JSON.stringify(state.config)) return;
-        let received = new Date().toISOString();
-        let onlyIntermediatesChanged = JSON.stringify(resolveIntermediateSources(latest)) === JSON.stringify(resolveIntermediateSources(state.config));
-        if (onlyIntermediatesChanged) {
-            console.log(`Storage routing switchover windows changed (version ${getConfigVersion(state.config)} -> ${getConfigVersion(latest)}), received ${received}, rebuilding sources. New config: ${JSON.stringify(latest)}`);
-        } else {
-            console.log(`Storage routing config changed (version ${getConfigVersion(state.config)} -> ${getConfigVersion(latest)}), received ${received}, rebuilding sources. New config: ${JSON.stringify(latest)}`);
-        }
-        let strippedKey = (config: SourceConfig) => JSON.stringify({ ...config, validWindow: undefined });
-        let oldByConfig = new Map<string, SourceWrapper[]>();
-        for (let source of state.sources) {
-            let key = strippedKey(source.config);
-            let list = oldByConfig.get(key);
-            if (!list) {
-                list = [];
-                oldByConfig.set(key, list);
-            }
-            list.push(source);
-        }
-        let readOnly = this.isReadOnly(latest);
-        let sources: SourceWrapper[] = [];
-        for (let sourceConfig of latest.sources.map(normalizeSource)) {
-            let old = oldByConfig.get(strippedKey(sourceConfig))?.shift();
-            if (old) {
-                old.updateValidWindow(sourceConfig.validWindow);
-                sources.push(old);
-            } else {
-                sources.push(await this.createChainSource(sourceConfig, readOnly));
-            }
-        }
-        for (let leftovers of oldByConfig.values()) {
-            for (let leftover of leftovers) {
-                leftover.dispose();
-            }
-        }
-        this.activeConfig = latest;
-        let newState: ChainState = { config: latest, sources };
-        this.latestState = newState;
-        this.statePromise = Promise.resolve(newState);
-    }
-
-    private lastAvailabilityRecheck = 0;
-    private availabilityRecheckInFlight: Promise<void> | undefined;
-    private recheckAvailability(): Promise<void> {
-        if (this.availabilityRecheckInFlight) return this.availabilityRecheckInFlight;
-        if (Date.now() - this.lastAvailabilityRecheck < AVAILABILITY_RECHECK_THROTTLE) return Promise.resolve();
-        this.lastAvailabilityRecheck = Date.now();
-        this.availabilityRecheckInFlight = this.recheckAvailabilityNow().finally(() => {
-            this.availabilityRecheckInFlight = undefined;
-        });
-        return this.availabilityRecheckInFlight;
-    }
-    private async recheckAvailabilityNow(): Promise<void> {
-        if (this.disposed || !this.statePromise) return;
-        let state: ChainState;
-        try {
-            state = await this.statePromise;
-        } catch {
-            return;
-        }
-        console.log(`Every storage source failed for ${this.getDebugName()}; re-contacting all ${state.sources.length} sources (routing re-read + connection re-attempt)`);
-        let results = await Promise.all(state.sources.map(async source => {
-            try {
-                return await source.readRoutingConfig();
-            } catch {
-                return undefined;
-            }
-        }));
-        let latest = results.find(x => x);
-        if (!latest) return;
-        await this.adoptNewConfig(state, latest);
-    }
-
     // The ONE dispatch for every operation: no fallbacks (all writes by default, and noFallbacks reads) -> the primary node only, via runPrimary; everything else -> the shared fallback loop, trying sources in config order (or latency order for fast reads) and only moving on when one fails. Writes in the loop differ from reads only in calling source.write.
     private async run<T>(state: ChainState, config: { apiOnly?: boolean; write?: boolean; route?: number; noFallbacks?: boolean; fallbacks?: boolean; fast?: boolean; timeout?: SmartTimeout }, run: (archives: IArchives, sourceUrl: string) => Promise<T>): Promise<T> {
         if (config.fast && config.noFallbacks) {
@@ -461,13 +167,13 @@ export class ArchivesChain implements IArchives {
                 }
             }
             if (wrongTarget) {
-                state = await this.getState();
+                state = await this.state.getState();
                 continue;
             }
             if (!recheckedAvailability) {
                 recheckedAvailability = true;
-                await this.recheckAvailability();
-                state = await this.getState();
+                await this.state.recheckAvailability();
+                state = await this.state.getState();
                 continue;
             }
             throw new Error(`All sources failed for ${this.getDebugName()}${config.route !== undefined && ` (route ${config.route})` || ""}: ${errors.join(" | ") || "no sources available"}`);
@@ -484,7 +190,7 @@ export class ArchivesChain implements IArchives {
         while (true) {
             attempt++;
             let attemptStart = Date.now();
-            let state = await this.getState();
+            let state = await this.state.getState();
             let target = state.sources.find(x => configWindowCurrent(x.config) && (config.route === undefined || routeContains(x.config.route, config.route)));
             try {
                 if (!target) {
@@ -525,7 +231,7 @@ export class ArchivesChain implements IArchives {
             }
             // At most one attempt per interval, in case the failure is fast
             await delay(Math.max(0, attemptStart + PRIMARY_RETRY_DELAY - Date.now()));
-            await this.recheckAvailability();
+            await this.state.recheckAvailability();
         }
     }
 
@@ -587,16 +293,16 @@ export class ArchivesChain implements IArchives {
         this.lastConfigRefresh = Date.now();
         let reason = kind === "window" && "wrong valid window" || kind === "route" && "wrong route" || "the store has no configuration entry on that server";
         console.log(`Write rejected by ${this.getDebugName()} (${reason}): our config disagrees with the server's (likely stale, or a deploy switchover); refreshing it and retrying`);
-        await this.refreshActiveConfig();
+        await this.state.refreshActiveConfig();
     }
 
     private async request<T>(config: { apiOnly?: boolean; write?: boolean; route?: number; noFallbacks?: boolean; fallbacks?: boolean; fast?: boolean; timeout?: SmartTimeout }, run: (archives: IArchives, sourceUrl: string) => Promise<T>): Promise<T> {
-        let state = await this.getState();
+        let state = await this.state.getState();
         return await this.run(state, config, run);
     }
 
     public async waitingForAccess(): Promise<{ link: string; machineId: string; ip: string } | undefined> {
-        let state = await this.getState();
+        let state = await this.state.getState();
         for (let source of state.sources) {
             // A source whose window has passed is never read or written again, so its access state is irrelevant - and asking a dead intermediate would just hang or throw. Future windows DO matter: access should be granted before their window starts.
             if (source.config.validWindow[1] <= Date.now()) continue;
@@ -662,7 +368,7 @@ export class ArchivesChain implements IArchives {
         let tries = 0;
         while (true) {
             tries++;
-            let state = await this.getState();
+            let state = await this.state.getState();
             // Errors name the OPERATION and the SPECIFIC sources that failed - "the find failed because source X is unavailable", never an anonymous failure attributed to the whole chain
             let outcome = await (async (): Promise<{ values: T[] } | { error: Error }> => {
                 let covering: SourceWrapper[];
@@ -715,7 +421,7 @@ export class ArchivesChain implements IArchives {
                 failed.clear();
             }
             console.error(`${error.message}. Retrying in ${COVERING_RETRY_DELAY / 1000}s (giving up at ${new Date(deadline).toISOString()}).`);
-            void this.recheckAvailability();
+            void this.state.recheckAvailability();
             await delay(COVERING_RETRY_DELAY);
         }
     }
@@ -767,13 +473,13 @@ export class ArchivesChain implements IArchives {
         };
     }
     public async getConfig(): Promise<ArchivesConfig> {
-        let state = await this.getState();
+        let state = await this.state.getState();
         if (!state.sources.some(x => x.api)) return { remoteConfig: state.config };
         let config = await this.run(state, { apiOnly: true }, archives => archives.getConfig());
         return { ...config, remoteConfig: state.config };
     }
     public async hasWriteAccess(): Promise<boolean> {
-        let state = await this.getState();
+        let state = await this.state.getState();
         for (let source of state.sources) {
             if (!configWindowCurrent(source.config)) continue;
             if (!await source.hasWriteAccess()) return false;
@@ -803,26 +509,33 @@ export class ArchivesChain implements IArchives {
     private async setRoutingConfig(data: Buffer, config?: { lastModified?: number }): Promise<string> {
         // Checked before the first node is written to, not just by each node as it arrives: every server would reject it anyway, and finding that out one node at a time is how a config write ends up half-applied
         assertValidRemoteConfig(parseRoutingData(data));
-        let state = await this.getState();
+        let state = await this.state.getState();
         let writeTime = config?.lastModified || Date.now();
         let written: string[] = [];
         let errors: string[] = [];
+        // One write per STORE, not per server: the write lands in the store the entry NAMES, so two entries sharing a URL but naming different stores are two separate deliveries - deduping by URL alone leaves the second store unconfigured forever
+        let targets: SourceWrapper[] = [];
         let seen = new Set<string>();
-        console.log(`Writing storage routing config for ${this.getDebugName()} to every node (write time ${new Date(writeTime).toISOString()}): ${data.toString("utf8").slice(0, 2000)}`);
         for (let source of state.sources) {
-            if (seen.has(source.config.url)) continue;
-            seen.add(source.config.url);
+            let key = `${source.config.url}|${source.config.name}`;
+            if (seen.has(key)) continue;
+            seen.add(key);
+            targets.push(source);
+        }
+        console.log(`Writing storage routing config for ${this.getDebugName()} to all ${targets.length} stores (write time ${new Date(writeTime).toISOString()}): ${data.toString("utf8").slice(0, 2000)}`);
+        await Promise.all(targets.map(async source => {
+            let label = `${source.config.url} (store ${JSON.stringify(source.config.name)})`;
             try {
                 await source.write(archives => archives.set(ROUTING_FILE, data, { lastModified: writeTime }));
-                written.push(source.config.url);
-                console.log(`Wrote the storage routing config to ${source.config.url}`);
+                written.push(label);
+                console.log(`Wrote the storage routing config to ${label}`);
             } catch (e) {
-                errors.push(`${source.config.url}: ${(e as Error).stack ?? e}`);
+                errors.push(`${label}: ${(e as Error).stack ?? e}`);
             }
-        }
-        await this.refreshActiveConfig();
+        }));
+        await this.state.refreshActiveConfig();
         if (errors.length) {
-            throw new Error(`Storage routing config write for ${this.getDebugName()} failed on ${errors.length} of ${seen.size} nodes (succeeded on: ${written.join(", ") || "none"}): ${errors.join(" | ")}`);
+            throw new Error(`Storage routing config write for ${this.getDebugName()} failed on ${errors.length} of ${targets.length} stores (succeeded on: ${written.join(", ") || "none"}): ${errors.join(" | ")}`);
         }
         return ROUTING_FILE;
     }
@@ -838,7 +551,7 @@ export class ArchivesChain implements IArchives {
         }
         let fromRoute = getRoute(config.fromPath);
         let toRoute = getRoute(config.toPath);
-        let state = await this.getState();
+        let state = await this.state.getState();
         let target = state.sources.find(x => configWindowCurrent(x.config) && routeContains(x.config.route, fromRoute));
         if (target && routeContains(target.config.route, toRoute)) {
             await this.request({ write: true, route: fromRoute }, async archives => {
@@ -884,7 +597,7 @@ export class ArchivesChain implements IArchives {
         if (!key.includes(VARIABLE_SHARD) || parseVariableRoute(key) !== undefined) {
             throw new Error(`getShardKey requires a key containing an unmaterialized ${JSON.stringify(VARIABLE_SHARD)}, got ${JSON.stringify(key)}`);
         }
-        let state = await this.getState();
+        let state = await this.state.getState();
         let target = this.getVariableShardTargets(state, { connectedOnly: true })[0]
             || this.getVariableShardTargets(state, { connectedOnly: false })[0];
         if (!target) {
@@ -898,7 +611,7 @@ export class ArchivesChain implements IArchives {
         // There's no point talking to a shard whose node was fast but is now disconnected - only when no connected shard works do we recheck availability and try every shard
         let connectedOnly = true;
         while (true) {
-            let state = await this.getState();
+            let state = await this.state.getState();
             let targets = this.getVariableShardTargets(state, { connectedOnly });
             let errors: string[] = [];
             for (let target of targets) {
@@ -928,7 +641,7 @@ export class ArchivesChain implements IArchives {
             if (!recheckedAvailability) {
                 recheckedAvailability = true;
                 connectedOnly = false;
-                await this.recheckAvailability();
+                await this.state.recheckAvailability();
                 continue;
             }
             throw new Error(`Every variable-shard write target failed for ${this.getDebugName()}: ${errors.join(" | ") || "no sources accept writes"}`);
@@ -959,12 +672,12 @@ export class ArchivesChain implements IArchives {
     private async setLargeFileOnce(config: SetLargeFileConfig, route: number): Promise<void> {
         let recheckedAvailability = false;
         while (true) {
-            let state = await this.getState();
+            let state = await this.state.getState();
             let target = state.sources.find(x => configWindowCurrent(x.config) && routeContains(x.config.route, route));
             if (!target) {
                 if (!recheckedAvailability) {
                     recheckedAvailability = true;
-                    await this.recheckAvailability();
+                    await this.state.recheckAvailability();
                     continue;
                 }
                 throw new Error(`No source accepts writes for setLargeFile on ${this.getDebugName()} (route ${route})`);
@@ -996,19 +709,19 @@ export class ArchivesChain implements IArchives {
 
     /** getURLs, but after the one await (initialization) the returned function is synchronous: everything underneath - route hashing, window checks, latencies, URL building - is synchronous, and the closure always reads the newest adopted config, so it stays correct across config refreshes. */
     public async getGetURLs(): Promise<(path: string) => string[]> {
-        let initialState = await this.getState();
+        let initialState = await this.state.getState();
         return this.makeGetURLs(initialState, { writeNodeFirst: true });
     }
 
     /** getGetURLs, but sorted purely by latency - the write node gets no special first position. For read-only consumers that just want the fastest host. */
     public async getGetFastURLs(): Promise<(path: string) => string[]> {
-        let initialState = await this.getState();
+        let initialState = await this.state.getState();
         return this.makeGetURLs(initialState, { writeNodeFirst: false });
     }
 
     private makeGetURLs(initialState: ChainState, config: { writeNodeFirst: boolean }): (path: string) => string[] {
         return (path: string) => {
-            let state = this.latestState || initialState;
+            let state = this.state.latest() || initialState;
             let route = getRoute(path);
             let sources: SourceWrapper[] = [];
             for (let source of state.sources) {
@@ -1032,22 +745,7 @@ export class ArchivesChain implements IArchives {
     }
 
     public dispose(): void {
-        this.disposed = true;
-        this.unsubscribeRoutingPush?.();
-        if (this.pollTimer) {
-            clearInterval(this.pollTimer);
-        }
-        if (this.initRetryTimer) {
-            clearTimeout(this.initRetryTimer);
-        }
-        let statePromise = this.statePromise;
-        if (statePromise) {
-            void statePromise.then(state => {
-                for (let source of state.sources) {
-                    source.dispose();
-                }
-            }, () => { });
-        }
+        this.state.dispose();
     }
 }
 

package/storage/remoteStorage/storageServerState.ts

@@ -34,7 +34,7 @@ export function getStore(account: string, bucketName: string, name: string): Blo
     let folder = getBucketFolder(name, account, bucketName);
     let existing = stores.get(folder);
     if (existing) return existing;
-    console.log(`Opening store ${JSON.stringify(name)} of bucket ${account}/${bucketName} (folder ${folder}) - created on this first use if it never existed`);
+    console.log(`Opening store ${JSON.stringify(name)} of bucket ${account}/${bucketName} (folder ${folder})`);
     let store = new BlobStore(folder, name, {
         // Which entries in ITS routing config are this server's copy of it
         isSelf: source => isSelfSource(source, account, bucketName),