sliftutils 1.7.15 → 1.7.17

package/storage/remoteStorage/createArchives.ts

@@ -36,6 +36,9 @@ const WRONG_TARGET_BOUNDARY_RETRY_DELAY = 15 * 1000;
 // Otherwise our config is stale; re-fetch it from the server, at most this often (when throttled,
 // retry with what we have)
 const CONFIG_REFRESH_THROTTLE = 30 * 1000;
+// When every source looks down, all of them are re-contacted (routing re-read + connection
+// re-attempt) before giving up - at most this often
+const AVAILABILITY_RECHECK_THROTTLE = 5 * 1000;
 
 // The direct API IArchives for one source, with no URL-form fallback and no chaining. Used by the
 // storage server for its synchronization sources. Denied calls throw immediately (registering the
@@ -124,115 +127,109 @@ export class ArchivesChain implements IArchives {
 
     private async init(): Promise<ChainState> {
         let configs = this.configured.sources.map(normalizeSource);
-        let probeErrors: string[] = [];
-        let found: { probe: SourceWrapper; existing: RemoteConfig | undefined } | undefined;
-        for (let sourceConfig of configs) {
+        // EVERY source is contacted for its picture of the routing config (per the spec: the config
+        // is duplicated into all of them) - which also establishes our connection to every node up
+        // front. Contact is parallel; ADOPTION is deterministic by config order among the sources
+        // that answered (the first up source is authoritative).
+        let fetches = await Promise.all(configs.map(async sourceConfig => {
             let probe = await SourceWrapper.create(sourceConfig, { background: false });
             try {
                 let data = await probe.read(archives => archives.get(ROUTING_FILE));
-                found = { probe, existing: data && parseRoutingData(data) || undefined };
-                break;
+                return { probe, sourceConfig, responded: true, existing: data && parseRoutingData(data) || undefined, error: "" };
             } catch (e) {
-                probeErrors.push(`${sourceConfig.url}: ${(e as Error).stack ?? e}`);
-                probe.dispose();
+                return { probe, sourceConfig, responded: false, existing: undefined, error: `${sourceConfig.url}: ${(e as Error).stack ?? e}` };
             }
-        }
-        if (!found) {
-            throw new Error(`Cannot initialize storage for ${this.getDebugName()}: no source answered. ${probeErrors.join(" | ")}`);
-        }
-        let active = this.configured;
-        let { probe, existing } = found;
-        let needsWrite = true;
-        if (existing && getConfigVersion(existing) >= getConfigVersion(this.configured)) {
-            if (getConfigVersion(existing) === getConfigVersion(this.configured) && JSON.stringify(existing) !== JSON.stringify(this.configured)) {
-                console.error(`Archives configuration updated without updating the version, for ${probe.config.url}. Updates will be ignored until you increase the version. Using: ${JSON.stringify(existing)}, ignoring: ${JSON.stringify(this.configured)}`);
-            }
-            active = existing;
-            needsWrite = false;
-        }
-        let sources = await this.buildSources(active);
-        if (needsWrite) {
-            // We decided to write (ours is newer than the authoritative first-up source's, or no
-            // routing exists yet). The write goes to all sources, so now EVERY reachable source's
-            // stored routing is read, and the write is refused unless our version is strictly
-            // greater than all of them - a source already at (or past) our version means this
-            // version number is taken, and writing anyway would leave sources at the same version
-            // with different content, each rejecting the other's pushes forever.
-            let best: RemoteConfig | undefined;
-            let conflictUrl: string | undefined;
-            for (let source of sources) {
-                let data: Buffer | undefined;
-                try {
-                    data = await source.read(archives => archives.get(ROUTING_FILE));
-                } catch {
-                    // Down sources are still protected by the server-side version guard when the
-                    // write eventually reaches them
-                    continue;
-                }
-                if (!data) continue;
-                let stored = parseRoutingData(data);
-                if (!best || getConfigVersion(stored) > getConfigVersion(best)) {
-                    best = stored;
-                }
-                if (getConfigVersion(stored) === getConfigVersion(this.configured) && JSON.stringify(stored) !== JSON.stringify(this.configured)) {
-                    conflictUrl = source.config.url;
-                }
+        }));
+        try {
+            let found = fetches.find(x => x.responded);
+            if (!found) {
+                throw new Error(`Cannot initialize storage for ${this.getDebugName()}: no source answered. ${fetches.map(x => x.error).join(" | ")}`);
             }
-            if (best && getConfigVersion(best) >= getConfigVersion(this.configured)) {
-                if (conflictUrl && getConfigVersion(best) === getConfigVersion(this.configured)) {
-                    console.error(`Archives configuration updated without updating the version, for ${conflictUrl}. Updates will be ignored until you increase the version. Using: ${JSON.stringify(best)}, ignoring: ${JSON.stringify(this.configured)}`);
+            let existing = found.existing;
+            let active = this.configured;
+            let needsWrite = true;
+            if (existing && getConfigVersion(existing) >= getConfigVersion(this.configured)) {
+                if (getConfigVersion(existing) === getConfigVersion(this.configured) && JSON.stringify(existing) !== JSON.stringify(this.configured)) {
+                    console.error(`Archives configuration updated without updating the version, for ${found.sourceConfig.url}. Updates will be ignored until you increase the version. Using: ${JSON.stringify(existing)}, ignoring: ${JSON.stringify(this.configured)}`);
                 }
-                active = best;
+                active = existing;
                 needsWrite = false;
-                for (let source of sources) {
-                    source.dispose();
-                }
-                sources = await this.buildSources(active);
             }
-        }
-        if (needsWrite) {
-            // The update only happens when EVERY source would accept it - a partial update would
-            // leave the sources out of sync, and a client without access retrying the write forever
-            // would never initialize. Without full access we run on the stored config (when there
-            // is one) and log the problem instead.
-            let missing: string[] = [];
-            for (let source of sources) {
-                try {
-                    if (!await source.hasWriteAccess()) {
-                        missing.push(source.config.url);
+            if (needsWrite) {
+                // We decided to write (ours is newer than the authoritative first-up source's, or
+                // no routing exists yet). The write goes to all sources, so it is refused unless
+                // our version is strictly greater than EVERY source's stored version (all already
+                // fetched above) - a source at (or past) our version means this version number is
+                // taken, and writing anyway would leave sources at the same version with different
+                // content, each rejecting the other's pushes forever. Down sources are still
+                // protected by the server-side version guard when the write eventually reaches them.
+                let best: RemoteConfig | undefined;
+                let conflictUrl: string | undefined;
+                for (let fetch of fetches) {
+                    let stored = fetch.existing;
+                    if (!stored) continue;
+                    if (!best || getConfigVersion(stored) > getConfigVersion(best)) {
+                        best = stored;
                     }
-                } catch (e) {
-                    missing.push(`${source.config.url} (check failed: ${(e as Error).stack ?? e})`);
+                    if (getConfigVersion(stored) === getConfigVersion(this.configured) && JSON.stringify(stored) !== JSON.stringify(this.configured)) {
+                        conflictUrl = fetch.sourceConfig.url;
+                    }
+                }
+                if (best && getConfigVersion(best) >= getConfigVersion(this.configured)) {
+                    if (conflictUrl && getConfigVersion(best) === getConfigVersion(this.configured)) {
+                        console.error(`Archives configuration updated without updating the version, for ${conflictUrl}. Updates will be ignored until you increase the version. Using: ${JSON.stringify(best)}, ignoring: ${JSON.stringify(this.configured)}`);
+                    }
+                    active = best;
+                    needsWrite = false;
                 }
             }
-            if (missing.length) {
-                console.error(`Not writing the storage routing config for ${this.getDebugName()} (version ${getConfigVersion(this.configured)}): no write access to ${missing.join(", ")}. ${existing && `Running on the stored config (version ${getConfigVersion(existing)}) instead.` || `No stored config exists, so the bucket cannot be created until write access is granted.`}`);
-                if (existing) {
-                    active = existing;
-                    for (let source of sources) {
-                        source.dispose();
+            let sources = await this.buildSources(active);
+            if (needsWrite) {
+                // The update only happens when EVERY source would accept it - a partial update
+                // would leave the sources out of sync, and a client without access retrying the
+                // write forever would never initialize. Without full access we run on the stored
+                // config (when there is one) and log the problem instead.
+                let missing: string[] = [];
+                for (let source of sources) {
+                    try {
+                        if (!await source.hasWriteAccess()) {
+                            missing.push(source.config.url);
+                        }
+                    } catch (e) {
+                        missing.push(`${source.config.url} (check failed: ${(e as Error).stack ?? e})`);
                     }
-                    sources = await this.buildSources(active);
                 }
-            } else {
-                try {
-                    // A rejected write fails init, which retries from scratch - re-reading the
-                    // routing, so losing a create race to another client just adopts their config
-                    // on the next attempt.
-                    await probe.write(archives => archives.set(ROUTING_FILE, serializeRemoteConfig(this.configured)));
-                } catch (e) {
-                    for (let source of sources) {
-                        source.dispose();
+                if (missing.length) {
+                    console.error(`Not writing the storage routing config for ${this.getDebugName()} (version ${getConfigVersion(this.configured)}): no write access to ${missing.join(", ")}. ${existing && `Running on the stored config (version ${getConfigVersion(existing)}) instead.` || `No stored config exists, so the bucket cannot be created until write access is granted.`}`);
+                    if (existing) {
+                        active = existing;
+                        for (let source of sources) {
+                            source.dispose();
+                        }
+                        sources = await this.buildSources(active);
+                    }
+                } else {
+                    try {
+                        // A rejected write fails init, which retries from scratch - re-reading the
+                        // routing, so losing a create race to another client just adopts their
+                        // config on the next attempt.
+                        await found.probe.write(archives => archives.set(ROUTING_FILE, serializeRemoteConfig(this.configured)));
+                    } catch (e) {
+                        for (let source of sources) {
+                            source.dispose();
+                        }
+                        throw e;
                     }
-                    probe.dispose();
-                    throw e;
                 }
             }
+            this.activeConfig = active;
+            this.startConfigPoll();
+            return { config: active, sources };
+        } finally {
+            for (let fetch of fetches) {
+                fetch.probe.dispose();
+            }
         }
-        probe.dispose();
-        this.activeConfig = active;
-        this.startConfigPoll();
-        return { config: active, sources };
     }
 
     private async buildSources(config: RemoteConfig): Promise<SourceWrapper[]> {
@@ -293,6 +290,10 @@ export class ArchivesChain implements IArchives {
         }
         let latest = await this.fetchLatestConfig(state);
         if (!latest) return;
+        await this.adoptNewConfig(state, latest);
+    }
+
+    private async adoptNewConfig(state: ChainState, latest: RemoteConfig): Promise<void> {
         if (JSON.stringify(latest) === JSON.stringify(state.config)) return;
         console.log(`Storage routing config changed for ${this.getDebugName()}, rebuilding sources`);
         let oldByConfig = new Map(state.sources.map(source => [JSON.stringify(source.config), source]));
@@ -304,11 +305,13 @@ export class ArchivesChain implements IArchives {
                 oldByConfig.delete(key);
                 sources.push(old);
             } else {
-                sources.push(await SourceWrapper.create(sourceConfig));
+                let source = await SourceWrapper.create(sourceConfig);
+                source.startPinging();
+                sources.push(source);
             }
         }
         // In-flight requests still hold the old wrappers and finish fine; dispose just stops any
-        // background reconnect loops
+        // background reconnect/ping loops
         for (let leftover of oldByConfig.values()) {
             leftover.dispose();
         }
@@ -316,6 +319,42 @@ export class ArchivesChain implements IArchives {
         this.statePromise = Promise.resolve({ config: latest, sources });
     }
 
+    // When every source looks down, the routing config is re-read from EVERY source - which both
+    // re-attempts their connections (our liveness re-check) and discovers a routing update we may
+    // have missed (adopting it re-initializes the source list). Concurrent callers share one pass.
+    private lastAvailabilityRecheck = 0;
+    private availabilityRecheckInFlight: Promise<void> | undefined;
+    private recheckAvailability(): Promise<void> {
+        if (this.availabilityRecheckInFlight) return this.availabilityRecheckInFlight;
+        if (Date.now() - this.lastAvailabilityRecheck < AVAILABILITY_RECHECK_THROTTLE) return Promise.resolve();
+        this.lastAvailabilityRecheck = Date.now();
+        this.availabilityRecheckInFlight = this.recheckAvailabilityNow().finally(() => {
+            this.availabilityRecheckInFlight = undefined;
+        });
+        return this.availabilityRecheckInFlight;
+    }
+    private async recheckAvailabilityNow(): Promise<void> {
+        if (this.disposed || !this.statePromise) return;
+        let state: ChainState;
+        try {
+            state = await this.statePromise;
+        } catch {
+            // Init is failing; its own retry loop handles that
+            return;
+        }
+        let results = await Promise.all(state.sources.map(async source => {
+            try {
+                let data = await source.read(archives => archives.get(ROUTING_FILE));
+                return data && parseRoutingData(data) || undefined;
+            } catch {
+                return undefined;
+            }
+        }));
+        let latest = results.find(x => x);
+        if (!latest) return;
+        await this.adoptNewConfig(state, latest);
+    }
+
     // Runs a request against the first available source (that covers the key's route, when one is
     // given), moving to the next one ONLY when the source's WebSocket is down (checked directly -
     // never by inspecting the error, which is arbitrary data). An error from a connected source is
@@ -324,32 +363,38 @@ export class ArchivesChain implements IArchives {
         if (config.write) {
             return await this.runWrite(config.route, run);
         }
-        let errors: string[] = [];
-        for (let source of state.sources) {
-            if (config.route !== undefined && !routeContains(source.config.route, config.route)) continue;
-            if (!configWindowCurrent(source.config)) continue;
-            if (!source.isConnected()) {
-                source.noteFailure();
-                errors.push(`${source.config.url} is not connected`);
-                continue;
-            }
-            try {
-                if (config.apiOnly) {
-                    let api = source.api;
-                    if (!api) {
-                        errors.push(`${source.config.url} has URL-only access, which cannot serve this operation`);
-                        continue;
+        let recheckedAvailability = false;
+        while (true) {
+            let errors: string[] = [];
+            for (let source of state.sources) {
+                if (config.route !== undefined && !routeContains(source.config.route, config.route)) continue;
+                if (!configWindowCurrent(source.config)) continue;
+                try {
+                    if (config.apiOnly) {
+                        let api = source.api;
+                        if (!api) {
+                            errors.push(`${source.config.url} has URL-only access, which cannot serve this operation`);
+                            continue;
+                        }
+                        return await run(api);
                     }
-                    return await run(api);
+                    return await source.read(run);
+                } catch (e) {
+                    if (source.isConnected()) throw e;
+                    source.noteFailure();
+                    errors.push(String((e as Error).stack ?? e));
                 }
-                return await source.read(run);
-            } catch (e) {
-                if (source.isConnected()) throw e;
-                source.noteFailure();
-                errors.push(String((e as Error).stack ?? e));
             }
+            // Every source failed: re-contact everything (routing re-read + connection re-attempt,
+            // possibly adopting a routing update) and try once more before giving up
+            if (!recheckedAvailability) {
+                recheckedAvailability = true;
+                await this.recheckAvailability();
+                state = await this.getState();
+                continue;
+            }
+            throw new Error(`All sources failed for ${this.getDebugName()}${config.route !== undefined && ` (route ${config.route})` || ""}: ${errors.join(" | ") || "no sources available"}`);
         }
-        throw new Error(`All sources failed for ${this.getDebugName()}${config.route !== undefined && ` (route ${config.route})` || ""}: ${errors.join(" | ") || "no sources available"}`);
     }
 
     // Writes are consistent: they go to the first currently-valid source covering the key's route
@@ -360,16 +405,20 @@ export class ArchivesChain implements IArchives {
     private async runWrite<T>(route: number | undefined, run: (archives: IArchives) => Promise<T>): Promise<T> {
         let retriedWrongWindow = false;
         let retriedWrongRoute = false;
+        let recheckedAvailability = false;
         while (true) {
             let state = await this.getState();
             let target = state.sources.find(x => configAcceptsWrites(x.config) && (route === undefined || routeContains(x.config.route, route)));
             if (!target) {
+                // Before giving up, re-contact every source once (routing re-read + connection
+                // re-attempt, possibly adopting a routing update that fixes the target)
+                if (!recheckedAvailability) {
+                    recheckedAvailability = true;
+                    await this.recheckAvailability();
+                    continue;
+                }
                 throw new Error(`No source accepts writes for ${this.getDebugName()}${route !== undefined && ` (route ${route})` || ""} (every source is outside its valid window or outside the key's route)`);
             }
-            if (!target.isConnected()) {
-                target.noteFailure();
-                throw new Error(`Cannot write: the write target ${target.config.url} is not connected (writes never fail over to other sources)`);
-            }
             try {
                 return await target.write(run);
             } catch (e) {
@@ -384,8 +433,15 @@ export class ArchivesChain implements IArchives {
                     await this.prepareWrongTargetRetry(state, "route");
                     continue;
                 }
+                // The connection died during (or before) the request. One full re-contact pass,
+                // then one retry - still the same consistent target, never a different node.
                 if (!target.isConnected()) {
                     target.noteFailure();
+                    if (!recheckedAvailability) {
+                        recheckedAvailability = true;
+                        await this.recheckAvailability();
+                        continue;
+                    }
                 }
                 throw e;
             }
@@ -442,8 +498,8 @@ export class ArchivesChain implements IArchives {
     // A minimal set of connected, currently-valid API sources whose routes cover [0, 1) - listing
     // operations must merge every shard. Greedy sweep: repeatedly take the source that extends
     // coverage the furthest. In practice this is one source (unsharded), or one per shard.
-    private selectCoveringSources(state: ChainState): SourceWrapper[] {
-        let candidates = state.sources.filter(x => configWindowCurrent(x.config) && x.api && x.isConnected());
+    private selectCoveringSources(state: ChainState, excluded: Set<SourceWrapper>): SourceWrapper[] {
+        let candidates = state.sources.filter(x => configWindowCurrent(x.config) && x.api && !excluded.has(x));
         let chosen: SourceWrapper[] = [];
         let covered = 0;
         while (covered < 1) {
@@ -466,21 +522,39 @@ export class ArchivesChain implements IArchives {
         return chosen;
     }
 
-    private async runOnApi<T>(source: SourceWrapper, run: (archives: IArchives) => Promise<T>): Promise<T> {
-        let api = source.api;
-        if (!api) {
-            throw new Error(`${source.config.url} has URL-only access, which cannot serve this operation`);
+    // Fans one call out over a covering set of shards. Sources are never pre-filtered by
+    // connectivity - the request is attempted, and a failure with the socket down afterwards
+    // excludes that source and re-selects the covering set (an error from a live source throws).
+    private async runOnCovering<T>(run: (archives: IArchives) => Promise<T>): Promise<T[]> {
+        let state = await this.getState();
+        let excluded = new Set<SourceWrapper>();
+        while (true) {
+            let covering = this.selectCoveringSources(state, excluded);
+            let results = await Promise.all(covering.map(async source => {
+                let api = source.api;
+                if (!api) {
+                    throw new Error(`${source.config.url} has URL-only access, which cannot serve this operation`);
+                }
+                try {
+                    return { value: await run(api) };
+                } catch (e) {
+                    if (source.isConnected()) throw e;
+                    source.noteFailure();
+                    excluded.add(source);
+                    return undefined;
+                }
+            }));
+            if (results.every(x => x)) {
+                return results.map(x => (x as { value: T }).value);
+            }
         }
-        return await run(api);
     }
 
     public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
         return (await this.findInfo(prefix, config)).map(x => x.path);
     }
     public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
-        let state = await this.getState();
-        let covering = this.selectCoveringSources(state);
-        let results = await Promise.all(covering.map(source => this.runOnApi(source, archives => archives.findInfo(prefix, config))));
+        let results = await this.runOnCovering(archives => archives.findInfo(prefix, config));
         // Overlapping shards can both report a path; the newest wins
         let byPath = new Map<string, ArchiveFileInfo>();
         for (let list of results) {
@@ -496,14 +570,12 @@ export class ArchivesChain implements IArchives {
         return merged;
     }
     public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
-        let state = await this.getState();
-        let covering = this.selectCoveringSources(state);
-        let results = await Promise.all(covering.map(source => this.runOnApi(source, async archives => {
+        let results = await this.runOnCovering(async archives => {
             if (!archives.getChangesAfter) {
                 throw new Error(`${archives.getDebugName()} does not support getChangesAfter`);
             }
             return await archives.getChangesAfter(time);
-        })));
+        });
         let byPath = new Map<string, ArchiveFileInfo>();
         for (let list of results) {
             for (let file of list) {
@@ -518,14 +590,12 @@ export class ArchivesChain implements IArchives {
         return merged;
     }
     public async getSyncStatus(): Promise<ArchivesSyncStatus> {
-        let state = await this.getState();
-        let covering = this.selectCoveringSources(state);
-        let statuses = await Promise.all(covering.map(source => this.runOnApi(source, async archives => {
+        let statuses = await this.runOnCovering(async archives => {
             if (!archives.getSyncStatus) {
                 throw new Error(`${archives.getDebugName()} does not support getSyncStatus`);
             }
             return await archives.getSyncStatus();
-        })));
+        });
         return {
             allScansComplete: statuses.every(x => x.allScansComplete),
             indexSize: statuses.reduce((sum, x) => sum + x.indexSize, 0),
@@ -569,58 +639,74 @@ export class ArchivesChain implements IArchives {
     // is down (error + socket down, same rule as reads) - each shard receives a different key, so
     // write consistency is preserved.
     private async setVariableShard(key: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
-        let state = await this.getState();
-        // Per-shard write consistency still holds: within a route, only the FIRST source may take
-        // writes - latency only picks WHICH shard the key materializes into
-        let targetsByRoute = new Map<string, SourceWrapper>();
-        for (let source of state.sources) {
-            if (!configAcceptsWrites(source.config)) continue;
-            let routeKey = JSON.stringify(source.config.route || FULL_ROUTE);
-            if (!targetsByRoute.has(routeKey)) {
-                targetsByRoute.set(routeKey, source);
+        let recheckedAvailability = false;
+        while (true) {
+            let state = await this.getState();
+            // Per-shard write consistency still holds: within a route, only the FIRST source may
+            // take writes - latency only picks WHICH shard the key materializes into
+            let targetsByRoute = new Map<string, SourceWrapper>();
+            for (let source of state.sources) {
+                if (!configAcceptsWrites(source.config)) continue;
+                let routeKey = JSON.stringify(source.config.route || FULL_ROUTE);
+                if (!targetsByRoute.has(routeKey)) {
+                    targetsByRoute.set(routeKey, source);
+                }
             }
-        }
-        let targets = [...targetsByRoute.values()];
-        sort(targets, x => x.getLatency());
-        let errors: string[] = [];
-        for (let target of targets) {
-            if (!target.isConnected()) {
-                target.noteFailure();
-                errors.push(`${target.config.url} is not connected`);
-                continue;
+            let targets = [...targetsByRoute.values()];
+            sort(targets, x => x.getLatency());
+            let errors: string[] = [];
+            for (let target of targets) {
+                let [start, end] = target.config.route || FULL_ROUTE;
+                let fullKey = key.replace(VARIABLE_SHARD, VARIABLE_SHARD + "_" + (start + Math.random() * (end - start)));
+                try {
+                    await target.write(archives => archives.set(fullKey, data, config));
+                    return fullKey;
+                } catch (e) {
+                    // NOTE: If we run into cases when transport level errors happen, but we are still connected, then we might want to add additional checking here, additional errors in which we will retry the other targets. However, ideally, checking if the WebSocket connection is still connected will handle all those cases. Distinguishing between application errors (which we can't retry) and transport errors.
+                    if (target.isConnected()) throw e;
+                    target.noteFailure();
+                    errors.push(String((e as Error).stack ?? e));
+                }
             }
-            let [start, end] = target.config.route || FULL_ROUTE;
-            let fullKey = key.replace(VARIABLE_SHARD, VARIABLE_SHARD + "_" + (start + Math.random() * (end - start)));
-            try {
-                await target.write(archives => archives.set(fullKey, data, config));
-                return fullKey;
-            } catch (e) {
-                // An error alone doesn't justify moving on (it would be an application error); the
-                // socket must also be down
-                if (target.isConnected()) throw e;
-                target.noteFailure();
-                errors.push(String((e as Error).stack ?? e));
+            // Every shard failed: re-contact everything and try once more before giving up
+            if (!recheckedAvailability) {
+                recheckedAvailability = true;
+                await this.recheckAvailability();
+                continue;
             }
+            throw new Error(`Every variable-shard write target failed for ${this.getDebugName()}: ${errors.join(" | ") || "no sources accept writes"}`);
         }
-        throw new Error(`Every variable-shard write target failed for ${this.getDebugName()}: ${errors.join(" | ") || "no sources accept writes"}`);
     }
 
     public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
         if (config.path.includes(VARIABLE_SHARD) && parseVariableRoute(config.path) === undefined) {
             throw new Error(`setLargeFile does not support VARIABLE_SHARD keys (there is no way to return the materialized key); write the file with set, or materialize the key yourself. Key: ${JSON.stringify(config.path)}`);
         }
-        let state = await this.getState();
-        // Same consistency rule as runWrite: the first currently-valid source for the route, or nothing
         let route = getRoute(config.path);
-        let target = state.sources.find(x => configAcceptsWrites(x.config) && routeContains(x.config.route, route));
-        if (!target) {
-            throw new Error(`No source accepts writes for setLargeFile on ${this.getDebugName()} (route ${route})`);
-        }
-        if (!target.isConnected()) {
-            target.noteFailure();
-            throw new Error(`Cannot write: the write target ${target.config.url} is not connected (writes never fail over to other sources)`);
+        let recheckedAvailability = false;
+        while (true) {
+            let state = await this.getState();
+            // Same consistency rule as runWrite: the first currently-valid source for the route, or nothing
+            let target = state.sources.find(x => configAcceptsWrites(x.config) && routeContains(x.config.route, route));
+            if (!target) {
+                if (!recheckedAvailability) {
+                    recheckedAvailability = true;
+                    await this.recheckAvailability();
+                    continue;
+                }
+                throw new Error(`No source accepts writes for setLargeFile on ${this.getDebugName()} (route ${route})`);
+            }
+            try {
+                await target.write(archives => archives.setLargeFile(config));
+                return;
+            } catch (e) {
+                // The data stream cannot be rewound, so a mid-upload disconnect cannot be retried
+                if (!target.isConnected()) {
+                    target.noteFailure();
+                }
+                throw e;
+            }
         }
-        await target.write(archives => archives.setLargeFile(config));
     }
 
     public async getURL(path: string): Promise<string> {