sliftutils 1.7.88 → 1.7.89

package/storage/remoteStorage/bucketDisk.d.ts

@@ -1,10 +1,31 @@
 /// <reference types="node" />
 /// <reference types="node" />
 import { RemoteConfig } from "../IArchives";
-export declare function getBucketFolder(account: string, bucketName: string, route?: [number, number]): string;
-export declare function readRoutingFile(folder: string): Promise<Buffer | undefined>;
+/** A store's folder, from the only three things that identify it. The name is the config entry's name and nothing else: the same name is the same storage, whatever its window or route say, and a different name is different storage even for the same URL. Nothing about a folder changes when the routing does. */
+export declare function getBucketFolder(name: string, account: string, bucketName: string): string;
+export type StoreFolder = {
+    account: string;
+    name: string;
+    bucketName: string;
+    folder: string;
+};
+/** Every store this server holds for an account, found by walking the disk. */
+export declare function listAccountStoreFolders(account: string): Promise<StoreFolder[]>;
+/** Every store this server holds for ONE bucket - one per name it has ever been given. */
+export declare function listBucketStoreFolders(account: string, bucketName: string): Promise<StoreFolder[]>;
+export declare function readRoutingFile(folder: string): Promise<{
+    data: Buffer;
+    writeTime: number;
+} | undefined>;
+/** The bucket's routing config as this server holds it: the newest copy among its stores. Each store keeps its own, and they converge - so when they disagree, the one written most recently is the one that has heard the most. */
+export declare function readNewestRoutingFile(account: string, bucketName: string): Promise<{
+    data: Buffer;
+    writeTime: number;
+    size: number;
+    name: string;
+} | undefined>;
 export declare function readRoutingFromDisk(account: string, bucketName: string): Promise<RemoteConfig | undefined>;
-/** The routing file lives ONLY in the plain (routeless) bucket folder - it is what DEFINES the per-route stores, so it cannot live inside any of them. Served directly for reads (the stores never hold it). */
+/** What an anonymous URL read of the routing file gets: the same newest copy. */
 export declare function getRoutingFileResult(account: string, bucketName: string): Promise<{
     data: Buffer;
     writeTime: number;

package/storage/remoteStorage/bucketDisk.ts

@@ -4,47 +4,81 @@ import { RemoteConfig } from "../IArchives";
 import { ROUTING_FILE, parseRoutingData } from "./remoteConfig";
 import { getStorageServerConfig } from "./serverConfig";
 
-// The on-disk layout of buckets, and the direct-disk reads of the files that exist outside every store: the per-route folder naming and the routing file.
+// The on-disk layout, and what can be learned from it without opening a store. The layout IS the state: <account>/<name>/<bucket> is a store, and the set of folders is the set of stores this server has - there is no separate record of them to fall out of step.
 
-/** Each ROUTE gets its own folder: the bucket name plus the route range. One store serves exactly one route, so different shards of the same bucket - even across processes on different ports of the same machine - never share a folder and can never mix their data. No route (or the full route) keeps the plain folder, which is also where the routing file always lives. */
-function getRouteFolderSuffix(route: [number, number] | undefined): string {
-    if (!route || route[0] === 0 && route[1] === 1) return "";
-    return `-route-${route[0]}-${route[1]}`;
+// Everything this server stores for buckets. "buckets3" because the layout under it changed shape - the previous generations named folders after the route, which meant re-routing renamed the storage.
+const BUCKETS_FOLDER = "buckets4";
+
+/** A store's folder, from the only three things that identify it. The name is the config entry's name and nothing else: the same name is the same storage, whatever its window or route say, and a different name is different storage even for the same URL. Nothing about a folder changes when the routing does. */
+export function getBucketFolder(name: string, account: string, bucketName: string): string {
+    return path.join(getStorageServerConfig().folder, BUCKETS_FOLDER, account, name, bucketName);
+}
+
+async function readDirectory(folder: string): Promise<string[]> {
+    try {
+        return await fs.promises.readdir(folder);
+    } catch (e) {
+        if ((e as { code?: string }).code === "ENOENT") return [];
+        throw e;
+    }
+}
+
+export type StoreFolder = { account: string; name: string; bucketName: string; folder: string };
+
+/** Every store this server holds for an account, found by walking the disk. */
+export async function listAccountStoreFolders(account: string): Promise<StoreFolder[]> {
+    let accountFolder = path.join(getStorageServerConfig().folder, BUCKETS_FOLDER, account);
+    let found: StoreFolder[] = [];
+    for (let name of await readDirectory(accountFolder)) {
+        for (let bucketName of await readDirectory(path.join(accountFolder, name))) {
+            found.push({ account, name, bucketName, folder: getBucketFolder(name, account, bucketName) });
+        }
+    }
+    return found;
 }
-export function getBucketFolder(account: string, bucketName: string, route?: [number, number]): string {
-    return path.join(getStorageServerConfig().folder, "buckets2", account, bucketName + getRouteFolderSuffix(route));
+
+/** Every store this server holds for ONE bucket - one per name it has ever been given. */
+export async function listBucketStoreFolders(account: string, bucketName: string): Promise<StoreFolder[]> {
+    return (await listAccountStoreFolders(account)).filter(x => x.bucketName === bucketName);
 }
 
 /** The routing file is ours, on our own disk, at a path we know - so it is read directly. Going through an ArchivesDisk would construct a whole store (handle cache sweep loop, uploads-folder cleanup) just to read one file. */
 function getRoutingFilePath(folder: string): string {
     return path.join(folder, "files", ROUTING_FILE);
 }
-export async function readRoutingFile(folder: string): Promise<Buffer | undefined> {
+export async function readRoutingFile(folder: string): Promise<{ data: Buffer; writeTime: number } | undefined> {
+    let filePath = getRoutingFilePath(folder);
     try {
-        return await fs.promises.readFile(getRoutingFilePath(folder));
+        let data = await fs.promises.readFile(filePath);
+        let stats = await fs.promises.stat(filePath);
+        return { data, writeTime: stats.mtimeMs };
     } catch (e) {
         if ((e as { code?: string }).code === "ENOENT") return undefined;
         throw e;
     }
 }
 
+/** The bucket's routing config as this server holds it: the newest copy among its stores. Each store keeps its own, and they converge - so when they disagree, the one written most recently is the one that has heard the most. */
+export async function readNewestRoutingFile(account: string, bucketName: string): Promise<{ data: Buffer; writeTime: number; size: number; name: string } | undefined> {
+    let newest: { data: Buffer; writeTime: number; size: number; name: string } | undefined;
+    for (let store of await listBucketStoreFolders(account, bucketName)) {
+        let file = await readRoutingFile(store.folder);
+        if (!file) continue;
+        if (newest && file.writeTime <= newest.writeTime) continue;
+        newest = { data: file.data, writeTime: file.writeTime, size: file.data.length, name: store.name };
+    }
+    return newest;
+}
+
 export async function readRoutingFromDisk(account: string, bucketName: string): Promise<RemoteConfig | undefined> {
-    let data = await readRoutingFile(getBucketFolder(account, bucketName));
-    if (!data) return undefined;
-    return parseRoutingData(data);
+    let newest = await readNewestRoutingFile(account, bucketName);
+    if (!newest) return undefined;
+    return parseRoutingData(newest.data);
 }
 
-/** The routing file lives ONLY in the plain (routeless) bucket folder - it is what DEFINES the per-route stores, so it cannot live inside any of them. Served directly for reads (the stores never hold it). */
+/** What an anonymous URL read of the routing file gets: the same newest copy. */
 export async function getRoutingFileResult(account: string, bucketName: string): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
-    let filePath = getRoutingFilePath(getBucketFolder(account, bucketName));
-    try {
-        let data = await fs.promises.readFile(filePath);
-        let stats = await fs.promises.stat(filePath);
-        return { data, writeTime: stats.mtimeMs, size: data.length };
-    } catch (e) {
-        if ((e as { code?: string }).code === "ENOENT") return undefined;
-        throw e;
-    }
+    return await readNewestRoutingFile(account, bucketName);
 }
 
 export type BucketDiskInfo = {

package/storage/remoteStorage/certTrustModal.d.ts

@@ -0,0 +1 @@
+export declare function showCertTrustModal(serverUrl: string): void;

package/storage/remoteStorage/certTrustModal.tsx

@@ -0,0 +1,48 @@
+import preact from "preact";
+import { isNode } from "socket-function/src/misc";
+import { css } from "typesafecss";
+
+// Shown in the browser when we can't establish a connection to a storage server - most often because it serves a self-signed certificate (see --selfSigned) that this browser has not accepted yet. Gives the user a link to the server, where the browser lets them accept the certificate (the ?trustCert=1 page, see accessPage.tsx); once accepted, the app's own background connection to that server starts working. A no-op in Node.
+
+// One modal per server origin, so repeated connection failures (there are many, on a retry loop) never stack modals
+const shownForOrigin = new Set<string>();
+
+export function showCertTrustModal(serverUrl: string): void {
+    if (isNode()) return;
+    let origin: string;
+    try {
+        origin = new URL(serverUrl).origin;
+    } catch {
+        return;
+    }
+    if (shownForOrigin.has(origin)) return;
+    shownForOrigin.add(origin);
+
+    let container = document.createElement("div");
+    document.body.appendChild(container);
+    let close = () => {
+        preact.render(null, container);
+        container.remove();
+        // Leave it in shownForOrigin: the user dismissed it, so don't re-pop it on the next failure of the same server
+    };
+    preact.render(<CertTrustModal origin={origin} onClose={close} />, container);
+}
+
+function CertTrustModal(props: { origin: string; onClose: () => void }) {
+    // Loading this page at all means the browser accepted the certificate; the page then tells the user they can come back
+    let trustUrl = props.origin + "/?trustCert=1";
+    return <div className={css.fixed.pos(0, 0).fillBoth.zIndex(2147483647)
+        .hbox(0).justifyContent("center").alignItems("center").hsla(0, 0, 0, 0.5)
+    }>
+        <div className={css.vbox(12).pad2(24).maxWidth(480).hsl(0, 0, 100).borderRadius(8)}>
+            <div>Can't reach the storage server at <code>{props.origin}</code>.</div>
+            <div>
+                If this server uses a self-signed certificate, the browser blocks the connection until you accept it. Open the server below, accept the certificate warning, then return to this page - it will connect automatically.
+            </div>
+            <a href={trustUrl} target="_blank" rel="noreferrer">Open {props.origin} and trust its certificate</a>
+            <div className={css.hbox(8).justifyContent("flex-end")}>
+                <button onClick={props.onClose}>Close</button>
+            </div>
+        </div>
+    </div>;
+}

package/storage/remoteStorage/cliArgs.d.ts

@@ -1 +1,3 @@
 export declare function getArg(name: string): string | undefined;
+/** A valueless boolean flag: true when --name is present (with nothing, or a following flag). */
+export declare function getFlag(name: string): boolean;

package/storage/remoteStorage/cliArgs.ts

@@ -7,3 +7,8 @@ export function getArg(name: string): string | undefined {
     }
     return value;
 }
+
+/** A valueless boolean flag: true when --name is present (with nothing, or a following flag). */
+export function getFlag(name: string): boolean {
+    return process.argv.includes(`--${name}`);
+}

package/storage/remoteStorage/createArchives.d.ts

@@ -1,10 +1,11 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, RemoteConfig, RemoteConfigBase, SourceConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, SetConfig } from "../IArchives";
+import { IArchives, RemoteConfig, RemoteConfigBase, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig } from "../IArchives";
 import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
 /** The address, port, account, and bucket name a bucket routing URL addresses. Throws when the URL isn't a hosted bucket routing URL (https://host:port/file/<account>/<bucketName>/storage/storagerouting.json). */
 export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteConfig";
-export declare function createApiArchives(source: SourceConfig): IArchives;
+/** A client for ONE source - see storeSources.ts. Re-exported here because a chain is built out of them. */
+export { createApiArchives } from "./storeSources";
 export type ArchivesChainOptions = {
     /** Outside of node we default to read-only downloads over the public URLs (no API connection) when the config has public sources. Set this to connect to the API anyway - needed for writing, listing, and any other operation the plain URL form cannot serve. */
     directConnect?: boolean;
@@ -91,15 +92,15 @@ export declare class ArchivesChain implements IArchives {
     set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     private setRoutingConfig;
     del(fileName: string, config?: DelConfig): Promise<void>;
+    /** See IArchives.move. When one node is the write target for BOTH paths, that node moves the file itself - the bytes never come through us - with the same wrong-window/route re-resolution as any write. When the paths route to different shards no single node holds both, so the move degrades to a copy through us plus a delete, CONFIRMED at the destination before the source is touched. No smart timeout on the node-side move: it can be a big file's worth of node-side work, which the upload-sized deadlines would misjudge. */
+    move(config: MoveFileConfig): Promise<void>;
     private getVariableShardTargets;
     /** The key setVariableShard would materialize for this VARIABLE_SHARD key (a value in the preferred shard's route range), without writing anything. */
     getShardKey(key: string): Promise<string>;
     private setVariableShard;
-    setLargeFile(config: {
-        path: string;
-        lastModified?: number;
-        getNextData(): Promise<Buffer | undefined>;
-    }): Promise<void>;
+    /** A large file is written exactly like a small one - same write node, same wrong-window/route re-resolution, same fallbacks - so a value's SIZE never decides its write semantics (set streams through here past LARGE_SET_THRESHOLD, and a file that grew past it must not suddenly lose the availability its caller asked for). The one difference: every attempt after the first has to rewind the stream, so a config without restartStream gets a single attempt. */
+    setLargeFile(config: SetLargeFileConfig): Promise<void>;
+    private setLargeFileOnce;
     getURL(path: string): Promise<string>;
     /** Every URL that could serve this path: public sources matching both the path's route and the current valid window. The first is the write node's (first matching source in config order, see runPrimary - the one guaranteed current); the rest are ranked fastest-first by measured latency. Empty when none qualify. */
     getURLs(path: string): Promise<string[]>;

package/storage/remoteStorage/createArchives.ts

@@ -2,11 +2,12 @@ import { isNode, sort, timeoutToUndefinedSilent, watchSlowPromise } from "socket
 import { delay } from "socket-function/src/batching";
 import {
     IArchives, RemoteConfig, RemoteConfigBase, SourceConfig,
-    ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, SetConfig, STORAGE_WRONG_VALID_WINDOW,
+    ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig, STORAGE_WRONG_VALID_WINDOW,
     STORAGE_WRONG_ROUTE, FULL_ROUTE, VARIABLE_SHARD, LARGE_SET_THRESHOLD, bufferChunkStream,
 } from "../IArchives";
+import { copyArchiveFile } from "../archiveHelpers";
 import {
-    ROUTING_FILE, getConfigVersion, parseHostedUrl, parseBackblazeUrl,
+    ROUTING_FILE, getConfigVersion, parseHostedUrl, parseBackblazeUrl, parseRoutingData, assertValidRemoteConfig,
     normalizeRemoteConfig, normalizeSource, serializeRemoteConfig,
     getRoute, routeContains, parseVariableRoute, getBucketBaseUrl, buildFileUrl,
 } from "./remoteConfig";
@@ -16,8 +17,8 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { ArchivesRemote, parseStorageUrl, authenticateStorage } from "./ArchivesRemote";
 import { onServerRoutingChanged } from "./storageClientController";
 import { ArchivesBackblaze } from "../backblaze";
-import { ServerBucketInfo, ActiveBucketInfo, getLocalArchives } from "./storageServerState";
-import { isOwnAddress } from "./serverConfig";
+import { ArchivesDisk } from "../ArchivesDisk";
+import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
 import { RemoteStorageController, STORAGE_NOT_AUTHENTICATED } from "./storageController";
 import { SourceWrapper, RETRY_START_DELAY, RETRY_MAX_DELAY, RETRY_GROWTH } from "./sourceWrapper";
 import { formatTime } from "socket-function/src/formatting/format";
@@ -50,16 +51,8 @@ type SmartTimeout = {
 /** The address, port, account, and bucket name a bucket routing URL addresses. Throws when the URL isn't a hosted bucket routing URL (https://host:port/file/<account>/<bucketName>/storage/storagerouting.json). */
 export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteConfig";
 
-export function createApiArchives(source: SourceConfig): IArchives {
-    if (source.type === "backblaze") {
-        return new ArchivesBackblaze({ bucketName: parseBackblazeUrl(source.url).bucketName, public: source.public, immutable: source.immutable, allowedOrigins: source.allowedOrigins });
-    }
-    let parsed = parseHostedUrl(source.url);
-    if (isNode() && isOwnAddress(parsed.address, parsed.port)) {
-        return getLocalArchives(parsed.account, parsed.bucketName, source);
-    }
-    return new ArchivesRemote({ url: source.url, waitForAccess: false, sourceConfig: source });
-}
+/** A client for ONE source - see storeSources.ts. Re-exported here because a chain is built out of them. */
+export { createApiArchives } from "./storeSources";
 
 type ChainState = {
     config: RemoteConfig;
@@ -77,10 +70,6 @@ function configWindowCurrent(config: SourceConfig): boolean {
     return start <= now && now < end;
 }
 
-function configAcceptsWrites(config: SourceConfig): boolean {
-    return configWindowCurrent(config);
-}
-
 function materializeShardKey(key: string, target: SourceWrapper): string {
     let [start, end] = target.config.route || FULL_ROUTE;
     return key.replace(VARIABLE_SHARD, VARIABLE_SHARD + "_" + (start + Math.random() * (end - start)));
@@ -498,7 +487,7 @@ export class ArchivesChain implements IArchives {
             attempt++;
             let attemptStart = Date.now();
             let state = await this.getState();
-            let target = state.sources.find(x => configAcceptsWrites(x.config) && (config.route === undefined || routeContains(x.config.route, config.route)));
+            let target = state.sources.find(x => configWindowCurrent(x.config) && (config.route === undefined || routeContains(x.config.route, config.route)));
             try {
                 if (!target) {
                     throw new Error(`No source is the ${config.write && "write target" || "primary read source"} for ${this.getDebugName()}${config.route !== undefined && ` (route ${config.route})` || ""} (every source is outside its valid window or outside the key's route)`);
@@ -782,7 +771,7 @@ export class ArchivesChain implements IArchives {
     public async hasWriteAccess(): Promise<boolean> {
         let state = await this.getState();
         for (let source of state.sources) {
-            if (!configAcceptsWrites(source.config)) continue;
+            if (!configWindowCurrent(source.config)) continue;
             if (!await source.hasWriteAccess()) return false;
         }
         return true;
@@ -799,8 +788,8 @@ export class ArchivesChain implements IArchives {
             return await this.setVariableShard(fileName, data, config);
         }
         if (data.length > LARGE_SET_THRESHOLD) {
-            // Streamed automatically so callers never have to call setLargeFile themselves when they already hold the buffer - one giant message would exceed the wire limit and lag every other client on the connection
-            await this.setLargeFile({ path: fileName, lastModified: config?.lastModified, getNextData: bufferChunkStream(data) });
+            // Streamed automatically so callers never have to call setLargeFile themselves when they already hold the buffer - one giant message would exceed the wire limit and lag every other client on the connection. The whole config goes with it (including fallbacks, which the buffer's restartStream makes possible): a file must not get different write semantics just for being big.
+            await this.setLargeFile({ path: fileName, ...config, ...bufferChunkStream(data) });
             return fileName;
         }
         await this.request({ write: true, fallbacks: config?.fallbacks, route: getRoute(fileName), timeout: { uploadBytes: data.length, label: `Upload of ${JSON.stringify(fileName)} (${data.length} bytes)` } }, archives => archives.set(fileName, data, config));
@@ -808,6 +797,8 @@ export class ArchivesChain implements IArchives {
     }
 
     private async setRoutingConfig(data: Buffer, config?: { lastModified?: number }): Promise<string> {
+        // Checked before the first node is written to, not just by each node as it arrives: every server would reject it anyway, and finding that out one node at a time is how a config write ends up half-applied
+        assertValidRemoteConfig(parseRoutingData(data));
         let state = await this.getState();
         let writeTime = config?.lastModified || Date.now();
         let written: string[] = [];
@@ -835,11 +826,42 @@ export class ArchivesChain implements IArchives {
         await this.request({ write: true, fallbacks: config?.fallbacks, route: getRoute(fileName), timeout: { uploadBytes: 0, label: `Deletion of ${JSON.stringify(fileName)}` } }, archives => archives.del(fileName, config));
     }
 
+    /** See IArchives.move. When one node is the write target for BOTH paths, that node moves the file itself - the bytes never come through us - with the same wrong-window/route re-resolution as any write. When the paths route to different shards no single node holds both, so the move degrades to a copy through us plus a delete, CONFIRMED at the destination before the source is touched. No smart timeout on the node-side move: it can be a big file's worth of node-side work, which the upload-sized deadlines would misjudge. */
+    public async move(config: MoveFileConfig): Promise<void> {
+        if (config.fromPath === config.toPath) return;
+        if (config.fromPath === ROUTING_FILE || config.toPath === ROUTING_FILE) {
+            throw new Error(`The routing config ${JSON.stringify(ROUTING_FILE)} cannot be moved`);
+        }
+        let fromRoute = getRoute(config.fromPath);
+        let toRoute = getRoute(config.toPath);
+        let state = await this.getState();
+        let target = state.sources.find(x => configWindowCurrent(x.config) && routeContains(x.config.route, fromRoute));
+        if (target && routeContains(target.config.route, toRoute)) {
+            await this.request({ write: true, route: fromRoute }, async archives => {
+                if (!archives.move) {
+                    throw new Error(`${archives.getDebugName()} does not support move (moving ${JSON.stringify(config.fromPath)} to ${JSON.stringify(config.toPath)})`);
+                }
+                await archives.move(config);
+            });
+            return;
+        }
+        // Inline rather than moveArchiveFile, which - given one archives that implements move - would just call back into this method
+        let copied = await copyArchiveFile({ from: this, to: this, path: config.fromPath, toPath: config.toPath });
+        if (!copied) {
+            throw new Error(`Cannot move ${JSON.stringify(config.fromPath)} to ${JSON.stringify(config.toPath)}: the source file does not exist on ${this.getDebugName()}`);
+        }
+        let confirmed = await this.getInfo(config.toPath);
+        if (!confirmed) {
+            throw new Error(`Not deleting ${JSON.stringify(config.fromPath)} after copying it to ${JSON.stringify(config.toPath)}: the destination does not report it (${this.getDebugName()}) - the file is left at the source`);
+        }
+        await this.del(config.fromPath);
+    }
+
     // One write target per route range, lowest latency first. Within a shard the target is ALWAYS the first source in config order (see runPrimary - writes must stay on the same node): connectivity only decides which SHARD we pick, never which node within it, so a shard whose node is disconnected is dropped entirely when connectedOnly is set.
     private getVariableShardTargets(state: ChainState, config: { connectedOnly: boolean }): SourceWrapper[] {
         let targetsByRoute = new Map<string, SourceWrapper>();
         for (let source of state.sources) {
-            if (!configAcceptsWrites(source.config)) continue;
+            if (!configWindowCurrent(source.config)) continue;
             let routeKey = JSON.stringify(source.config.route || FULL_ROUTE);
             if (!targetsByRoute.has(routeKey)) {
                 targetsByRoute.set(routeKey, source);
@@ -881,7 +903,7 @@ export class ArchivesChain implements IArchives {
                     // The shard picking already retries across shards, so a stuck shard just costs its timeout and we move on. Large data streams via setLargeFile - the key is already materialized here, so the no-VARIABLE_SHARD restriction on setLargeFile doesn't apply.
                     await this.applySmartTimeout({ uploadBytes: data.length, label: `Variable-shard upload of ${JSON.stringify(fullKey)} (${data.length} bytes)` }, target, () => {
                         if (data.length > LARGE_SET_THRESHOLD) {
-                            return target.write(archives => archives.setLargeFile({ path: fullKey, lastModified: config?.lastModified, getNextData: bufferChunkStream(data) }));
+                            return target.write(archives => archives.setLargeFile({ path: fullKey, ...config, ...bufferChunkStream(data) }));
                         }
                         return target.write(archives => archives.set(fullKey, data, config)) as any;
                     });
@@ -909,15 +931,32 @@ export class ArchivesChain implements IArchives {
         }
     }
 
-    public async setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
+    /** A large file is written exactly like a small one - same write node, same wrong-window/route re-resolution, same fallbacks - so a value's SIZE never decides its write semantics (set streams through here past LARGE_SET_THRESHOLD, and a file that grew past it must not suddenly lose the availability its caller asked for). The one difference: every attempt after the first has to rewind the stream, so a config without restartStream gets a single attempt. */
+    public async setLargeFile(config: SetLargeFileConfig): Promise<void> {
         if (config.path.includes(VARIABLE_SHARD) && parseVariableRoute(config.path) === undefined) {
             throw new Error(`setLargeFile does not support VARIABLE_SHARD keys (there is no way to return the materialized key); write the file with set, or materialize the key yourself. Key: ${JSON.stringify(config.path)}`);
         }
         let route = getRoute(config.path);
+        const restartStream = config.restartStream;
+        if (!restartStream) {
+            await this.setLargeFileOnce(config, route);
+            return;
+        }
+        let attempt = 0;
+        await this.request({ write: true, fallbacks: config.fallbacks, route }, async archives => {
+            attempt++;
+            // The previous attempt consumed some (or all) of the stream, and this source needs the file from its first byte
+            if (attempt > 1) await restartStream();
+            await watchSlowPromise(`setLargeFile|${config.path}`, archives.setLargeFile(config));
+        });
+    }
+
+    // A stream that cannot rewind gets exactly ONE attempt at the write node: retrying anywhere (another source, or this one again) would send whatever is left of an already-consumed stream as if it were the whole file.
+    private async setLargeFileOnce(config: SetLargeFileConfig, route: number): Promise<void> {
         let recheckedAvailability = false;
         while (true) {
             let state = await this.getState();
-            let target = state.sources.find(x => configAcceptsWrites(x.config) && routeContains(x.config.route, route));
+            let target = state.sources.find(x => configWindowCurrent(x.config) && routeContains(x.config.route, route));
             if (!target) {
                 if (!recheckedAvailability) {
                     recheckedAvailability = true;

package/storage/remoteStorage/deployTakeover.ts

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import { startIntermediateMaintenance } from "./storageServerState";
+import { startIntermediateMaintenance } from "./intermediateManagement";
 
 const PARAMETERS_TIMELINE_FILE_REGEX = /^(\d+)-parameters\.json$/;
 const DEPLOY_DETECT_RETRY_DELAY = 5 * 1000;