sliftutils 1.7.29 → 1.7.33

package/storage/remoteStorage/storageServerState.ts

@@ -1,5 +1,7 @@
 import path from "path";
 import fs from "fs";
+import { lazy } from "socket-function/src/caching";
+import { runInfinitePoll } from "socket-function/src/batching";
 import { getFileStorageNested2 } from "../FileFolderAPI";
 import { TransactionStorage } from "../TransactionStorage";
 import { JSONStorage } from "../JSONStorage";
@@ -10,32 +12,21 @@ import {
     ArchivesSyncStatus,
     STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
 } from "../IArchives";
-import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
-import { applyDeployRemap, getTakeoverAltPort, getOwnWindowEndClip, onTakeoverEvent } from "./deployTakeover";
+import { ROUTING_FILE, parseRoutingData, serializeRemoteConfig, parseHostedUrl, replaceHostedUrlPort, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
+import { injectIntermediateSource, expireIntermediateSources, getIntermediateSources, findSplitUrl, nextIntermediateVersion, INTERMEDIATE_EXPIRE_GRACE } from "./intermediateSources";
+import { getTakeoverIntermediate } from "./deployTakeover";
 import { createApiArchives } from "./createArchives";
 import type { IStorage } from "../IStorage";
+import { broadcastRoutingChanged } from "./storageController";
 import type { AccessRequest, TrustRecord } from "./storageController";
 import { getArg } from "./cliArgs";
 
-// The storage server's global server-side state. hostStorageServer sets the config once at
-// startup; everything else (system storages, buckets) is a global cache created lazily on first
-// use, throwing if the config was never set.
-//
-// Buckets have no separate registry: each bucket's configuration (a RemoteConfig) lives inside the
-// bucket itself, at ROUTING_FILE. A bucket exists iff that file exists. Writing it creates the
-// bucket (building its store, which may immediately start synchronization scans); overwriting it —
-// directly or via a file pulled in from a synchronization source — rebuilds the store with the new
-// sources (cancelling the old store's scans).
-
 export type StorageServerConfig = {
     domain: string;
     port: number;
     rootDomain: string;
-    // user@externalIp of the storage machine, for generated ssh commands
     sshTarget: string;
-    // Absolute command that runs the grantAccess CLI on the storage machine (admin args appended)
     serverCommand: string;
-    // The server's root storage folder (absolute)
     folder: string;
 };
 
@@ -49,15 +40,10 @@ export function getStorageServerConfig(): StorageServerConfig {
     }
     return config;
 }
-// For self-detection: code that CAN run outside the storage server (createArchives) uses this to
-// check "is this URL our own process?" without throwing.
 export function getStorageServerConfigOptional(): StorageServerConfig | undefined {
     return config;
 }
 
-// When set, all write-path operations (creating files, appending large uploads, creating buckets)
-// throw this message. Reads, findInfo, and deletes still work — so clients can free space. Managed
-// by hostStorageServer's disk-space monitor.
 let writesRejectedReason: string | undefined;
 export function setWritesRejectedReason(reason: string | undefined): void {
     writesRejectedReason = reason;
@@ -69,12 +55,6 @@ export function assertWritesAllowed(): void {
     if (writesRejectedReason) throw new Error(writesRejectedReason);
 }
 
-// The system storages (trust, requests) are the same kind of thing — a JSON key/value store under
-// <folder>/system2/<name> — so they share one cache, keyed by name. ("system2" because the layout
-// changed when bucket configs moved into the buckets themselves; old "system"/"buckets" folders
-// are simply ignored.)
-// setTrustedMachines is called by consumers BEFORE starting the server, so when the config isn't
-// set yet we fall back to the same --folder arg the CLI starts the server with.
 function getStorageFolder(): string {
     let config = getStorageServerConfigOptional();
     if (config) return config.folder;
@@ -106,7 +86,95 @@ export function getRequests(): Promise<IStorage<AccessRequest[]>> {
     return getSystemStorage<AccessRequest[]>("requests");
 }
 
-/** Makes machineIds the complete trust list for the account: machines not in the list lose access, machines already trusted keep their existing record, and missing ones are added. */
+export type BucketWriteStats = {
+    /** Every set call the bucket accepted */
+    originalWrites: number;
+    originalBytes: number;
+    /** What actually reached the sources. Fast writes coalesce repeated writes to the same key, so this is lower than the original counts (and is what the disk actually did). */
+    flushedWrites: number;
+    flushedBytes: number;
+};
+function emptyWriteStats(): BucketWriteStats {
+    return { originalWrites: 0, originalBytes: 0, flushedWrites: 0, flushedBytes: 0 };
+}
+function addWriteStats(a: BucketWriteStats, b: BucketWriteStats): BucketWriteStats {
+    return {
+        originalWrites: a.originalWrites + b.originalWrites,
+        originalBytes: a.originalBytes + b.originalBytes,
+        flushedWrites: a.flushedWrites + b.flushedWrites,
+        flushedBytes: a.flushedBytes + b.flushedBytes,
+    };
+}
+function getWriteStatsStorage(): Promise<IStorage<BucketWriteStats>> {
+    return getSystemStorage<BucketWriteStats>("writeStats");
+}
+
+// Counted in memory as deltas since the last flush, so counting a write never touches the disk. The flush reads the stored totals and adds the delta, which also makes it correct across restarts.
+const writeStatDeltas = new Map<string, BucketWriteStats>();
+
+async function flushWriteStats(): Promise<void> {
+    if (!writeStatDeltas.size) return;
+    let pending = [...writeStatDeltas];
+    writeStatDeltas.clear();
+    let storage = await getWriteStatsStorage();
+    for (let [key, delta] of pending) {
+        try {
+            await storage.set(key, addWriteStats(await storage.get(key) || emptyWriteStats(), delta));
+        } catch (e) {
+            // Put the delta back, so a failed flush loses nothing
+            writeStatDeltas.set(key, addWriteStats(writeStatDeltas.get(key) || emptyWriteStats(), delta));
+            console.error(`Flushing write stats for bucket ${key} failed: ${(e as Error).stack ?? e}`);
+        }
+    }
+}
+
+const startWriteStatsFlushing = lazy(() => {
+    runInfinitePoll(WRITE_STATS_FLUSH_INTERVAL, flushWriteStats);
+});
+
+function countBucketWrite(key: string, kind: "original" | "flushed", bytes: number): void {
+    let delta = writeStatDeltas.get(key);
+    if (!delta) {
+        delta = emptyWriteStats();
+        writeStatDeltas.set(key, delta);
+    }
+    if (kind === "original") {
+        delta.originalWrites++;
+        delta.originalBytes += bytes;
+    } else {
+        delta.flushedWrites++;
+        delta.flushedBytes += bytes;
+    }
+    startWriteStatsFlushing();
+}
+
+async function getBucketWriteStats(key: string): Promise<BucketWriteStats> {
+    let storage = await getWriteStatsStorage();
+    let stored = await storage.get(key) || emptyWriteStats();
+    let delta = writeStatDeltas.get(key);
+    return delta && addWriteStats(stored, delta) || stored;
+}
+
+/** Zeroes the write statistics of every bucket in the account, including counts not yet flushed. */
+export async function clearAccountWriteStats(account: string): Promise<number> {
+    let storage = await getWriteStatsStorage();
+    let prefix = `${account}/`;
+    let cleared = 0;
+    for (let key of await storage.getKeys()) {
+        if (!key.startsWith(prefix)) continue;
+        writeStatDeltas.delete(key);
+        await storage.remove(key);
+        cleared++;
+    }
+    for (let key of [...writeStatDeltas.keys()]) {
+        if (!key.startsWith(prefix)) continue;
+        writeStatDeltas.delete(key);
+        cleared++;
+    }
+    console.log(`Cleared the write statistics of ${cleared} buckets in account ${account}`);
+    return cleared;
+}
+
 export async function setTrustedMachines(config: { account: string; machineIds: string[] }): Promise<void> {
     let trust = await getTrust();
     let prefix = `${config.account}|`;
@@ -127,70 +195,49 @@ export async function setTrustedMachines(config: { account: string; machineIds:
     }
 }
 
-// ── buckets ──
-
 export type LoadedBucket = {
     account: string;
     bucketName: string;
-    // The bucket's parsed routing config (the contents of ROUTING_FILE)
     routing: RemoteConfig;
-    // JSON of routing, for change detection
     routingJSON: string;
-    // ALL of our own entries in routing.sources (this bucket on this server). More than one when
-    // the bucket's options change over time - one entry per valid window (e.g. immutable dropped,
-    // or fast enabled, starting at a scheduled instant).
     selfEntries: HostedConfig[];
-    // The entry valid when the store was built - the structural options (rawDisk, which entries
-    // are downstream). Per-write options (fast/immutable) are instead evaluated at each write's
-    // time, see getWriteConfig/assertMutable. undefined when the config doesn't mention us.
     self: HostedConfig | undefined;
     store: IBucketStore;
-    // The store plan minus its valid windows (see StorePlan.structureKey): when only windows
-    // change, the store is updated in place instead of rebuilt
     structureKey: string;
 };
 
 const buckets = new Map<string, Promise<LoadedBucket | undefined>>();
 
-// setTimeout cannot represent longer delays; more distant window boundaries re-check after this
 const MAX_REBUILD_TIMER_DELAY = 2 ** 31 - 1;
-// Rebuild slightly after the boundary, so the newly-valid entry is unambiguously active
 const REBUILD_BOUNDARY_BUFFER = 1000;
+const WRITE_STATS_FLUSH_INTERVAL = 5 * 60 * 1000;
 
 function getBucketFolder(account: string, bucketName: string): string {
     return path.join(getStorageServerConfig().folder, "buckets2", account, bucketName);
 }
 
-// Ports we are listening on beyond the main config port - a deploy-takeover successor serves on a
-// temporary alternate port, and must recognize alternate-port source URLs as itself
 const extraListenPorts = new Set<number>();
 export function addExtraListenPort(port: number): void {
     extraListenPorts.add(port);
 }
+export function removeExtraListenPort(port: number): void {
+    extraListenPorts.delete(port);
+}
 function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[] {
     let { domain, port } = getStorageServerConfig();
     let indexes: number[] = [];
-    // The takeover's alternate port is the OTHER process of the switchover, on OUR machine with
-    // OUR disk - self for sync purposes on both sides (the old node must not scan/push its own
-    // successor; the shared folder plus the switchover disk scans reconcile them)
-    let takeoverAltPort = getTakeoverAltPort();
     for (let i = 0; i < routing.sources.length; i++) {
         let source = routing.sources[i];
         if (typeof source === "string" || source.type !== "remote") continue;
         let parsed = parseHostedUrl(source.url);
         if (parsed.address !== domain || parsed.account !== account || parsed.bucketName !== bucketName) continue;
-        if (parsed.port === port || extraListenPorts.has(parsed.port) || parsed.port === takeoverAltPort) {
+        if (parsed.port === port || extraListenPorts.has(parsed.port)) {
             indexes.push(i);
         }
     }
     return indexes;
 }
 
-// The entry whose valid window contains the time (among the entries covering the key's route,
-// when one is given - a server can hold different shards with different options). Half-open
-// ([start, end)), so clean breaks resolve unambiguously. Falls back to the nearest window when
-// none contains the time - gaps shouldn't exist, but a write must still resolve to SOME
-// configuration.
 function selectEntryAt(entries: HostedConfig[], time: number, route?: number): HostedConfig | undefined {
     if (route !== undefined) {
         let covering = entries.filter(x => routeContains(x.route, route));
@@ -212,9 +259,6 @@ function selectEntryAt(entries: HostedConfig[], time: number, route?: number): H
     return best;
 }
 
-// Bucket options can change over time (multiple self entries with different valid windows), and
-// the structural ones (rawDisk, the downstream list) are baked into the store - so the store is
-// rebuilt shortly after each of our window boundaries passes, making the newly-valid entry apply.
 function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
     let now = Date.now();
     let boundaries = loaded.selfEntries.flatMap(x => x.validWindow).filter(t => t > now);
@@ -223,7 +267,6 @@ function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
     let key = `${loaded.account}/${loaded.bucketName}`;
     let timer = setTimeout(() => {
         void (async () => {
-            // A routing change may have already replaced this store; only IT gets a boundary timer
             let current = await buckets.get(key);
             if (current !== loaded) return;
             if (Date.now() < nextBoundary) {
@@ -236,19 +279,10 @@ function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
     (timer as { unref?: () => void }).unref?.();
 }
 
-// Extra scans around each of OUR valid-window starts (a deploy switchover is just this too - its
-// remap adds valid windows), catching last-moment writes the previous window's owner accepted near
-// the boundary. By the final scan the previous owner has flushed everything (fast writes never
-// delay past a window's end), so nothing is missed.
 const BOUNDARY_SCAN_OFFSETS = [-30 * 1000, 2 * 1000, 30 * 1000];
-// How far back a boundary scan asks the previous owner for changes: anything older has already
-// arrived through normal synchronization, and this covers the longest a write can sit unflushed
 const BOUNDARY_SCAN_LOOKBACK = DEFAULT_FAST_WRITE_DELAY + WINDOW_END_FLUSH_MARGIN;
-// The catch-up scan for a window that was already active when we built the store
 const STARTUP_BOUNDARY_SCAN_DELAY = 30 * 1000;
 
-// Each (bucket, boundary, offset) is scheduled exactly once, surviving store rebuilds - the timer
-// re-resolves the bucket (and recomputes owners from the then-current routing) when it fires
 const scheduledBoundaryScans = new Set<string>();
 const boundaryScansRunning = new Set<string>();
 
@@ -262,10 +296,6 @@ function scheduleBoundaryScans(loaded: LoadedBucket): void {
             starts.add(start);
         }
     }
-    // A window we are ALREADY inside when the store is built (startup, or a config that made us
-    // valid immediately) got none of the lead-up scans - so one runs shortly after, in case we
-    // came up just as the previous owner is realizing it must shut down. Only recent boundaries
-    // qualify: older trailing writes have long since propagated through normal synchronization.
     for (let self of loaded.selfEntries) {
         let [start, end] = self.validWindow;
         if (!(start <= now && now < end) || start <= 0) continue;
@@ -287,8 +317,6 @@ function scheduleBoundaryScans(loaded: LoadedBucket): void {
             scheduledBoundaryScans.add(scheduleKey);
             let arm = () => {
                 let timer = setTimeout(() => {
-                    // Timer length is capped (far-future boundaries), so re-arm until the time
-                    // actually arrives
                     if (Date.now() < at) {
                         arm();
                         return;
@@ -305,7 +333,6 @@ function scheduleBoundaryScans(loaded: LoadedBucket): void {
 async function runBoundaryScan(bucketKey: string, windowStart: number, offset: number, offsetLabel?: string): Promise<void> {
     let label = `bucket ${bucketKey}, window starting ${new Date(windowStart).toISOString()}, offset ${offsetLabel || `${offset / 1000}s`}`;
     if (boundaryScansRunning.has(bucketKey)) {
-        // Boundary scans never run in parallel; a slow one simply swallows later scheduled points
         console.log(`Skipping boundary scan (${label}): the previous boundary scan is still running`);
         return;
     }
@@ -315,23 +342,17 @@ async function runBoundaryScan(bucketKey: string, windowStart: number, offset: n
         if (!loaded) return;
         let store = loaded.store;
         if (!(store instanceof BlobStore)) return;
-        // Owners are recomputed at fire time - the routing (or the takeover remap) may have
-        // changed since this scan was scheduled
-        let effective = applyDeployRemap(loaded.routing);
+        let effective = loaded.routing;
         let selfIndexes = findSelfIndexes(effective, loaded.account, loaded.bucketName);
         let selfIndexSet = new Set(selfIndexes);
         let selves = selfIndexes.map(i => effective.sources[i] as HostedConfig).filter(x => x.validWindow[0] === windowStart);
         if (!selves.length) return;
         let prevTime = windowStart - 1;
         let needDiskScan = false;
-        // sourceIndex -> the slice of our route that source owned in the previous window
         let remoteOwners = new Map<number, [number, number]>();
         for (let self of selves) {
             let selfRoute = self.route || FULL_ROUTE;
             let idx = effective.sources.indexOf(self);
-            // These scans only run on the write node - the first source valid at the new window's
-            // start for our route. An earlier entry fully covering our route means writes go there,
-            // and IT does the boundary scans.
             let shadowed = false;
             for (let j = 0; j < idx; j++) {
                 let other = effective.sources[j];
@@ -345,8 +366,6 @@ async function runBoundaryScan(bucketKey: string, windowStart: number, offset: n
                 }
             }
             if (shadowed) continue;
-            // The previous window's owners: walking the config in order (write-target order),
-            // each source claims whatever slice of our route no earlier source already owns
             let uncovered: [number, number][] = [[selfRoute[0], selfRoute[1]]];
             for (let j = 0; j < effective.sources.length && uncovered.length; j++) {
                 let other = effective.sources[j];
@@ -369,8 +388,6 @@ async function runBoundaryScan(bucketKey: string, windowStart: number, offset: n
                 uncovered = remaining;
                 if (!claimed) continue;
                 if (selfIndexSet.has(j)) {
-                    // The previous owner shares our storage (same machine + bucket, e.g. the other
-                    // process of a deploy switchover), so a disk rescan sees its writes
                     needDiskScan = true;
                 } else {
                     let existing = remoteOwners.get(j);
@@ -399,57 +416,27 @@ async function runBoundaryScan(bucketKey: string, windowStart: number, offset: n
 }
 
 type StorePlan = {
-    effective: RemoteConfig;
     selfEntries: HostedConfig[];
     self: HostedConfig | undefined;
     rawDisk: boolean;
-    // The BlobStore sources this plan builds: [0] is the disk, the rest are downstream configs
     sourceSpecs: { sourceConfig?: HostedConfig | BackblazeConfig; validWindow: [number, number]; route?: [number, number]; noFullSync?: boolean }[];
     readerDiskLimit?: number;
-    // Everything about the plan EXCEPT the valid windows. Two plans with equal structure keys
-    // differ only in windows, which the running store applies in place - it must survive the
-    // routine config evolution of reducing the last forever-window and appending a new entry,
-    // instead of being disposed and rebuilt (losing sync state and rescanning everything).
     structureKey: string;
 };
 
-// The stable identity of a source endpoint: its config with the in-place-updatable parts
-// (windows, routes) stripped, so a source is recognized across config changes
 function sourceIdentity(sourceConfig: HostedConfig | BackblazeConfig | undefined): string {
     if (!sourceConfig) return "disk";
     return JSON.stringify({ ...sourceConfig, validWindow: undefined, route: undefined });
 }
 
 function computeStorePlan(account: string, bucketName: string, routing: RemoteConfig): StorePlan {
-    // The deploy-takeover remap is an INTERPRETATION overlay: everything behavioral (self entries,
-    // windows, downstream sources) uses the remapped view, while loaded.routing/routingJSON stay
-    // the STORED config - so version guards, change detection, and synchronization never see (or
-    // persist) the remap
-    let effective = applyDeployRemap(routing);
-    let selfIndexes = findSelfIndexes(effective, account, bucketName);
-    let selfEntries = selfIndexes.map(i => effective.sources[i] as HostedConfig);
+    let selfIndexes = findSelfIndexes(routing, account, bucketName);
+    let selfEntries = selfIndexes.map(i => routing.sources[i] as HostedConfig);
     let self = selectEntryAt(selfEntries, Date.now());
     let selfIndex = -1;
     if (self) {
-        selfIndex = effective.sources.indexOf(self);
-    }
-    // Our own disk is the base source; only the routing entries DOWNSTREAM from us (after our
-    // currently-valid entry) become synchronization sources. Upstream sources sync from us, so
-    // writing to or scanning them would just echo our own data back (and make our availability
-    // depend on theirs). A server NOT in the list has been removed from the bucket: it keeps
-    // its disk data (the config may re-add it), but stops all synchronization - no one
-    // contacts a removed source, and scanning/pushing as if we were still one would fight the
-    // real chain. The config-level validWindow rides on each ArchivesSource; the disk source
-    // shares our currently-valid entry's window (it holds our copy of the data).
-    // Our own disk gets no route filter: everything it holds is ours to index and serve.
-    // Being removed from the config entirely is a valid window of NOTHING ([0, 0]): fast
-    // writes flush immediately (nothing may sit in memory on a node that's been cut out),
-    // while the disk data and index stay served.
-    // Internally every self entry is the SAME store - one process listening on one or more ports
-    // (a takeover's alternate-port middle window included). So the disk window merges all
-    // contiguous own windows: an options boundary or the port split must never force a pointless
-    // internal flush between two windows that are both us. Only the DYING process of a takeover
-    // clips its end - from the write handoff on, the data belongs to the successor process.
+        selfIndex = routing.sources.indexOf(self);
+    }
     let diskWindow: [number, number] = [0, 0];
     if (self) {
         let [start, end] = self.validWindow;
@@ -466,10 +453,6 @@ function computeStorePlan(account: string, bucketName: string, routing: RemoteCo
                 }
             }
         }
-        let clip = getOwnWindowEndClip();
-        if (clip !== undefined && clip < end) {
-            end = clip;
-        }
         diskWindow = [start, end];
     }
     let ownIndexes = new Set(selfIndexes);
@@ -477,27 +460,20 @@ function computeStorePlan(account: string, bucketName: string, routing: RemoteCo
         validWindow: diskWindow,
     }];
     if (selfIndex !== -1) {
-        for (let i = selfIndex + 1; i < effective.sources.length; i++) {
-            let source = effective.sources[i];
+        for (let i = selfIndex + 1; i < routing.sources.length; i++) {
+            let source = routing.sources[i];
             if (typeof source === "string" || ownIndexes.has(i)) continue;
-            // Disjoint-shard sources never talk to each other; a partial overlap syncs only
-            // the intersection (scans ignore the rest, writes only send matching keys)
             let sharedRoute = routeIntersection(self?.route, source.route);
             if (!sharedRoute) continue;
-            // A bounded-cache server (noFullSync on our own entry) must not full-sync from
-            // ANY source, or its disk would fill regardless of the limit
             sourceSpecs.push({ sourceConfig: source, validWindow: source.validWindow, route: sharedRoute, noFullSync: source.noFullSync || self?.noFullSync });
         }
     }
     let rawDisk = !!self?.rawDisk;
-    // Only what the running store genuinely cannot change in place: the store TYPE (rawDisk) and
-    // whether the disk-limit poll runs. Source additions/removals and window/route changes are all
-    // applied live via updateSources - a config change must never destroy the running store.
     let structureKey = JSON.stringify({
         rawDisk,
         readerDiskLimit: self?.readerDiskLimit,
     });
-    return { effective, selfEntries, self, rawDisk, sourceSpecs, readerDiskLimit: self?.readerDiskLimit, structureKey };
+    return { selfEntries, self, rawDisk, sourceSpecs, readerDiskLimit: self?.readerDiskLimit, structureKey };
 }
 
 function buildBucket(account: string, bucketName: string, routing: RemoteConfig, plan?: StorePlan): LoadedBucket {
@@ -522,12 +498,11 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig,
         }));
         store = new BlobStore(folder, sources, {
             onIndexChanged: key => {
-                // Fires for our own routing writes AND routing files pulled in via synchronization
-                // (only ever newer ones, see IArchives.set) — either way, apply the new config
                 if (key !== ROUTING_FILE) return;
                 void scheduleRoutingReload(account, bucketName);
             },
             readerDiskLimit: plan.readerDiskLimit,
+            onWriteCounted: (kind, bytes) => countBucketWrite(`${account}/${bucketName}`, kind, bytes),
         });
     }
     let loaded: LoadedBucket = { account, bucketName, routing, routingJSON: JSON.stringify(routing), selfEntries, self, store, structureKey: plan.structureKey };
@@ -536,10 +511,29 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig,
     return loaded;
 }
 
-async function loadBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined> {
-    let data = await new ArchivesDisk(getBucketFolder(account, bucketName)).get(ROUTING_FILE);
+/** The routing file is ours, on our own disk, at a path we know - so it is read directly. Going through an ArchivesDisk would construct a whole store (handle cache sweep loop, uploads-folder cleanup) just to read one file. */
+function getRoutingFilePath(folder: string): string {
+    return path.join(folder, "files", ROUTING_FILE);
+}
+async function readRoutingFile(folder: string): Promise<Buffer | undefined> {
+    try {
+        return await fs.promises.readFile(getRoutingFilePath(folder));
+    } catch (e) {
+        if ((e as { code?: string }).code === "ENOENT") return undefined;
+        throw e;
+    }
+}
+
+async function readRoutingFromDisk(account: string, bucketName: string): Promise<RemoteConfig | undefined> {
+    let data = await readRoutingFile(getBucketFolder(account, bucketName));
     if (!data) return undefined;
-    return buildBucket(account, bucketName, parseRoutingData(data));
+    return parseRoutingData(data);
+}
+
+async function loadBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined> {
+    let routing = await readRoutingFromDisk(account, bucketName);
+    if (!routing) return undefined;
+    return buildBucket(account, bucketName, routing);
 }
 
 export function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined> {
@@ -548,8 +542,6 @@ export function getLoadedBucket(account: string, bucketName: string): Promise<Lo
     if (!loaded) {
         loaded = loadBucket(account, bucketName);
         buckets.set(key, loaded);
-        // Missing buckets are not cached — they can be created at any time (guarding against a
-        // concurrent create having already replaced our entry)
         void loaded.then(bucket => {
             if (!bucket && buckets.get(key) === loaded) {
                 buckets.delete(key);
@@ -563,20 +555,7 @@ export function getLoadedBucket(account: string, bucketName: string): Promise<Lo
     return loaded;
 }
 
-// Called whenever any bucket's routing config is applied (created, updated in place, or rebuilt).
-// storageController wires this to its connected-client broadcast, so every client is pushed the
-// change immediately - never waiting for a poll. (A callback, not an import - storageController
-// imports us, so importing it back would be a cycle.)
-let routingChangedBroadcaster: (() => void) | undefined;
-export function setRoutingChangedBroadcaster(broadcaster: () => void): void {
-    routingChangedBroadcaster = broadcaster;
-}
-function notifyRoutingChanged(): void {
-    routingChangedBroadcaster?.();
-}
 
-// Routing reloads are serialized per bucket, so concurrent writes/syncs can't rebuild the same
-// store twice in parallel
 const routingReloads = new Map<string, Promise<void>>();
 function scheduleRoutingReload(account: string, bucketName: string, config?: { force?: boolean; reason?: string }): Promise<void> {
     let key = `${account}/${bucketName}`;
@@ -591,18 +570,11 @@ async function checkRoutingChanged(account: string, bucketName: string, config?:
     let key = `${account}/${bucketName}`;
     let loaded = await buckets.get(key);
     if (!loaded) return;
-    // The routing config is only ever read off our own disk - it is never pulled from another
-    // source (it reaches us exclusively as a version-validated write, which lands on disk first)
-    let data = await new ArchivesDisk(getBucketFolder(account, bucketName)).get(ROUTING_FILE);
-    if (!data) return;
-    let routing = parseRoutingData(data);
+    let routing = await readRoutingFromDisk(account, bucketName);
+    if (!routing) return;
     if (!config?.force && JSON.stringify(routing) === loaded.routingJSON) return;
     let reason = config?.force && (config.reason || "forced") || "routing config changed";
     let plan = computeStorePlan(account, bucketName, routing);
-    // Config changes are applied to the RUNNING store in place - windows/routes mutate, added
-    // sources start syncing, removed sources' slots go dead - so the index, pending fast writes
-    // (re-capped to the new deadline), and unaffected sync loops all survive. The store is only
-    // destroyed for what it structurally cannot express (a rawDisk flip).
     if (plan.structureKey === loaded.structureKey && loaded.store instanceof BlobStore) {
         console.log(`Applying the routing config to the running store for bucket ${key} (${reason})`);
         loaded.store.updateSources(plan.sourceSpecs.map(spec => {
@@ -622,7 +594,7 @@ async function checkRoutingChanged(account: string, bucketName: string, config?:
         buckets.set(key, Promise.resolve(updated));
         scheduleWindowBoundaryRebuild(updated);
         scheduleBoundaryScans(updated);
-        notifyRoutingChanged();
+        broadcastRoutingChanged();
         return;
     }
     console.log(`Rebuilding the store for bucket ${key} (${reason})`);
@@ -630,12 +602,9 @@ async function checkRoutingChanged(account: string, bucketName: string, config?:
         await loaded.store.dispose();
     }
     buckets.set(key, Promise.resolve(buildBucket(account, bucketName, routing, plan)));
-    notifyRoutingChanged();
+    broadcastRoutingChanged();
 }
 
-// Per-write options are evaluated at the WRITE's time (and the key's route), not the current
-// config: the write time is stamped exactly once (at the first node that receives the write) and
-// propagates with the data, so every node resolves the same self entry - and the same options.
 function getWriteConfig(bucket: LoadedBucket, writeTime: number, route: number): { fast?: boolean; writeDelay?: number } {
     let self = selectEntryAt(bucket.selfEntries, writeTime, route);
     return { fast: self?.fast, writeDelay: self?.writeDelay };
@@ -649,8 +618,6 @@ export async function assertMutable(bucket: LoadedBucket, filePath: string, writ
     }
 }
 
-// Wrong-target rejections are rare and important (they ARE the switchover handoff), but a burst of
-// racing writes at a boundary must not flood the log - so at most one line a minute
 const WRONG_TARGET_LOG_THROTTLE = 60 * 1000;
 let lastWrongTargetLog = 0;
 function logWrongTargetRejection(message: string): void {
@@ -659,32 +626,39 @@ function logWrongTargetRejection(message: string): void {
     console.log(message);
 }
 
-// Routing writes are serialized per bucket, so two concurrent creates can't both build a store
-// (the loser's store would leak with its sync loops running forever)
 const routingWrites = new Map<string, Promise<void>>();
 
 async function writeRoutingConfig(account: string, bucketName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
     let key = `${account}/${bucketName}`;
-    // Validates before storing anything, so a bad config can't brick the bucket
     let incoming = parseRoutingData(data);
     let loaded = await getLoadedBucket(account, bucketName);
     if (!loaded) {
         await new ArchivesDisk(getBucketFolder(account, bucketName)).set(ROUTING_FILE, data, { lastModified: config?.lastModified });
         buckets.set(key, Promise.resolve(buildBucket(account, bucketName, incoming)));
-        notifyRoutingChanged();
+        broadcastRoutingChanged();
         console.log(`Created bucket ${key}`);
         return;
     }
-    // A changed config must carry a higher version than the one currently stored — otherwise two
-    // clients with different configs at the same version would keep clobbering each other. Same
-    // content at any version is a harmless no-op and is allowed through.
     if (JSON.stringify(incoming) !== loaded.routingJSON && getConfigVersion(incoming) <= getConfigVersion(loaded.routing)) {
-        throw new Error(`Cannot update routing config for bucket ${key}: the new config differs from the current one but its version (${getConfigVersion(incoming)}) is not greater than the current version (${getConfigVersion(loaded.routing)}). Increment the version to update it. Current: ${loaded.routingJSON}. Attempted: ${JSON.stringify(incoming)}`);
+        let message = `Cannot update routing config for bucket ${key}: the new config differs from the current one but its version (${getConfigVersion(incoming)}) is not greater than the current version (${getConfigVersion(loaded.routing)}). Increment the version to update it. Current: ${loaded.routingJSON}. Attempted: ${JSON.stringify(incoming)}`;
+        console.error(message);
+        throw new Error(message);
+    }
+    let stored = incoming;
+    let reinjected = 0;
+    for (let intermediate of getIntermediateSources(expireIntermediateSources(loaded.routing, Date.now()))) {
+        stored = injectIntermediateSource(stored, {
+            splitUrl: findSplitUrl(loaded.routing, intermediate) || intermediate.url,
+            intermediateUrl: intermediate.url,
+            start: intermediate.validWindow[0],
+            end: intermediate.validWindow[1],
+        });
+        reinjected++;
     }
-    // Routing writes bypass fast mode (the config must apply immediately and survive a crash)
-    // and ignore immutable (the routing file must always stay updatable). An older lastModified
-    // no-ops inside set, in which case the reload below sees no change.
-    await loaded.store.set(ROUTING_FILE, data, { lastModified: config?.lastModified });
+    if (JSON.stringify(stored) !== JSON.stringify(incoming)) {
+        console.log(`Re-injected ${reinjected} in-flight switchover windows into the incoming routing config for bucket ${key} (version ${getConfigVersion(incoming)})`);
+    }
+    await loaded.store.set(ROUTING_FILE, Buffer.from(serializeRemoteConfig(stored)), { lastModified: config?.lastModified });
     await scheduleRoutingReload(account, bucketName);
 }
 
@@ -693,8 +667,6 @@ export async function writeBucketFile(account: string, bucketName: string, fileP
     if (filePath === ROUTING_FILE) {
         let key = `${account}/${bucketName}`;
         let run = (routingWrites.get(key) || Promise.resolve()).then(() => writeRoutingConfig(account, bucketName, data, config));
-        // The chain must survive a failed write, so the stored link swallows the error (the caller
-        // still gets it from run)
         routingWrites.set(key, run.then(() => { }, () => { }));
         return await run;
     }
@@ -702,13 +674,7 @@ export async function writeBucketFile(account: string, bucketName: string, fileP
     if (!loaded) {
         throw new Error(`Bucket ${account}/${bucketName} does not exist. Write its routing config to ${JSON.stringify(ROUTING_FILE)} to create it.`);
     }
-    // The write time is determined exactly once, HERE at the first node that receives the write;
-    // it propagates as lastModified, so every node evaluates the bucket options at the same instant
     let writeTime = config?.lastModified || Date.now();
-    // Only freshly-stamped writes are checked: a write with an explicit lastModified is
-    // synchronization/backfill (old data legitimately lands on sources whose window has moved on),
-    // but a fresh write reaching us outside our windows/routes means the client resolved its
-    // target from a stale view - it must re-resolve and retry at the correct source.
     let route = getRoute(filePath);
     if (!config?.lastModified && loaded.selfEntries.length) {
         let timeValid = loaded.selfEntries.filter(x => writeTime >= x.validWindow[0] && writeTime < x.validWindow[1]);
@@ -729,9 +695,7 @@ export function getBucketConfig(bucket: LoadedBucket): ArchivesConfig {
     let progress = bucket.store.getSyncProgress?.();
     return {
         supportsChangesAfter: !!bucket.store.getChangesAfter,
-        // The remapped (interpretation) view: getConfig is how clients learn of an in-progress
-        // deploy takeover, since the remap is never written into the routing file itself
-        remoteConfig: applyDeployRemap(bucket.routing),
+        remoteConfig: bucket.routing,
         index: progress?.index,
         indexSources: progress?.sources,
         readerDiskLimit: progress?.readerDiskLimit,
@@ -742,30 +706,118 @@ export function getBucketConfig(bucket: LoadedBucket): ArchivesConfig {
 export async function rebuildAllLoadedBuckets(): Promise<void> {
     for (let key of [...buckets.keys()]) {
         let slash = key.indexOf("/");
-        await scheduleRoutingReload(key.slice(0, slash), key.slice(slash + 1), { force: true, reason: "deploy takeover state changed" });
+        await scheduleRoutingReload(key.slice(0, slash), key.slice(slash + 1), { force: true, reason: "all loaded buckets were asked to rebuild" });
     }
 }
 
-onTakeoverEvent(event => {
-    if (event === "remapChanged") {
-        // The rebuild applies the remapped windows, and scheduleBoundaryScans (part of the
-        // rebuild) schedules the handoff scans around the new windows' starts
-        void rebuildAllLoadedBuckets();
+const OWN_CONFIG_WRITE_THROTTLE = 30 * 1000;
+const INTERMEDIATE_MAINTAIN_INTERVAL = 60 * 1000;
+let lastOwnConfigWrite = 0;
+
+async function writeOwnRoutingConfig(loaded: LoadedBucket, updated: RemoteConfig, reason: string): Promise<void> {
+    let key = `${loaded.account}/${loaded.bucketName}`;
+    let sinceLast = Date.now() - lastOwnConfigWrite;
+    if (sinceLast < OWN_CONFIG_WRITE_THROTTLE) {
+        console.log(`Not writing our own routing config update for bucket ${key} (${reason}): the last one was ${sinceLast}ms ago, under the ${OWN_CONFIG_WRITE_THROTTLE}ms throttle. Retrying on the next maintenance pass. Wanted: ${JSON.stringify(updated)}`);
+        return;
     }
+    lastOwnConfigWrite = Date.now();
+    let version = nextIntermediateVersion(getConfigVersion(loaded.routing));
+    let next: RemoteConfig = { ...updated, version };
+    console.log(`Writing our own routing config update for bucket ${key} (${reason}), version ${getConfigVersion(loaded.routing)} -> ${version}: ${JSON.stringify(next)}`);
+    let data = Buffer.from(serializeRemoteConfig(next));
+    await writeBucketFile(loaded.account, loaded.bucketName, ROUTING_FILE, data);
+    await propagateRoutingConfig(loaded, next, data);
+}
+
+async function propagateRoutingConfig(loaded: LoadedBucket, next: RemoteConfig, data: Buffer): Promise<void> {
+    let targets = new Map<string, HostedConfig | BackblazeConfig>();
+    for (let source of [...loaded.routing.sources, ...next.sources]) {
+        if (typeof source === "string" || source.intermediate) continue;
+        if (source.type === "remote") {
+            let parsed = parseHostedUrl(source.url);
+            if (parsed.account !== loaded.account || parsed.bucketName !== loaded.bucketName) continue;
+        }
+        if (targets.has(source.url)) continue;
+        targets.set(source.url, source);
+    }
+    for (let [url, source] of targets) {
+        try {
+            await createApiArchives(source).set(ROUTING_FILE, data);
+            console.log(`Propagated our routing config update to ${url}`);
+        } catch (e) {
+            console.error(`Propagating our routing config update to ${url} failed: ${(e as Error).stack ?? e}`);
+        }
+    }
+}
+
+async function maintainIntermediates(): Promise<void> {
+    let { domain, port } = getStorageServerConfig();
+    let takeover = getTakeoverIntermediate();
+    for (let key of [...buckets.keys()]) {
+        let loadedPromise = buckets.get(key);
+        if (!loadedPromise) continue;
+        let loaded = await loadedPromise.catch(() => undefined);
+        if (!loaded) continue;
+        let expired = expireIntermediateSources(loaded.routing, Date.now());
+        if (JSON.stringify(expired) !== JSON.stringify(loaded.routing)) {
+            await writeOwnRoutingConfig(loaded, expired, `switchover windows expired more than ${INTERMEDIATE_EXPIRE_GRACE / 1000}s ago`);
+            continue;
+        }
+        if (!takeover) continue;
+        let mainUrl = loaded.routing.sources.find(x => {
+            if (typeof x === "string" || x.type !== "remote" || x.intermediate) return false;
+            let parsed = parseHostedUrl(x.url);
+            return parsed.address === domain && parsed.port === port;
+        }) as HostedConfig | undefined;
+        if (!mainUrl) continue;
+        let injected = injectIntermediateSource(loaded.routing, {
+            splitUrl: mainUrl.url,
+            intermediateUrl: replaceHostedUrlPort(mainUrl.url, takeover.altPort),
+            start: takeover.start,
+            end: takeover.end,
+        });
+        if (JSON.stringify(injected) === JSON.stringify(loaded.routing)) continue;
+        await writeOwnRoutingConfig(loaded, injected, `we are a deploy successor: writes route to our alternate port ${takeover.altPort} from ${new Date(takeover.start).toISOString()} until our predecessor is killed at ${new Date(takeover.end).toISOString()}`);
+    }
+}
+
+/** Started by deployTakeover once we are actually a deploy successor listening on an alternate port. Until then there are no switchover windows to write or expire, so nothing polls. */
+export const startIntermediateMaintenance = lazy(() => {
+    void maintainIntermediates().catch((e: Error) => console.error(`Maintaining switchover routing windows failed: ${e.stack ?? e}`));
+    runInfinitePoll(INTERMEDIATE_MAINTAIN_INTERVAL, maintainIntermediates);
 });
 
+export type BucketDiskInfo = {
+    totalBytes: number;
+    freeBytes: number;
+    usedBytes: number;
+};
 export type ServerBucketInfo = {
     bucketName: string;
-    // Loaded in memory (created or accessed since startup), so its synchronization is running
     active: boolean;
+    /** Where the bucket's data lives on this server */
+    folder: string;
+    /** The drive that folder is on. Buckets sharing a drive report the same numbers. */
+    disk?: BucketDiskInfo;
+    diskError?: string;
+    writeStats?: BucketWriteStats;
     config?: ArchivesConfig;
     error?: string;
 };
 
-/** Every bucket the account has on this server, active or not, each with its configuration.
- *  Inactive buckets are inspected straight from disk WITHOUT loading them - loading would start
- *  their synchronization, and old invalid buckets must stay inert (their parse error is reported
- *  instead). */
+async function getDiskInfo(folder: string): Promise<BucketDiskInfo> {
+    let stats = await fs.promises.statfs(folder);
+    let blockSize = Number(stats.bsize);
+    let totalBytes = Number(stats.blocks) * blockSize;
+    return {
+        totalBytes,
+        // Matches the server's own low-space check: what an unprivileged process can actually use
+        freeBytes: Number(stats.bavail) * blockSize,
+        usedBytes: totalBytes - Number(stats.bfree) * blockSize,
+    };
+}
+
 export async function listAccountBuckets(account: string): Promise<ServerBucketInfo[]> {
     let accountFolder = path.join(getStorageServerConfig().folder, "buckets2", account);
     let names: string[];
@@ -774,33 +826,40 @@ export async function listAccountBuckets(account: string): Promise<ServerBucketI
     } catch {
         return [];
     }
-    let results: ServerBucketInfo[] = [];
-    for (let bucketName of names) {
-        let loadedPromise = buckets.get(`${account}/${bucketName}`);
+    return await Promise.all(names.map(async bucketName => {
+        let key = `${account}/${bucketName}`;
+        let folder = getBucketFolder(account, bucketName);
+        let base: ServerBucketInfo = { bucketName, active: false, folder };
+        let [disk, writeStats] = await Promise.all([
+            getDiskInfo(folder).catch((e: Error) => {
+                base.diskError = String(e.stack ?? e).slice(0, 500);
+                return undefined;
+            }),
+            getBucketWriteStats(key),
+        ]);
+        base.disk = disk;
+        base.writeStats = writeStats;
+        let loadedPromise = buckets.get(key);
         if (loadedPromise) {
             try {
                 let loaded = await loadedPromise;
                 if (loaded) {
-                    results.push({ bucketName, active: true, config: getBucketConfig(loaded) });
-                    continue;
+                    return { ...base, active: true, config: getBucketConfig(loaded) };
                 }
             } catch (e) {
-                results.push({ bucketName, active: true, error: String((e as Error).stack ?? e).slice(0, 500) });
-                continue;
+                return { ...base, active: true, error: String((e as Error).stack ?? e).slice(0, 500) };
             }
         }
         try {
-            let data = await new ArchivesDisk(getBucketFolder(account, bucketName)).get(ROUTING_FILE);
+            let data = await readRoutingFile(folder);
             if (!data) {
-                results.push({ bucketName, active: false, error: `No routing file (${ROUTING_FILE})` });
-                continue;
+                return { ...base, error: `No routing file (${ROUTING_FILE})` };
             }
-            results.push({ bucketName, active: false, config: { remoteConfig: parseRoutingData(data) } });
+            return { ...base, config: { remoteConfig: parseRoutingData(data) } };
         } catch (e) {
-            results.push({ bucketName, active: false, error: String((e as Error).stack ?? e).slice(0, 500) });
+            return { ...base, error: String((e as Error).stack ?? e).slice(0, 500) };
         }
-    }
-    return results;
+    }));
 }
 
 export async function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void> {
@@ -812,12 +871,8 @@ export async function deleteBucketFile(account: string, bucketName: string, file
     await loaded.store.del(filePath, getWriteConfig(loaded, Date.now(), getRoute(filePath)));
 }
 
-// ── local IArchives access ──
-
 const LARGE_FILE_PART_SIZE = 8 * 1024 * 1024;
 
-// The in-process IArchives for a bucket hosted by THIS server — used when a RemoteConfig source
-// URL points at ourselves, so we don't talk to ourselves over HTTPS. One singleton per bucket.
 class ArchivesLocalBucket implements IArchives {
     constructor(private account: string, private bucketName: string) { }
 
@@ -885,7 +940,6 @@ class ArchivesLocalBucket implements IArchives {
     }
     public async getConfig(): Promise<ArchivesConfig> {
         let bucket = await this.getBucket();
-        // Missing buckets say true, matching what they become once created (the default store type)
         if (!bucket) return { supportsChangesAfter: true };
         return getBucketConfig(bucket);
     }