sliftutils 1.7.32 → 1.7.34

package/storage/remoteStorage/storageServerState.ts

@@ -1,5 +1,7 @@
 import path from "path";
 import fs from "fs";
+import { lazy } from "socket-function/src/caching";
+import { runInfinitePoll } from "socket-function/src/batching";
 import { getFileStorageNested2 } from "../FileFolderAPI";
 import { TransactionStorage } from "../TransactionStorage";
 import { JSONStorage } from "../JSONStorage";
@@ -10,33 +12,21 @@ import {
     ArchivesSyncStatus,
     STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
 } from "../IArchives";
-import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
-import { applyDeployRemap, getTakeoverAltPort, getOwnWindowEndClip, onTakeoverEvent } from "./deployTakeover";
+import { ROUTING_FILE, parseRoutingData, serializeRemoteConfig, parseHostedUrl, replaceHostedUrlPort, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
+import { injectIntermediateSource, expireIntermediateSources, getIntermediateSources, findSplitUrl, nextIntermediateVersion, INTERMEDIATE_EXPIRE_GRACE } from "./intermediateSources";
+import { getTakeoverIntermediate } from "./deployTakeover";
 import { createApiArchives } from "./createArchives";
 import type { IStorage } from "../IStorage";
 import { broadcastRoutingChanged } from "./storageController";
 import type { AccessRequest, TrustRecord } from "./storageController";
 import { getArg } from "./cliArgs";
 
-// The storage server's global server-side state. hostStorageServer sets the config once at
-// startup; everything else (system storages, buckets) is a global cache created lazily on first
-// use, throwing if the config was never set.
-//
-// Buckets have no separate registry: each bucket's configuration (a RemoteConfig) lives inside the
-// bucket itself, at ROUTING_FILE. A bucket exists iff that file exists. Writing it creates the
-// bucket (building its store, which may immediately start synchronization scans); overwriting it —
-// directly or via a file pulled in from a synchronization source — rebuilds the store with the new
-// sources (cancelling the old store's scans).
-
 export type StorageServerConfig = {
     domain: string;
     port: number;
     rootDomain: string;
-    // user@externalIp of the storage machine, for generated ssh commands
     sshTarget: string;
-    // Absolute command that runs the grantAccess CLI on the storage machine (admin args appended)
     serverCommand: string;
-    // The server's root storage folder (absolute)
     folder: string;
 };
 
@@ -50,15 +40,10 @@ export function getStorageServerConfig(): StorageServerConfig {
     }
     return config;
 }
-// For self-detection: code that CAN run outside the storage server (createArchives) uses this to
-// check "is this URL our own process?" without throwing.
 export function getStorageServerConfigOptional(): StorageServerConfig | undefined {
     return config;
 }
 
-// When set, all write-path operations (creating files, appending large uploads, creating buckets)
-// throw this message. Reads, findInfo, and deletes still work — so clients can free space. Managed
-// by hostStorageServer's disk-space monitor.
 let writesRejectedReason: string | undefined;
 export function setWritesRejectedReason(reason: string | undefined): void {
     writesRejectedReason = reason;
@@ -70,12 +55,6 @@ export function assertWritesAllowed(): void {
     if (writesRejectedReason) throw new Error(writesRejectedReason);
 }
 
-// The system storages (trust, requests) are the same kind of thing — a JSON key/value store under
-// <folder>/system2/<name> — so they share one cache, keyed by name. ("system2" because the layout
-// changed when bucket configs moved into the buckets themselves; old "system"/"buckets" folders
-// are simply ignored.)
-// setTrustedMachines is called by consumers BEFORE starting the server, so when the config isn't
-// set yet we fall back to the same --folder arg the CLI starts the server with.
 function getStorageFolder(): string {
     let config = getStorageServerConfigOptional();
     if (config) return config.folder;
@@ -107,7 +86,95 @@ export function getRequests(): Promise<IStorage<AccessRequest[]>> {
     return getSystemStorage<AccessRequest[]>("requests");
 }
 
-/** Makes machineIds the complete trust list for the account: machines not in the list lose access, machines already trusted keep their existing record, and missing ones are added. */
+export type BucketWriteStats = {
+    /** Every set call the bucket accepted */
+    originalWrites: number;
+    originalBytes: number;
+    /** What actually reached the sources. Fast writes coalesce repeated writes to the same key, so this is lower than the original counts (and is what the disk actually did). */
+    flushedWrites: number;
+    flushedBytes: number;
+};
+function emptyWriteStats(): BucketWriteStats {
+    return { originalWrites: 0, originalBytes: 0, flushedWrites: 0, flushedBytes: 0 };
+}
+function addWriteStats(a: BucketWriteStats, b: BucketWriteStats): BucketWriteStats {
+    return {
+        originalWrites: a.originalWrites + b.originalWrites,
+        originalBytes: a.originalBytes + b.originalBytes,
+        flushedWrites: a.flushedWrites + b.flushedWrites,
+        flushedBytes: a.flushedBytes + b.flushedBytes,
+    };
+}
+function getWriteStatsStorage(): Promise<IStorage<BucketWriteStats>> {
+    return getSystemStorage<BucketWriteStats>("writeStats");
+}
+
+// Counted in memory as deltas since the last flush, so counting a write never touches the disk. The flush reads the stored totals and adds the delta, which also makes it correct across restarts.
+const writeStatDeltas = new Map<string, BucketWriteStats>();
+
+async function flushWriteStats(): Promise<void> {
+    if (!writeStatDeltas.size) return;
+    let pending = [...writeStatDeltas];
+    writeStatDeltas.clear();
+    let storage = await getWriteStatsStorage();
+    for (let [key, delta] of pending) {
+        try {
+            await storage.set(key, addWriteStats(await storage.get(key) || emptyWriteStats(), delta));
+        } catch (e) {
+            // Put the delta back, so a failed flush loses nothing
+            writeStatDeltas.set(key, addWriteStats(writeStatDeltas.get(key) || emptyWriteStats(), delta));
+            console.error(`Flushing write stats for bucket ${key} failed: ${(e as Error).stack ?? e}`);
+        }
+    }
+}
+
+const startWriteStatsFlushing = lazy(() => {
+    runInfinitePoll(WRITE_STATS_FLUSH_INTERVAL, flushWriteStats);
+});
+
+function countBucketWrite(key: string, kind: "original" | "flushed", bytes: number): void {
+    let delta = writeStatDeltas.get(key);
+    if (!delta) {
+        delta = emptyWriteStats();
+        writeStatDeltas.set(key, delta);
+    }
+    if (kind === "original") {
+        delta.originalWrites++;
+        delta.originalBytes += bytes;
+    } else {
+        delta.flushedWrites++;
+        delta.flushedBytes += bytes;
+    }
+    startWriteStatsFlushing();
+}
+
+async function getBucketWriteStats(key: string): Promise<BucketWriteStats> {
+    let storage = await getWriteStatsStorage();
+    let stored = await storage.get(key) || emptyWriteStats();
+    let delta = writeStatDeltas.get(key);
+    return delta && addWriteStats(stored, delta) || stored;
+}
+
+/** Zeroes the write statistics of every bucket in the account, including counts not yet flushed. */
+export async function clearAccountWriteStats(account: string): Promise<number> {
+    let storage = await getWriteStatsStorage();
+    let prefix = `${account}/`;
+    let cleared = 0;
+    for (let key of await storage.getKeys()) {
+        if (!key.startsWith(prefix)) continue;
+        writeStatDeltas.delete(key);
+        await storage.remove(key);
+        cleared++;
+    }
+    for (let key of [...writeStatDeltas.keys()]) {
+        if (!key.startsWith(prefix)) continue;
+        writeStatDeltas.delete(key);
+        cleared++;
+    }
+    console.log(`Cleared the write statistics of ${cleared} buckets in account ${account}`);
+    return cleared;
+}
+
 export async function setTrustedMachines(config: { account: string; machineIds: string[] }): Promise<void> {
     let trust = await getTrust();
     let prefix = `${config.account}|`;
@@ -128,70 +195,49 @@ export async function setTrustedMachines(config: { account: string; machineIds:
     }
 }
 
-// ── buckets ──
-
 export type LoadedBucket = {
     account: string;
     bucketName: string;
-    // The bucket's parsed routing config (the contents of ROUTING_FILE)
     routing: RemoteConfig;
-    // JSON of routing, for change detection
     routingJSON: string;
-    // ALL of our own entries in routing.sources (this bucket on this server). More than one when
-    // the bucket's options change over time - one entry per valid window (e.g. immutable dropped,
-    // or fast enabled, starting at a scheduled instant).
     selfEntries: HostedConfig[];
-    // The entry valid when the store was built - the structural options (rawDisk, which entries
-    // are downstream). Per-write options (fast/immutable) are instead evaluated at each write's
-    // time, see getWriteConfig/assertMutable. undefined when the config doesn't mention us.
     self: HostedConfig | undefined;
     store: IBucketStore;
-    // The store plan minus its valid windows (see StorePlan.structureKey): when only windows
-    // change, the store is updated in place instead of rebuilt
     structureKey: string;
 };
 
 const buckets = new Map<string, Promise<LoadedBucket | undefined>>();
 
-// setTimeout cannot represent longer delays; more distant window boundaries re-check after this
 const MAX_REBUILD_TIMER_DELAY = 2 ** 31 - 1;
-// Rebuild slightly after the boundary, so the newly-valid entry is unambiguously active
 const REBUILD_BOUNDARY_BUFFER = 1000;
+const WRITE_STATS_FLUSH_INTERVAL = 5 * 60 * 1000;
 
 function getBucketFolder(account: string, bucketName: string): string {
     return path.join(getStorageServerConfig().folder, "buckets2", account, bucketName);
 }
 
-// Ports we are listening on beyond the main config port - a deploy-takeover successor serves on a
-// temporary alternate port, and must recognize alternate-port source URLs as itself
 const extraListenPorts = new Set<number>();
 export function addExtraListenPort(port: number): void {
     extraListenPorts.add(port);
 }
+export function removeExtraListenPort(port: number): void {
+    extraListenPorts.delete(port);
+}
 function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[] {
     let { domain, port } = getStorageServerConfig();
     let indexes: number[] = [];
-    // The takeover's alternate port is the OTHER process of the switchover, on OUR machine with
-    // OUR disk - self for sync purposes on both sides (the old node must not scan/push its own
-    // successor; the shared folder plus the switchover disk scans reconcile them)
-    let takeoverAltPort = getTakeoverAltPort();
     for (let i = 0; i < routing.sources.length; i++) {
         let source = routing.sources[i];
         if (typeof source === "string" || source.type !== "remote") continue;
         let parsed = parseHostedUrl(source.url);
         if (parsed.address !== domain || parsed.account !== account || parsed.bucketName !== bucketName) continue;
-        if (parsed.port === port || extraListenPorts.has(parsed.port) || parsed.port === takeoverAltPort) {
+        if (parsed.port === port || extraListenPorts.has(parsed.port)) {
             indexes.push(i);
         }
     }
     return indexes;
 }
 
-// The entry whose valid window contains the time (among the entries covering the key's route,
-// when one is given - a server can hold different shards with different options). Half-open
-// ([start, end)), so clean breaks resolve unambiguously. Falls back to the nearest window when
-// none contains the time - gaps shouldn't exist, but a write must still resolve to SOME
-// configuration.
 function selectEntryAt(entries: HostedConfig[], time: number, route?: number): HostedConfig | undefined {
     if (route !== undefined) {
         let covering = entries.filter(x => routeContains(x.route, route));
@@ -213,9 +259,6 @@ function selectEntryAt(entries: HostedConfig[], time: number, route?: number): H
     return best;
 }
 
-// Bucket options can change over time (multiple self entries with different valid windows), and
-// the structural ones (rawDisk, the downstream list) are baked into the store - so the store is
-// rebuilt shortly after each of our window boundaries passes, making the newly-valid entry apply.
 function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
     let now = Date.now();
     let boundaries = loaded.selfEntries.flatMap(x => x.validWindow).filter(t => t > now);
@@ -224,7 +267,6 @@ function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
     let key = `${loaded.account}/${loaded.bucketName}`;
     let timer = setTimeout(() => {
         void (async () => {
-            // A routing change may have already replaced this store; only IT gets a boundary timer
             let current = await buckets.get(key);
             if (current !== loaded) return;
             if (Date.now() < nextBoundary) {
@@ -237,19 +279,10 @@ function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
     (timer as { unref?: () => void }).unref?.();
 }
 
-// Extra scans around each of OUR valid-window starts (a deploy switchover is just this too - its
-// remap adds valid windows), catching last-moment writes the previous window's owner accepted near
-// the boundary. By the final scan the previous owner has flushed everything (fast writes never
-// delay past a window's end), so nothing is missed.
 const BOUNDARY_SCAN_OFFSETS = [-30 * 1000, 2 * 1000, 30 * 1000];
-// How far back a boundary scan asks the previous owner for changes: anything older has already
-// arrived through normal synchronization, and this covers the longest a write can sit unflushed
 const BOUNDARY_SCAN_LOOKBACK = DEFAULT_FAST_WRITE_DELAY + WINDOW_END_FLUSH_MARGIN;
-// The catch-up scan for a window that was already active when we built the store
 const STARTUP_BOUNDARY_SCAN_DELAY = 30 * 1000;
 
-// Each (bucket, boundary, offset) is scheduled exactly once, surviving store rebuilds - the timer
-// re-resolves the bucket (and recomputes owners from the then-current routing) when it fires
 const scheduledBoundaryScans = new Set<string>();
 const boundaryScansRunning = new Set<string>();
 
@@ -263,10 +296,6 @@ function scheduleBoundaryScans(loaded: LoadedBucket): void {
             starts.add(start);
         }
     }
-    // A window we are ALREADY inside when the store is built (startup, or a config that made us
-    // valid immediately) got none of the lead-up scans - so one runs shortly after, in case we
-    // came up just as the previous owner is realizing it must shut down. Only recent boundaries
-    // qualify: older trailing writes have long since propagated through normal synchronization.
     for (let self of loaded.selfEntries) {
         let [start, end] = self.validWindow;
         if (!(start <= now && now < end) || start <= 0) continue;
@@ -288,8 +317,6 @@ function scheduleBoundaryScans(loaded: LoadedBucket): void {
             scheduledBoundaryScans.add(scheduleKey);
             let arm = () => {
                 let timer = setTimeout(() => {
-                    // Timer length is capped (far-future boundaries), so re-arm until the time
-                    // actually arrives
                     if (Date.now() < at) {
                         arm();
                         return;
@@ -306,7 +333,6 @@ function scheduleBoundaryScans(loaded: LoadedBucket): void {
 async function runBoundaryScan(bucketKey: string, windowStart: number, offset: number, offsetLabel?: string): Promise<void> {
     let label = `bucket ${bucketKey}, window starting ${new Date(windowStart).toISOString()}, offset ${offsetLabel || `${offset / 1000}s`}`;
     if (boundaryScansRunning.has(bucketKey)) {
-        // Boundary scans never run in parallel; a slow one simply swallows later scheduled points
         console.log(`Skipping boundary scan (${label}): the previous boundary scan is still running`);
         return;
     }
@@ -316,23 +342,17 @@ async function runBoundaryScan(bucketKey: string, windowStart: number, offset: n
         if (!loaded) return;
         let store = loaded.store;
         if (!(store instanceof BlobStore)) return;
-        // Owners are recomputed at fire time - the routing (or the takeover remap) may have
-        // changed since this scan was scheduled
-        let effective = applyDeployRemap(loaded.routing);
+        let effective = loaded.routing;
         let selfIndexes = findSelfIndexes(effective, loaded.account, loaded.bucketName);
         let selfIndexSet = new Set(selfIndexes);
         let selves = selfIndexes.map(i => effective.sources[i] as HostedConfig).filter(x => x.validWindow[0] === windowStart);
         if (!selves.length) return;
         let prevTime = windowStart - 1;
         let needDiskScan = false;
-        // sourceIndex -> the slice of our route that source owned in the previous window
         let remoteOwners = new Map<number, [number, number]>();
         for (let self of selves) {
             let selfRoute = self.route || FULL_ROUTE;
             let idx = effective.sources.indexOf(self);
-            // These scans only run on the write node - the first source valid at the new window's
-            // start for our route. An earlier entry fully covering our route means writes go there,
-            // and IT does the boundary scans.
             let shadowed = false;
             for (let j = 0; j < idx; j++) {
                 let other = effective.sources[j];
@@ -346,8 +366,6 @@ async function runBoundaryScan(bucketKey: string, windowStart: number, offset: n
                 }
             }
             if (shadowed) continue;
-            // The previous window's owners: walking the config in order (write-target order),
-            // each source claims whatever slice of our route no earlier source already owns
             let uncovered: [number, number][] = [[selfRoute[0], selfRoute[1]]];
             for (let j = 0; j < effective.sources.length && uncovered.length; j++) {
                 let other = effective.sources[j];
@@ -370,8 +388,6 @@ async function runBoundaryScan(bucketKey: string, windowStart: number, offset: n
                 uncovered = remaining;
                 if (!claimed) continue;
                 if (selfIndexSet.has(j)) {
-                    // The previous owner shares our storage (same machine + bucket, e.g. the other
-                    // process of a deploy switchover), so a disk rescan sees its writes
                     needDiskScan = true;
                 } else {
                     let existing = remoteOwners.get(j);
@@ -400,57 +416,27 @@ async function runBoundaryScan(bucketKey: string, windowStart: number, offset: n
 }
 
 type StorePlan = {
-    effective: RemoteConfig;
     selfEntries: HostedConfig[];
     self: HostedConfig | undefined;
     rawDisk: boolean;
-    // The BlobStore sources this plan builds: [0] is the disk, the rest are downstream configs
     sourceSpecs: { sourceConfig?: HostedConfig | BackblazeConfig; validWindow: [number, number]; route?: [number, number]; noFullSync?: boolean }[];
     readerDiskLimit?: number;
-    // Everything about the plan EXCEPT the valid windows. Two plans with equal structure keys
-    // differ only in windows, which the running store applies in place - it must survive the
-    // routine config evolution of reducing the last forever-window and appending a new entry,
-    // instead of being disposed and rebuilt (losing sync state and rescanning everything).
     structureKey: string;
 };
 
-// The stable identity of a source endpoint: its config with the in-place-updatable parts
-// (windows, routes) stripped, so a source is recognized across config changes
 function sourceIdentity(sourceConfig: HostedConfig | BackblazeConfig | undefined): string {
     if (!sourceConfig) return "disk";
     return JSON.stringify({ ...sourceConfig, validWindow: undefined, route: undefined });
 }
 
 function computeStorePlan(account: string, bucketName: string, routing: RemoteConfig): StorePlan {
-    // The deploy-takeover remap is an INTERPRETATION overlay: everything behavioral (self entries,
-    // windows, downstream sources) uses the remapped view, while loaded.routing/routingJSON stay
-    // the STORED config - so version guards, change detection, and synchronization never see (or
-    // persist) the remap
-    let effective = applyDeployRemap(routing);
-    let selfIndexes = findSelfIndexes(effective, account, bucketName);
-    let selfEntries = selfIndexes.map(i => effective.sources[i] as HostedConfig);
+    let selfIndexes = findSelfIndexes(routing, account, bucketName);
+    let selfEntries = selfIndexes.map(i => routing.sources[i] as HostedConfig);
     let self = selectEntryAt(selfEntries, Date.now());
     let selfIndex = -1;
     if (self) {
-        selfIndex = effective.sources.indexOf(self);
-    }
-    // Our own disk is the base source; only the routing entries DOWNSTREAM from us (after our
-    // currently-valid entry) become synchronization sources. Upstream sources sync from us, so
-    // writing to or scanning them would just echo our own data back (and make our availability
-    // depend on theirs). A server NOT in the list has been removed from the bucket: it keeps
-    // its disk data (the config may re-add it), but stops all synchronization - no one
-    // contacts a removed source, and scanning/pushing as if we were still one would fight the
-    // real chain. The config-level validWindow rides on each ArchivesSource; the disk source
-    // shares our currently-valid entry's window (it holds our copy of the data).
-    // Our own disk gets no route filter: everything it holds is ours to index and serve.
-    // Being removed from the config entirely is a valid window of NOTHING ([0, 0]): fast
-    // writes flush immediately (nothing may sit in memory on a node that's been cut out),
-    // while the disk data and index stay served.
-    // Internally every self entry is the SAME store - one process listening on one or more ports
-    // (a takeover's alternate-port middle window included). So the disk window merges all
-    // contiguous own windows: an options boundary or the port split must never force a pointless
-    // internal flush between two windows that are both us. Only the DYING process of a takeover
-    // clips its end - from the write handoff on, the data belongs to the successor process.
+        selfIndex = routing.sources.indexOf(self);
+    }
     let diskWindow: [number, number] = [0, 0];
     if (self) {
         let [start, end] = self.validWindow;
@@ -467,10 +453,6 @@ function computeStorePlan(account: string, bucketName: string, routing: RemoteCo
                 }
             }
         }
-        let clip = getOwnWindowEndClip();
-        if (clip !== undefined && clip < end) {
-            end = clip;
-        }
         diskWindow = [start, end];
     }
     let ownIndexes = new Set(selfIndexes);
@@ -478,27 +460,20 @@ function computeStorePlan(account: string, bucketName: string, routing: RemoteCo
         validWindow: diskWindow,
     }];
     if (selfIndex !== -1) {
-        for (let i = selfIndex + 1; i < effective.sources.length; i++) {
-            let source = effective.sources[i];
+        for (let i = selfIndex + 1; i < routing.sources.length; i++) {
+            let source = routing.sources[i];
             if (typeof source === "string" || ownIndexes.has(i)) continue;
-            // Disjoint-shard sources never talk to each other; a partial overlap syncs only
-            // the intersection (scans ignore the rest, writes only send matching keys)
             let sharedRoute = routeIntersection(self?.route, source.route);
             if (!sharedRoute) continue;
-            // A bounded-cache server (noFullSync on our own entry) must not full-sync from
-            // ANY source, or its disk would fill regardless of the limit
             sourceSpecs.push({ sourceConfig: source, validWindow: source.validWindow, route: sharedRoute, noFullSync: source.noFullSync || self?.noFullSync });
         }
     }
     let rawDisk = !!self?.rawDisk;
-    // Only what the running store genuinely cannot change in place: the store TYPE (rawDisk) and
-    // whether the disk-limit poll runs. Source additions/removals and window/route changes are all
-    // applied live via updateSources - a config change must never destroy the running store.
     let structureKey = JSON.stringify({
         rawDisk,
         readerDiskLimit: self?.readerDiskLimit,
     });
-    return { effective, selfEntries, self, rawDisk, sourceSpecs, readerDiskLimit: self?.readerDiskLimit, structureKey };
+    return { selfEntries, self, rawDisk, sourceSpecs, readerDiskLimit: self?.readerDiskLimit, structureKey };
 }
 
 function buildBucket(account: string, bucketName: string, routing: RemoteConfig, plan?: StorePlan): LoadedBucket {
@@ -523,12 +498,11 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig,
         }));
         store = new BlobStore(folder, sources, {
             onIndexChanged: key => {
-                // Fires for our own routing writes AND routing files pulled in via synchronization
-                // (only ever newer ones, see IArchives.set) — either way, apply the new config
                 if (key !== ROUTING_FILE) return;
                 void scheduleRoutingReload(account, bucketName);
             },
             readerDiskLimit: plan.readerDiskLimit,
+            onWriteCounted: (kind, bytes) => countBucketWrite(`${account}/${bucketName}`, kind, bytes),
         });
     }
     let loaded: LoadedBucket = { account, bucketName, routing, routingJSON: JSON.stringify(routing), selfEntries, self, store, structureKey: plan.structureKey };
@@ -537,10 +511,29 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig,
     return loaded;
 }
 
-async function loadBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined> {
-    let data = await new ArchivesDisk(getBucketFolder(account, bucketName)).get(ROUTING_FILE);
+/** The routing file is ours, on our own disk, at a path we know - so it is read directly. Going through an ArchivesDisk would construct a whole store (handle cache sweep loop, uploads-folder cleanup) just to read one file. */
+function getRoutingFilePath(folder: string): string {
+    return path.join(folder, "files", ROUTING_FILE);
+}
+async function readRoutingFile(folder: string): Promise<Buffer | undefined> {
+    try {
+        return await fs.promises.readFile(getRoutingFilePath(folder));
+    } catch (e) {
+        if ((e as { code?: string }).code === "ENOENT") return undefined;
+        throw e;
+    }
+}
+
+async function readRoutingFromDisk(account: string, bucketName: string): Promise<RemoteConfig | undefined> {
+    let data = await readRoutingFile(getBucketFolder(account, bucketName));
     if (!data) return undefined;
-    return buildBucket(account, bucketName, parseRoutingData(data));
+    return parseRoutingData(data);
+}
+
+async function loadBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined> {
+    let routing = await readRoutingFromDisk(account, bucketName);
+    if (!routing) return undefined;
+    return buildBucket(account, bucketName, routing);
 }
 
 export function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined> {
@@ -549,8 +542,6 @@ export function getLoadedBucket(account: string, bucketName: string): Promise<Lo
     if (!loaded) {
         loaded = loadBucket(account, bucketName);
         buckets.set(key, loaded);
-        // Missing buckets are not cached — they can be created at any time (guarding against a
-        // concurrent create having already replaced our entry)
         void loaded.then(bucket => {
             if (!bucket && buckets.get(key) === loaded) {
                 buckets.delete(key);
@@ -565,8 +556,6 @@ export function getLoadedBucket(account: string, bucketName: string): Promise<Lo
 }
 
 
-// Routing reloads are serialized per bucket, so concurrent writes/syncs can't rebuild the same
-// store twice in parallel
 const routingReloads = new Map<string, Promise<void>>();
 function scheduleRoutingReload(account: string, bucketName: string, config?: { force?: boolean; reason?: string }): Promise<void> {
     let key = `${account}/${bucketName}`;
@@ -581,18 +570,11 @@ async function checkRoutingChanged(account: string, bucketName: string, config?:
     let key = `${account}/${bucketName}`;
     let loaded = await buckets.get(key);
     if (!loaded) return;
-    // The routing config is only ever read off our own disk - it is never pulled from another
-    // source (it reaches us exclusively as a version-validated write, which lands on disk first)
-    let data = await new ArchivesDisk(getBucketFolder(account, bucketName)).get(ROUTING_FILE);
-    if (!data) return;
-    let routing = parseRoutingData(data);
+    let routing = await readRoutingFromDisk(account, bucketName);
+    if (!routing) return;
     if (!config?.force && JSON.stringify(routing) === loaded.routingJSON) return;
     let reason = config?.force && (config.reason || "forced") || "routing config changed";
     let plan = computeStorePlan(account, bucketName, routing);
-    // Config changes are applied to the RUNNING store in place - windows/routes mutate, added
-    // sources start syncing, removed sources' slots go dead - so the index, pending fast writes
-    // (re-capped to the new deadline), and unaffected sync loops all survive. The store is only
-    // destroyed for what it structurally cannot express (a rawDisk flip).
     if (plan.structureKey === loaded.structureKey && loaded.store instanceof BlobStore) {
         console.log(`Applying the routing config to the running store for bucket ${key} (${reason})`);
         loaded.store.updateSources(plan.sourceSpecs.map(spec => {
@@ -623,9 +605,6 @@ async function checkRoutingChanged(account: string, bucketName: string, config?:
     broadcastRoutingChanged();
 }
 
-// Per-write options are evaluated at the WRITE's time (and the key's route), not the current
-// config: the write time is stamped exactly once (at the first node that receives the write) and
-// propagates with the data, so every node resolves the same self entry - and the same options.
 function getWriteConfig(bucket: LoadedBucket, writeTime: number, route: number): { fast?: boolean; writeDelay?: number } {
     let self = selectEntryAt(bucket.selfEntries, writeTime, route);
     return { fast: self?.fast, writeDelay: self?.writeDelay };
@@ -639,8 +618,6 @@ export async function assertMutable(bucket: LoadedBucket, filePath: string, writ
     }
 }
 
-// Wrong-target rejections are rare and important (they ARE the switchover handoff), but a burst of
-// racing writes at a boundary must not flood the log - so at most one line a minute
 const WRONG_TARGET_LOG_THROTTLE = 60 * 1000;
 let lastWrongTargetLog = 0;
 function logWrongTargetRejection(message: string): void {
@@ -649,13 +626,10 @@ function logWrongTargetRejection(message: string): void {
     console.log(message);
 }
 
-// Routing writes are serialized per bucket, so two concurrent creates can't both build a store
-// (the loser's store would leak with its sync loops running forever)
 const routingWrites = new Map<string, Promise<void>>();
 
 async function writeRoutingConfig(account: string, bucketName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
     let key = `${account}/${bucketName}`;
-    // Validates before storing anything, so a bad config can't brick the bucket
     let incoming = parseRoutingData(data);
     let loaded = await getLoadedBucket(account, bucketName);
     if (!loaded) {
@@ -665,16 +639,26 @@ async function writeRoutingConfig(account: string, bucketName: string, data: Buf
         console.log(`Created bucket ${key}`);
         return;
     }
-    // A changed config must carry a higher version than the one currently stored — otherwise two
-    // clients with different configs at the same version would keep clobbering each other. Same
-    // content at any version is a harmless no-op and is allowed through.
     if (JSON.stringify(incoming) !== loaded.routingJSON && getConfigVersion(incoming) <= getConfigVersion(loaded.routing)) {
-        throw new Error(`Cannot update routing config for bucket ${key}: the new config differs from the current one but its version (${getConfigVersion(incoming)}) is not greater than the current version (${getConfigVersion(loaded.routing)}). Increment the version to update it. Current: ${loaded.routingJSON}. Attempted: ${JSON.stringify(incoming)}`);
+        let message = `Cannot update routing config for bucket ${key}: the new config differs from the current one but its version (${getConfigVersion(incoming)}) is not greater than the current version (${getConfigVersion(loaded.routing)}). Increment the version to update it. Current: ${loaded.routingJSON}. Attempted: ${JSON.stringify(incoming)}`;
+        console.error(message);
+        throw new Error(message);
     }
-    // Routing writes bypass fast mode (the config must apply immediately and survive a crash)
-    // and ignore immutable (the routing file must always stay updatable). An older lastModified
-    // no-ops inside set, in which case the reload below sees no change.
-    await loaded.store.set(ROUTING_FILE, data, { lastModified: config?.lastModified });
+    let stored = incoming;
+    let reinjected = 0;
+    for (let intermediate of getIntermediateSources(expireIntermediateSources(loaded.routing, Date.now()))) {
+        stored = injectIntermediateSource(stored, {
+            splitUrl: findSplitUrl(loaded.routing, intermediate) || intermediate.url,
+            intermediateUrl: intermediate.url,
+            start: intermediate.validWindow[0],
+            end: intermediate.validWindow[1],
+        });
+        reinjected++;
+    }
+    if (JSON.stringify(stored) !== JSON.stringify(incoming)) {
+        console.log(`Re-injected ${reinjected} in-flight switchover windows into the incoming routing config for bucket ${key} (version ${getConfigVersion(incoming)})`);
+    }
+    await loaded.store.set(ROUTING_FILE, Buffer.from(serializeRemoteConfig(stored)), { lastModified: config?.lastModified });
     await scheduleRoutingReload(account, bucketName);
 }
 
@@ -683,8 +667,6 @@ export async function writeBucketFile(account: string, bucketName: string, fileP
     if (filePath === ROUTING_FILE) {
         let key = `${account}/${bucketName}`;
         let run = (routingWrites.get(key) || Promise.resolve()).then(() => writeRoutingConfig(account, bucketName, data, config));
-        // The chain must survive a failed write, so the stored link swallows the error (the caller
-        // still gets it from run)
         routingWrites.set(key, run.then(() => { }, () => { }));
         return await run;
     }
@@ -692,13 +674,7 @@ export async function writeBucketFile(account: string, bucketName: string, fileP
     if (!loaded) {
         throw new Error(`Bucket ${account}/${bucketName} does not exist. Write its routing config to ${JSON.stringify(ROUTING_FILE)} to create it.`);
     }
-    // The write time is determined exactly once, HERE at the first node that receives the write;
-    // it propagates as lastModified, so every node evaluates the bucket options at the same instant
     let writeTime = config?.lastModified || Date.now();
-    // Only freshly-stamped writes are checked: a write with an explicit lastModified is
-    // synchronization/backfill (old data legitimately lands on sources whose window has moved on),
-    // but a fresh write reaching us outside our windows/routes means the client resolved its
-    // target from a stale view - it must re-resolve and retry at the correct source.
     let route = getRoute(filePath);
     if (!config?.lastModified && loaded.selfEntries.length) {
         let timeValid = loaded.selfEntries.filter(x => writeTime >= x.validWindow[0] && writeTime < x.validWindow[1]);
@@ -719,9 +695,7 @@ export function getBucketConfig(bucket: LoadedBucket): ArchivesConfig {
     let progress = bucket.store.getSyncProgress?.();
     return {
         supportsChangesAfter: !!bucket.store.getChangesAfter,
-        // The remapped (interpretation) view: getConfig is how clients learn of an in-progress
-        // deploy takeover, since the remap is never written into the routing file itself
-        remoteConfig: applyDeployRemap(bucket.routing),
+        remoteConfig: bucket.routing,
         index: progress?.index,
         indexSources: progress?.sources,
         readerDiskLimit: progress?.readerDiskLimit,
@@ -732,65 +706,235 @@ export function getBucketConfig(bucket: LoadedBucket): ArchivesConfig {
 export async function rebuildAllLoadedBuckets(): Promise<void> {
     for (let key of [...buckets.keys()]) {
         let slash = key.indexOf("/");
-        await scheduleRoutingReload(key.slice(0, slash), key.slice(slash + 1), { force: true, reason: "deploy takeover state changed" });
+        await scheduleRoutingReload(key.slice(0, slash), key.slice(slash + 1), { force: true, reason: "all loaded buckets were asked to rebuild" });
+    }
+}
+
+const OWN_CONFIG_WRITE_THROTTLE = 30 * 1000;
+const INTERMEDIATE_MAINTAIN_INTERVAL = 60 * 1000;
+let lastOwnConfigWrite = 0;
+
+async function writeOwnRoutingConfig(loaded: LoadedBucket, updated: RemoteConfig, reason: string): Promise<void> {
+    let key = `${loaded.account}/${loaded.bucketName}`;
+    let sinceLast = Date.now() - lastOwnConfigWrite;
+    if (sinceLast < OWN_CONFIG_WRITE_THROTTLE) {
+        console.log(`Not writing our own routing config update for bucket ${key} (${reason}): the last one was ${sinceLast}ms ago, under the ${OWN_CONFIG_WRITE_THROTTLE}ms throttle. Retrying on the next maintenance pass. Wanted: ${JSON.stringify(updated)}`);
+        return;
     }
+    lastOwnConfigWrite = Date.now();
+    let version = nextIntermediateVersion(getConfigVersion(loaded.routing));
+    let next: RemoteConfig = { ...updated, version };
+    console.log(`Writing our own routing config update for bucket ${key} (${reason}), version ${getConfigVersion(loaded.routing)} -> ${version}: ${JSON.stringify(next)}`);
+    let data = Buffer.from(serializeRemoteConfig(next));
+    await writeBucketFile(loaded.account, loaded.bucketName, ROUTING_FILE, data);
+    await propagateRoutingConfig(loaded, next, data);
 }
 
-onTakeoverEvent(event => {
-    if (event === "remapChanged") {
-        // The rebuild applies the remapped windows, and scheduleBoundaryScans (part of the
-        // rebuild) schedules the handoff scans around the new windows' starts
-        void rebuildAllLoadedBuckets();
+async function propagateRoutingConfig(loaded: LoadedBucket, next: RemoteConfig, data: Buffer): Promise<void> {
+    let targets = new Map<string, HostedConfig | BackblazeConfig>();
+    for (let source of [...loaded.routing.sources, ...next.sources]) {
+        if (typeof source === "string" || source.intermediate) continue;
+        if (source.type === "remote") {
+            let parsed = parseHostedUrl(source.url);
+            if (parsed.account !== loaded.account || parsed.bucketName !== loaded.bucketName) continue;
+        }
+        if (targets.has(source.url)) continue;
+        targets.set(source.url, source);
+    }
+    for (let [url, source] of targets) {
+        try {
+            await createApiArchives(source).set(ROUTING_FILE, data);
+            console.log(`Propagated our routing config update to ${url}`);
+        } catch (e) {
+            console.error(`Propagating our routing config update to ${url} failed: ${(e as Error).stack ?? e}`);
+        }
     }
+}
+
+async function maintainIntermediates(): Promise<void> {
+    let { domain, port } = getStorageServerConfig();
+    let takeover = getTakeoverIntermediate();
+    for (let key of [...buckets.keys()]) {
+        let loadedPromise = buckets.get(key);
+        if (!loadedPromise) continue;
+        let loaded = await loadedPromise.catch(() => undefined);
+        if (!loaded) continue;
+        let expired = expireIntermediateSources(loaded.routing, Date.now());
+        if (JSON.stringify(expired) !== JSON.stringify(loaded.routing)) {
+            await writeOwnRoutingConfig(loaded, expired, `switchover windows expired more than ${INTERMEDIATE_EXPIRE_GRACE / 1000}s ago`);
+            continue;
+        }
+        if (!takeover) continue;
+        let mainUrl = loaded.routing.sources.find(x => {
+            if (typeof x === "string" || x.type !== "remote" || x.intermediate) return false;
+            let parsed = parseHostedUrl(x.url);
+            return parsed.address === domain && parsed.port === port;
+        }) as HostedConfig | undefined;
+        if (!mainUrl) continue;
+        let injected = injectIntermediateSource(loaded.routing, {
+            splitUrl: mainUrl.url,
+            intermediateUrl: replaceHostedUrlPort(mainUrl.url, takeover.altPort),
+            start: takeover.start,
+            end: takeover.end,
+        });
+        if (JSON.stringify(injected) === JSON.stringify(loaded.routing)) continue;
+        await writeOwnRoutingConfig(loaded, injected, `we are a deploy successor: writes route to our alternate port ${takeover.altPort} from ${new Date(takeover.start).toISOString()} until our predecessor is killed at ${new Date(takeover.end).toISOString()}`);
+    }
+}
+
+/** Started by deployTakeover once we are actually a deploy successor listening on an alternate port. Until then there are no switchover windows to write or expire, so nothing polls. */
+export const startIntermediateMaintenance = lazy(() => {
+    void maintainIntermediates().catch((e: Error) => console.error(`Maintaining switchover routing windows failed: ${e.stack ?? e}`));
+    runInfinitePoll(INTERMEDIATE_MAINTAIN_INTERVAL, maintainIntermediates);
 });
 
+export type BucketDiskInfo = {
+    totalBytes: number;
+    freeBytes: number;
+    usedBytes: number;
+};
 export type ServerBucketInfo = {
     bucketName: string;
-    // Loaded in memory (created or accessed since startup), so its synchronization is running
     active: boolean;
+    /** Where the bucket's data lives on this server */
+    folder: string;
+    /** The drive that folder is on. Buckets sharing a drive report the same numbers. */
+    disk?: BucketDiskInfo;
+    diskError?: string;
+    writeStats?: BucketWriteStats;
     config?: ArchivesConfig;
     error?: string;
 };
 
-/** Every bucket the account has on this server, active or not, each with its configuration.
- *  Inactive buckets are inspected straight from disk WITHOUT loading them - loading would start
- *  their synchronization, and old invalid buckets must stay inert (their parse error is reported
- *  instead). */
+async function getDiskInfo(folder: string): Promise<BucketDiskInfo> {
+    let stats = await fs.promises.statfs(folder);
+    let blockSize = Number(stats.bsize);
+    let totalBytes = Number(stats.blocks) * blockSize;
+    return {
+        totalBytes,
+        // Matches the server's own low-space check: what an unprivileged process can actually use
+        freeBytes: Number(stats.bavail) * blockSize,
+        usedBytes: totalBytes - Number(stats.bfree) * blockSize,
+    };
+}
+
+export type ActiveBucketInfo = {
+    folder: string;
+    /** The routing config the bucket is RUNNING on, straight from memory - including switchover windows written since it loaded */
+    routing: RemoteConfig;
+    /** Our own entries in that config, and the one currently valid */
+    selfEntries: HostedConfig[];
+    self?: HostedConfig;
+    config: ArchivesConfig;
+};
+
+/** The live in-memory state of ONE bucket, answered without touching the disk (no routing file read, no statfs, no stored write stats). Returns an error string when the bucket is not loaded here, which is the normal state for a bucket nothing has accessed since startup. */
+export async function getActiveBucket(account: string, bucketName: string): Promise<ActiveBucketInfo | string> {
+    let key = `${account}/${bucketName}`;
+    let loadedPromise = buckets.get(key);
+    if (!loadedPromise) {
+        return `Bucket ${key} is not loaded on this server, so it has no live state (it loads on first access)`;
+    }
+    let loaded: LoadedBucket | undefined;
+    try {
+        loaded = await loadedPromise;
+    } catch (e) {
+        return `Bucket ${key} failed to load on this server: ${String((e as Error).stack ?? e).slice(0, 500)}`;
+    }
+    if (!loaded) {
+        return `Bucket ${key} does not exist on this server`;
+    }
+    return toActiveBucketInfo(loaded);
+}
+
+function toActiveBucketInfo(loaded: LoadedBucket): ActiveBucketInfo {
+    return {
+        folder: getBucketFolder(loaded.account, loaded.bucketName),
+        routing: loaded.routing,
+        selfEntries: loaded.selfEntries,
+        self: loaded.self,
+        config: getBucketConfig(loaded),
+    };
+}
+
+/** Loads a bucket that exists on this server's disk into memory, which starts its synchronization and window timers, and returns its live state. Nothing is written and no other server is contacted - unlike building an ArchivesChain for it, which would probe every source and could write the routing config. Already-loaded buckets just return their state. */
+export async function activateBucket(account: string, bucketName: string): Promise<ActiveBucketInfo | string> {
+    let key = `${account}/${bucketName}`;
+    let wasLoaded = buckets.has(key);
+    let loaded: LoadedBucket | undefined;
+    try {
+        loaded = await getLoadedBucket(account, bucketName);
+    } catch (e) {
+        return `Bucket ${key} failed to load on this server: ${String((e as Error).stack ?? e).slice(0, 500)}`;
+    }
+    if (!loaded) {
+        return `Bucket ${key} does not exist on this server (no routing file in ${getBucketFolder(account, bucketName)})`;
+    }
+    if (!wasLoaded) {
+        console.log(`Activated bucket ${key} on request: it is now loaded and synchronizing`);
+    }
+    return toActiveBucketInfo(loaded);
+}
+
 export async function listAccountBuckets(account: string): Promise<ServerBucketInfo[]> {
+    let start = Date.now();
+    let timings = new Map<string, number>();
+    async function timed<T>(name: string, run: () => Promise<T>): Promise<T> {
+        let began = Date.now();
+        try {
+            return await run();
+        } finally {
+            timings.set(name, (timings.get(name) || 0) + (Date.now() - began));
+        }
+    }
     let accountFolder = path.join(getStorageServerConfig().folder, "buckets2", account);
     let names: string[];
     try {
-        names = await fs.promises.readdir(accountFolder);
+        names = await timed("readdir", () => fs.promises.readdir(accountFolder));
     } catch {
         return [];
     }
-    let results: ServerBucketInfo[] = [];
-    for (let bucketName of names) {
-        let loadedPromise = buckets.get(`${account}/${bucketName}`);
-        if (loadedPromise) {
+    try {
+        return await Promise.all(names.map(async bucketName => {
+            let key = `${account}/${bucketName}`;
+            let folder = getBucketFolder(account, bucketName);
+            let base: ServerBucketInfo = { bucketName, active: false, folder };
+            let [disk, writeStats] = await Promise.all([
+                timed("statfs", () => getDiskInfo(folder)).catch((e: Error) => {
+                    base.diskError = String(e.stack ?? e).slice(0, 500);
+                    return undefined;
+                }),
+                timed("writeStats", () => getBucketWriteStats(key)),
+            ]);
+            base.disk = disk;
+            base.writeStats = writeStats;
+            const loadedPromise = buckets.get(key);
+            if (loadedPromise) {
+                try {
+                    let loaded = await timed("awaitLoaded", () => loadedPromise);
+                    if (loaded) {
+                        return { ...base, active: true, config: getBucketConfig(loaded) };
+                    }
+                } catch (e) {
+                    return { ...base, active: true, error: String((e as Error).stack ?? e).slice(0, 500) };
+                }
+            }
             try {
-                let loaded = await loadedPromise;
-                if (loaded) {
-                    results.push({ bucketName, active: true, config: getBucketConfig(loaded) });
-                    continue;
+                const data = await timed("readRoutingFile", () => readRoutingFile(folder));
+                if (!data) {
+                    return { ...base, error: `No routing file (${ROUTING_FILE})` };
                 }
+                let parsed = await timed("parseRouting", async () => parseRoutingData(data));
+                return { ...base, config: { remoteConfig: parsed } };
             } catch (e) {
-                results.push({ bucketName, active: true, error: String((e as Error).stack ?? e).slice(0, 500) });
-                continue;
-            }
-        }
-        try {
-            let data = await new ArchivesDisk(getBucketFolder(account, bucketName)).get(ROUTING_FILE);
-            if (!data) {
-                results.push({ bucketName, active: false, error: `No routing file (${ROUTING_FILE})` });
-                continue;
+                return { ...base, error: String((e as Error).stack ?? e).slice(0, 500) };
             }
-            results.push({ bucketName, active: false, config: { remoteConfig: parseRoutingData(data) } });
-        } catch (e) {
-            results.push({ bucketName, active: false, error: String((e as Error).stack ?? e).slice(0, 500) });
-        }
+        }));
+    } finally {
+        // The parts run concurrently, so these are summed wall times per part, not a breakdown of the total
+        let parts = [...timings].map(([name, ms]) => `${name} ${ms}ms`).join(", ");
+        console.log(`listAccountBuckets(${account}) took ${Date.now() - start}ms for ${names.length} buckets: ${parts}`);
     }
-    return results;
 }
 
 export async function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void> {
@@ -802,12 +946,8 @@ export async function deleteBucketFile(account: string, bucketName: string, file
     await loaded.store.del(filePath, getWriteConfig(loaded, Date.now(), getRoute(filePath)));
 }
 
-// ── local IArchives access ──
-
 const LARGE_FILE_PART_SIZE = 8 * 1024 * 1024;
 
-// The in-process IArchives for a bucket hosted by THIS server — used when a RemoteConfig source
-// URL points at ourselves, so we don't talk to ourselves over HTTPS. One singleton per bucket.
 class ArchivesLocalBucket implements IArchives {
     constructor(private account: string, private bucketName: string) { }
 
@@ -875,7 +1015,6 @@ class ArchivesLocalBucket implements IArchives {
     }
     public async getConfig(): Promise<ArchivesConfig> {
         let bucket = await this.getBucket();
-        // Missing buckets say true, matching what they become once created (the default store type)
         if (!bucket) return { supportsChangesAfter: true };
         return getBucketConfig(bucket);
     }