sliftutils 1.7.21 → 1.7.23

package/storage/remoteStorage/blobStore.ts

@@ -5,7 +5,7 @@ import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import {
     IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, assertValidLastModified,
-    windowAcceptsWrites, SyncActivity,
+    windowAcceptsWrites, SyncActivity, FULL_ROUTE,
 } from "../IArchives";
 import { ArchivesDisk, applyFindInfoShape } from "../ArchivesDisk";
 import { ArchivesBackblaze } from "../backblaze";
@@ -22,6 +22,10 @@ import { wrapHandle, NodeJSDirectoryHandleWrapper, DirectoryWrapper } from "../F
 
 export const DEFAULT_FAST_WRITE_DELAY = timeInMinute * 5;
 const FAST_FLUSH_POLL = 1000 * 15;
+// Fast writes are never delayed past our own valid window, and within this margin of the window's
+// end they write through immediately - so when the next window's source takes over, the writes are
+// already on disk
+export const WINDOW_END_FLUSH_MARGIN = timeInMinute * 5;
 // Index changes are buffered in memory and written to the BulkDatabase2 in batches
 const INDEX_FLUSH_INTERVAL = 1000 * 30;
 // Sources that support getChangesAfter are polled this often
@@ -120,6 +124,17 @@ type SourceState = {
     lastMissCheck: number;
 };
 
+// What a scanned listing entry meant when compared against our index
+type ScanOutcome = "filtered" | "new" | "updated" | "tombstone" | "unchanged";
+type ScanTally = Record<ScanOutcome, number>;
+function newScanTally(): ScanTally {
+    return { filtered: 0, new: 0, updated: 0, tombstone: 0, unchanged: 0 };
+}
+function formatScanTally(tally: ScanTally, total: number): string {
+    let pct = (n: number) => `${Math.round(n / Math.max(total, 1) * 1000) / 10}%`;
+    return `${tally.new} new paths (${pct(tally.new)}), ${tally.updated} newer writes (${pct(tally.updated)}), ${tally.tombstone} deletions (${pct(tally.tombstone)}), ${tally.unchanged} unchanged (${pct(tally.unchanged)}), ${tally.filtered} outside window/route (${pct(tally.filtered)})`;
+}
+
 export class BlobStore implements IBucketStore {
     constructor(
         private folder: string,
@@ -130,10 +145,6 @@ export class BlobStore implements IBucketStore {
             onIndexChanged?: (key: string) => void;
             // LRU-bound the disk (base source) to this many bytes; see CommonConfig.readerDiskLimit
             readerDiskLimit?: number;
-            // Deploy takeover: fast-write flush delays never extend past this time, and after it
-            // fast writes flush immediately (nothing may sit in memory when the write window
-            // transfers to the successor process)
-            getFlushDeadline?: () => number | undefined;
         }
     ) { }
 
@@ -205,6 +216,11 @@ export class BlobStore implements IBucketStore {
             let source = sourceMap.get(entry.key);
             // Explicit checks, as 0 is a valid size and a valid source number
             if (size === undefined || source === undefined) continue;
+            // The routing config is only ever read off our own disk (see applyScanned), and a
+            // loaded bucket always has it there - a persisted entry pointing elsewhere is stale
+            if (entry.key === ROUTING_FILE) {
+                source = 0;
+            }
             let full: IndexEntry = { writeTime: entry.value, size, source, changedAt: entry.time, lastAccess: entry.time };
             this.mem.set(entry.key, full);
             this.countEntry(full, 1);
@@ -238,13 +254,63 @@ export class BlobStore implements IBucketStore {
         this.dirty.set(key, undefined);
     }
 
-    /** Rescans our own disk's metadata into the index - used around deploy switchovers, where the
-     *  other process wrote files to the shared folder that our index hasn't seen. */
+    /** Rescans our own disk's metadata into the index - used around valid window handoffs, where
+     *  another process wrote files to the shared folder that our index hasn't seen. */
     public async rescanBase(): Promise<void> {
         await this.init();
         await this.scanSource(0);
     }
 
+    /** A boundary scan of the node that owned (part of) our route in the valid window before ours,
+     *  when that node is different storage (a disk rescan can't see its writes): just its changes
+     *  since the boundary neighborhood, with matching values pulled onto our own disk. */
+    public async boundaryScanRemote(source: IArchives, config: { since: number; route?: [number, number] }): Promise<void> {
+        await this.init();
+        let scanStart = Date.now();
+        console.log(`Boundary scan of ${source.getDebugName()} starting: changes since ${new Date(config.since).toISOString()}, route ${JSON.stringify(config.route || FULL_ROUTE)} (store ${this.folder})`);
+        let changes: ArchiveFileInfo[];
+        if (source.getChangesAfter) {
+            changes = await source.getChangesAfter(config.since);
+        } else {
+            changes = (await source.findInfo("")).filter(x => x.createTime > config.since);
+        }
+        let tally = newScanTally();
+        for (let file of changes) {
+            if (file.path === ROUTING_FILE || !routeContains(config.route, getRoute(file.path))) {
+                tally.filtered++;
+                continue;
+            }
+            let overlayEntry = this.overlay.get(file.path);
+            let entry = this.mem.get(file.path);
+            let currentTime = overlayEntry && overlayEntry.t || entry && entry.writeTime || 0;
+            if (file.createTime <= currentTime) {
+                tally.unchanged++;
+                continue;
+            }
+            if (file.size === 0) {
+                // A tombstone stores nothing on our own source - the index entry alone records it
+                this.setIndexEntry(file.path, { writeTime: file.createTime, size: 0, source: 0 });
+                tally.tombstone++;
+                continue;
+            }
+            let result = await source.get2(file.path);
+            if (!result) continue;
+            if (result.data.length === 0) {
+                this.setIndexEntry(file.path, { writeTime: result.writeTime, size: 0, source: 0 });
+                tally.tombstone++;
+                continue;
+            }
+            await this.sources[0].source.set(file.path, result.data, { lastModified: result.writeTime });
+            this.setIndexEntry(file.path, { writeTime: result.writeTime, size: result.data.length, source: 0 });
+            if (entry || overlayEntry) {
+                tally.updated++;
+            } else {
+                tally.new++;
+            }
+        }
+        console.log(`Boundary scan of ${source.getDebugName()} finished in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.folder}): ${changes.length} changes: ${formatScanTally(tally, changes.length)}`);
+    }
+
     /** The cheap always-current totals plus any in-progress background synchronization. */
     public getSyncProgress(): {
         index: { fileCount: number; byteCount: number };
@@ -358,14 +424,13 @@ export class BlobStore implements IBucketStore {
     // Full metadata scan (size, writeTime, path) of one source, applied to the index. Returns the
     // source's listing (path -> write time), which reconcileSource uses for the push direction.
     private async scanSource(sourceIndex: number): Promise<Map<string, number>> {
-        let { source } = this.sources[sourceIndex];
+        let { source, validWindow, route } = this.sources[sourceIndex];
         let state = this.sourceStates[sourceIndex];
         let scanStart = Date.now();
-        let scanLogged = false;
         let activity: SyncActivity = { type: "metadataScan", sourceDebugName: source.getDebugName(), startTime: scanStart };
         this.syncActivities.add(activity);
+        console.log(`Metadata scan of ${source.getDebugName()} starting (store ${this.folder})`);
         let progressTimer = setInterval(() => {
-            scanLogged = true;
             console.log(`Metadata scan of ${source.getDebugName()} still running (${Math.round((Date.now() - scanStart) / 1000)}s, store ${this.folder})`);
         }, SYNC_PROGRESS_LOG_INTERVAL);
         (progressTimer as { unref?: () => void }).unref?.();
@@ -376,26 +441,43 @@ export class BlobStore implements IBucketStore {
             clearInterval(progressTimer);
             this.syncActivities.delete(activity);
         }
-        if (scanLogged) {
-            console.log(`Metadata scan of ${source.getDebugName()} finished: ${files.length} files in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.folder})`);
-        }
+        let indexSizeBefore = this.mem.size;
         let seen = new Map<string, number>();
+        let tally = newScanTally();
+        let newPaths = 0;
         for (let file of files) {
             seen.set(file.path, file.createTime);
-            this.applyScanned(sourceIndex, file);
+            if (!this.mem.has(file.path)) {
+                newPaths++;
+            }
+            tally[this.applyScanned(sourceIndex, file)]++;
         }
         state.scannedCount = files.length;
         // Index entries this source was the holder of, but that vanished from it (e.g. deleted
         // while we were offline), come out of the index. Entries changed after the scan started
         // are kept — the scan listing may simply predate them. Tombstones have no physical file
         // for a listing to vouch for, so they're exempt (cleanupTombstones expires them instead).
+        let removedFromIndex = 0;
+        let missingOnSource = 0;
         for (let [key, entry] of this.mem) {
-            if (entry.source !== sourceIndex) continue;
-            if (entry.size === 0) continue;
             if (seen.has(key)) continue;
-            if (entry.changedAt >= scanStart) continue;
-            this.deleteIndexEntry(key);
+            if (entry.source === sourceIndex && entry.size !== 0 && entry.changedAt < scanStart) {
+                this.deleteIndexEntry(key);
+                removedFromIndex++;
+                continue;
+            }
+            // Counted only when the source SHOULD hold the entry (its window/route match) - these
+            // are what the reconcile pass pushes to it
+            if (entry.size === 0 || key === ROUTING_FILE) continue;
+            if (entry.writeTime < validWindow[0] || entry.writeTime > validWindow[1]) continue;
+            if (!routeContains(route, getRoute(key))) continue;
+            missingOnSource++;
         }
+        // Percentages are of the union of both sides (our index + their listing), so every count
+        // has a stable denominator
+        let union = indexSizeBefore + newPaths;
+        let pct = (n: number) => `${Math.round(n / Math.max(union, 1) * 1000) / 10}%`;
+        console.log(`Metadata scan of ${source.getDebugName()} finished in ${Math.round((Date.now() - scanStart) / 1000)}s (store ${this.folder}): ${files.length} listed vs ${indexSizeBefore} indexed (union ${union}): ${formatScanTally(tally, union)}, ${missingOnSource} in index but missing on source (${pct(missingOnSource)}), ${removedFromIndex} removed from index (${pct(removedFromIndex)})`);
         state.changesAfterTime = Math.max(state.changesAfterTime, scanStart - CHANGES_POLL_OVERLAP);
         return seen;
     }
@@ -411,12 +493,11 @@ export class BlobStore implements IBucketStore {
             for (let [key, entry] of this.mem) {
                 if (this.stopped.stop) return;
                 if (entry.source === sourceIndex) continue;
-                // Past-window sources only receive the routing file (see writeToSources), and
-                // sharded sources only the keys routing into them
-                if (key !== ROUTING_FILE) {
-                    if (!acceptsWrites) continue;
-                    if (!routeContains(route, getRoute(key))) continue;
-                }
+                // The routing file is NEVER synchronized between storage nodes - it is only ever
+                // written directly to each node, and only ever read off our own disk
+                if (key === ROUTING_FILE) continue;
+                if (!acceptsWrites) continue;
+                if (!routeContains(route, getRoute(key))) continue;
                 let theirTime = listing.get(key);
                 if (theirTime !== undefined && theirTime >= entry.writeTime) continue;
                 if (entry.size === 0) {
@@ -435,16 +516,28 @@ export class BlobStore implements IBucketStore {
         }
     }
 
-    private applyScanned(sourceIndex: number, file: ArchiveFileInfo): void {
-        let { validWindow, route } = this.sources[sourceIndex];
-        let [windowStart, windowEnd] = validWindow;
-        if (file.createTime < windowStart || file.createTime > windowEnd) return;
-        // A partially-overlapping shard's listing includes keys outside our route; ignore them
-        if (file.path !== ROUTING_FILE && !routeContains(route, getRoute(file.path))) return;
+    private applyScanned(sourceIndex: number, file: ArchiveFileInfo): ScanOutcome {
+        if (file.path === ROUTING_FILE) {
+            // The routing config is NEVER pulled from other sources - it only ever arrives as an
+            // explicit, version-validated write, and is only ever read off our own disk. Route and
+            // valid-window filters can't possibly apply to it either: it is the file DEFINING
+            // them, so filtering it would mean certain sources could never have their routing
+            // config updated, ever.
+            if (sourceIndex !== 0) return "filtered";
+        } else {
+            let { validWindow, route } = this.sources[sourceIndex];
+            let [windowStart, windowEnd] = validWindow;
+            if (file.createTime < windowStart || file.createTime > windowEnd) return "filtered";
+            // A partially-overlapping shard's listing includes keys outside our route; ignore them
+            if (!routeContains(route, getRoute(file.path))) return "filtered";
+        }
         let existing = this.mem.get(file.path);
         // The highest write time wins across all sources (ties keep the existing entry)
-        if (existing && file.createTime <= existing.writeTime) return;
+        if (existing && file.createTime <= existing.writeTime) return "unchanged";
         this.setIndexEntry(file.path, { writeTime: file.createTime, size: file.size, source: sourceIndex });
+        if (file.size === 0) return "tombstone";
+        if (existing) return "updated";
+        return "new";
     }
 
     private async pollChanges(sourceIndex: number): Promise<void> {
@@ -453,8 +546,13 @@ export class BlobStore implements IBucketStore {
         let state = this.sourceStates[sourceIndex];
         let pollStart = Date.now();
         let changes = await source.getChangesAfter(state.changesAfterTime);
+        let tally = newScanTally();
         for (let file of changes) {
-            this.applyScanned(sourceIndex, file);
+            tally[this.applyScanned(sourceIndex, file)]++;
+        }
+        // Polls run constantly, so only the ones that actually changed the index get a line
+        if (tally.new || tally.updated || tally.tombstone) {
+            console.log(`Changes poll of ${source.getDebugName()} (store ${this.folder}): ${changes.length} changes: ${formatScanTally(tally, changes.length)}`);
         }
         state.scannedCount += changes.length;
         state.changesAfterTime = pollStart - CHANGES_POLL_OVERLAP;
@@ -625,6 +723,12 @@ export class BlobStore implements IBucketStore {
             }
             return result;
         }
+        // The routing file is only ever read off our own disk - falling back to another source's
+        // copy would synchronize it between nodes through the read path, which it never is
+        if (key === ROUTING_FILE) {
+            if (holderError) throw holderError;
+            return undefined;
+        }
         // The holder is down or lost the file. ANY other source's copy beats no value - even an
         // OLDER one - and it's copied onto our disk so the next read doesn't depend on luck.
         for (let i = 0; i < this.sources.length; i++) {
@@ -669,22 +773,24 @@ export class BlobStore implements IBucketStore {
         }
         let writeTime = lastModified || Date.now();
         if (config?.fast) {
-            let writeDelay = config.writeDelay || DEFAULT_FAST_WRITE_DELAY;
-            let flushAt = Date.now() + writeDelay;
-            let deadline = this.config?.getFlushDeadline?.();
-            if (deadline !== undefined) {
-                if (Date.now() >= deadline) {
-                    if (!this.loggedFlushDeadline) {
-                        this.loggedFlushDeadline = true;
-                        console.log(`Deploy switchover flush deadline passed (store ${this.folder}): fast writes now write through immediately`);
-                    }
-                    this.overlay.delete(key);
-                    await this.writeToSources(key, data, writeTime);
-                    return key;
-                }
-                flushAt = Math.min(flushAt, deadline);
+            // A writeDelay of zero is a real choice (no delay at all), so only an omitted delay
+            // gets the default
+            let writeDelay = config.writeDelay;
+            if (writeDelay === undefined) {
+                writeDelay = DEFAULT_FAST_WRITE_DELAY;
+            }
+            // The delay never extends past our own valid window's end (minus the margin, so the
+            // writes are on disk before the next window's source takes over - a deploy switchover
+            // is just this too, since its remap ends our window). Past that point fast writes
+            // write through immediately.
+            let deadline = this.sources[0].validWindow[1] - WINDOW_END_FLUSH_MARGIN;
+            if (writeDelay > 0 && Date.now() < deadline) {
+                let flushAt = Math.min(Date.now() + writeDelay, deadline);
+                this.overlay.set(key, { data, t: writeTime, flushAt });
+                return key;
             }
-            this.overlay.set(key, { data, t: writeTime, flushAt });
+            this.overlay.delete(key);
+            await this.writeToSources(key, data, writeTime);
             return key;
         }
         this.overlay.delete(key);
@@ -709,9 +815,10 @@ export class BlobStore implements IBucketStore {
     }
 
     private async writeToSources(key: string, data: Buffer, writeTime: number): Promise<void> {
-        // The routing file is exempt from the valid-window and route write filters: the CONFIG
-        // must keep flowing to every source, or a client probing one of them first would adopt a
-        // stale config. Only data writes stop.
+        // The routing file is NEVER synchronized between storage nodes: the writer writes it
+        // directly to each node, so we store it on our own disk only (no valid-window filter -
+        // routing/valid windows can't possibly apply to the file defining them) and never forward
+        // it to other sources.
         let isRouting = key === ROUTING_FILE;
         let writable = this.getWritableSources({ ignoreWindow: isRouting });
         let first = writable.shift();
@@ -728,9 +835,10 @@ export class BlobStore implements IBucketStore {
             await this.sources[first].source.set(key, data, { lastModified: writeTime });
         }
         this.setIndexEntry(key, { writeTime, size: data.length, source: first });
-        let route = !isRouting && getRoute(key) || 0;
+        if (isRouting) return;
+        let route = getRoute(key);
         for (let i of writable) {
-            if (!isRouting && !routeContains(this.sources[i].route, route)) continue;
+            if (!routeContains(this.sources[i].route, route)) continue;
             // Downstream sources receive tombstones as actual empty writes, so their listings show
             // the deletion (size 0) and other stores scan it in as a tombstone
             void this.sources[i].source.set(key, data, { lastModified: writeTime }).catch((e: Error) => {
@@ -858,7 +966,6 @@ export class BlobStore implements IBucketStore {
     // holds a same-or-newer copy (the only copy of a file is never deleted), and the index entry
     // repoints to that source so reads keep working (re-caching on the next read).
     private evicting = false;
-    private loggedFlushDeadline = false;
     private async enforceDiskLimit(): Promise<void> {
         let limit = this.config?.readerDiskLimit;
         if (!limit || this.evicting) return;

package/storage/remoteStorage/createArchives.ts

@@ -46,7 +46,7 @@ const AVAILABILITY_RECHECK_THROTTLE = 5 * 1000;
 // explicit writes surface the denial (with grant instructions) to the caller instead of hanging.
 export function createApiArchives(source: HostedConfig | BackblazeConfig): IArchives {
     if (source.type === "backblaze") {
-        return new ArchivesBackblaze({ bucketName: parseBackblazeUrl(source.url).bucketName, public: source.public, immutable: source.immutable });
+        return new ArchivesBackblaze({ bucketName: parseBackblazeUrl(source.url).bucketName, public: source.public, immutable: source.immutable, allowedOrigins: source.allowedOrigins });
     }
     let parsed = parseHostedUrl(source.url);
     let server = isNode() && getStorageServerConfigOptional() || undefined;
@@ -214,7 +214,17 @@ export class ArchivesChain implements IArchives {
                         // A rejected write fails init, which retries from scratch - re-reading the
                         // routing, so losing a create race to another client just adopts their
                         // config on the next attempt.
-                        await found.probe.write(archives => archives.set(ROUTING_FILE, serializeRemoteConfig(this.configured)));
+                        // The routing file is NEVER synchronized between storage nodes, so it is
+                        // written directly to EVERY node, with one shared write time (the latest
+                        // write time wins on each node independently)
+                        let routingData = serializeRemoteConfig(this.configured);
+                        let routingWriteTime = Date.now();
+                        let writtenUrls = new Set<string>();
+                        for (let source of sources) {
+                            if (writtenUrls.has(source.config.url)) continue;
+                            writtenUrls.add(source.config.url);
+                            await source.write(archives => archives.set(ROUTING_FILE, routingData, { lastModified: routingWriteTime }));
+                        }
                     } catch (e) {
                         for (let source of sources) {
                             source.dispose();

package/storage/remoteStorage/deployTakeover.d.ts

@@ -1,8 +1,8 @@
 import { RemoteConfig } from "../IArchives";
-export type TakeoverEvent = "remapChanged" | "diskScan";
+export type TakeoverEvent = "remapChanged";
 /** Starts the takeover machinery. Port fallback (alternate port + registry + acquisition polling)
  *  works regardless; without a deploy timeline folder the switchover-specific parts (the remap,
- *  the flush deadline, the tighter acquisition pacing) simply stay inert. */
+ *  the tighter acquisition pacing) simply stay inert. */
 export declare function initDeployTakeover(config: {
     domain: string;
     mainPort: number;
@@ -20,9 +20,9 @@ export declare function applyDeployRemap(routing: RemoteConfig): RemoteConfig;
  *  client learns of a takeover within one ping interval, instead of waiting for its config poll
  *  or a write rejection. */
 export declare function getTakeoverStamp(): string | undefined;
-/** For the dying process: fast-write flush delays must never extend past this time, and after it
- *  fast writes flush immediately - so nothing is left in memory when the write window transfers. */
-export declare function getFlushDeadline(): number | undefined;
+/** The middle-window alternate port of an active remap. The OTHER process of the takeover lives on
+ *  this port on OUR machine (same disk!), so sources pointing at it are self, never sync targets. */
+export declare function getTakeoverAltPort(): number | undefined;
 /** How long to wait between main-port acquisition attempts: tight around the predecessor's
  *  scheduled death (when the port actually frees), relaxed otherwise. */
 export declare function getMainPortAcquireDelay(): number;

package/storage/remoteStorage/deployTakeover.ts

@@ -29,12 +29,6 @@ const PORT_REGISTRY_FOLDER_NAME = "storagePortRegistry";
 const ACQUIRE_SLOW_DELAY = 30 * 1000;
 const ACQUIRE_FAST_DELAY = 5 * 1000;
 const ACQUIRE_FAST_WINDOW = 60 * 1000;
-// Disk rescans fire this far before and after the write handoff, catching files the other process
-// wrote (the BulkDatabase2 index is atomic but its entries land at write time, not file time)
-const SWITCH_SCAN_LEAD = 10 * 1000;
-// The dying process must have flushed everything before the handoff: fast-write delays never
-// extend past S + overlap * this fraction, and after that point fast writes flush immediately
-const FLUSH_DEADLINE_FRACTION = 0.25;
 const BOUNDARY_A_FRACTION = 0.5;
 const BOUNDARY_B_FRACTION = 1.5;
 const REMAP_EXPIRE_FRACTION = 2;
@@ -58,7 +52,7 @@ type TakeoverComputed = {
     predecessorEnd?: number;
 };
 
-export type TakeoverEvent = "remapChanged" | "diskScan";
+export type TakeoverEvent = "remapChanged";
 
 let initialized: { domain: string; mainPort: number; storageFolder: string } | undefined;
 let ancestorPids = new Set<number>();
@@ -202,18 +196,32 @@ function scheduleEvents(): void {
         (timer as { unref?: () => void }).unref?.();
         eventTimers.push(timer);
     };
-    let remap = current.remap;
+    const remap = current.remap;
     if (remap) {
         let altPort = remap.altPort;
-        schedule(remap.boundaryA - SWITCH_SCAN_LEAD, () => emit("diskScan"));
-        schedule(remap.boundaryA, () => console.log(`Deploy switchover write handoff boundary passed: fresh writes now belong to the alternate-port side (port ${altPort})`));
-        schedule(remap.boundaryA + SWITCH_SCAN_LEAD, () => emit("diskScan"));
-        schedule(remap.boundaryB, () => console.log(`Deploy switchover second boundary passed: fresh writes return to the main port`));
+        schedule(remap.boundaryA, () => console.log(`${logPrefix()} Deploy switchover write handoff boundary ${iso(remap.boundaryA)} passed: fresh writes now belong to the alternate-port side (port ${altPort})`));
+        schedule(remap.boundaryB, () => console.log(`${logPrefix()} Deploy switchover second boundary ${iso(remap.boundaryB)} passed: fresh writes return to the main port`));
         schedule(remap.expire, () => void recompute().catch((e: Error) => console.error(`Deploy takeover recompute failed: ${e.stack ?? e}`)));
     }
 }
 
-async function recompute(): Promise<void> {
+// Multiple processes (predecessor + successor) often share one journal during a switchover, so
+// every lifecycle line identifies its process
+function logPrefix(): string {
+    return `[pid ${process.pid}]`;
+}
+
+// Recompute is invoked from init, registerAltPort, the poll, and the expiry timer - possibly
+// concurrently. Runs are serialized, or two in-flight runs would both pass the changed-state check
+// and double-fire every transition (logs, rebuild events, timers).
+let recomputeChain = Promise.resolve();
+function recompute(): Promise<void> {
+    let next = recomputeChain.then(() => recomputeNow());
+    recomputeChain = next.then(() => { }, () => { });
+    return next;
+}
+
+async function recomputeNow(): Promise<void> {
     if (!initialized || !timelineFolder) return;
     let now = Date.now();
     let entries = await readTimelineEntries();
@@ -227,11 +235,11 @@ async function recompute(): Promise<void> {
     // Matching failures must be loud, or takeover detection fails silently forever
     if (!us && entries.length && !loggedNoMatch) {
         loggedNoMatch = true;
-        console.warn(`No deploy timeline entry matches our pid chain (${[...ancestorPids].join(", ")}) - takeover detection cannot work. Entries: ${entries.map(x => `pid=${x.pid ?? "none"} releaseTime=${x.parameters.releaseTime}`).join(" | ")}`);
+        console.warn(`${logPrefix()} No deploy timeline entry matches our pid chain (${[...ancestorPids].join(", ")}) - takeover detection cannot work. Entries: ${entries.map(x => `pid=${x.pid ?? "none"} releaseTime=${x.parameters.releaseTime}`).join(" | ")}`);
     }
     if (us && !loggedIdentity) {
         loggedIdentity = true;
-        console.log(`Deploy timeline entry matched: we are pid=${us.pid}, releaseTime=${us.parameters.releaseTime !== undefined && iso(us.parameters.releaseTime) || "unknown"}, ${entries.length} entries total`);
+        console.log(`${logPrefix()} Deploy timeline entry matched: we are pid=${us.pid}, releaseTime=${us.parameters.releaseTime !== undefined && iso(us.parameters.releaseTime) || "unknown"}, ${entries.length} entries total`);
     }
     let computed: TakeoverComputed = {};
     let usRelease = us?.parameters.releaseTime;
@@ -254,9 +262,10 @@ async function recompute(): Promise<void> {
         if (next) {
             let successorStart = next.release;
             let overlap = next.entry.parameters.overlapTime || us.parameters.overlapTime || 0;
-            // Even a ZERO-overlap deploy needs the dying behaviors - draining fast-write flushes
-            // by the release and writing through after it, or acknowledged data dies with us.
-            // Only the remap (the alternate-port middle window) needs an actual overlap.
+            // Tracked even for a ZERO-overlap deploy (the scheduled-switchover log must still
+            // fire); only the remap (the alternate-port middle window) needs an actual overlap.
+            // Flushing before the handoff needs nothing special here: the remap ends our valid
+            // window, and fast writes always flush ahead of their window's end.
             computed.dying = { successorStart, overlap };
             if (overlap > 0 && now < successorStart + overlap * REMAP_EXPIRE_FRACTION) {
                 const successorPid = next.entry.pid;
@@ -294,31 +303,31 @@ function iso(time: number): string {
 function logTransitions(prev: TakeoverComputed, next: TakeoverComputed): void {
     if (!prev.dying && next.dying) {
         let { successorStart, overlap } = next.dying;
-        console.log(`Deploy switchover scheduled: our successor starts at ${iso(successorStart)}; fast-write flushes drain by ${iso(successorStart + overlap * FLUSH_DEADLINE_FRACTION)}, fresh writes hand off at ${iso(successorStart + overlap * BOUNDARY_A_FRACTION)}, and we are killed at ${iso(successorStart + overlap)}`);
+        console.log(`${logPrefix()} Deploy switchover scheduled: our successor starts at ${iso(successorStart)}; fresh writes hand off at ${iso(successorStart + overlap * BOUNDARY_A_FRACTION)}, and we are killed at ${iso(successorStart + overlap)}`);
     }
     if (!prev.predecessorEnd && next.predecessorEnd) {
-        console.log(`We are a deploy successor: our predecessor holds the main port until ${iso(next.predecessorEnd)}`);
+        console.log(`${logPrefix()} We are a deploy successor: our predecessor holds the main port until ${iso(next.predecessorEnd)}`);
     }
     if (!prev.remap && next.remap) {
-        console.log(`Deploy switchover remap active: sources pointing at us are split so [${iso(next.remap.boundaryA)} .. ${iso(next.remap.boundaryB)}] routes to alternate port ${next.remap.altPort} (remap expires at ${iso(next.remap.expire)})`);
+        console.log(`${logPrefix()} Deploy switchover remap active: sources pointing at us are split so [${iso(next.remap.boundaryA)} .. ${iso(next.remap.boundaryB)}] routes to alternate port ${next.remap.altPort} (remap expires at ${iso(next.remap.expire)})`);
     }
     if (prev.remap && !next.remap) {
-        console.log(`Deploy switchover remap ended; running on the plain stored routing config`);
+        console.log(`${logPrefix()} Deploy switchover remap ended; running on the plain stored routing config`);
     }
 }
 
 /** Starts the takeover machinery. Port fallback (alternate port + registry + acquisition polling)
  *  works regardless; without a deploy timeline folder the switchover-specific parts (the remap,
- *  the flush deadline, the tighter acquisition pacing) simply stay inert. */
+ *  the tighter acquisition pacing) simply stay inert. */
 export async function initDeployTakeover(config: { domain: string; mainPort: number; storageFolder: string }): Promise<void> {
     initialized = config;
     ancestorPids = await getAncestorPids();
     timelineFolder = getTimelineFolder();
     let entries = await readTimelineEntries();
     if (!entries.length) {
-        console.log(`No deploy timeline entries in ${timelineFolder}; deploy takeover inactive until they appear (port fallback still works)`);
+        console.log(`${logPrefix()} No deploy timeline entries in ${timelineFolder}; deploy takeover inactive until they appear (port fallback still works)`);
     } else {
-        console.log(`Deploy timeline found at ${timelineFolder} (${entries.length} entries); our pid chain: ${[...ancestorPids].join(" -> ")}`);
+        console.log(`${logPrefix()} Deploy timeline found at ${timelineFolder} (${entries.length} entries); our pid chain: ${[...ancestorPids].join(" -> ")}`);
     }
     await recompute();
     let poll = setInterval(() => {
@@ -401,12 +410,10 @@ export function getTakeoverStamp(): string | undefined {
     return JSON.stringify(remap);
 }
 
-/** For the dying process: fast-write flush delays must never extend past this time, and after it
- *  fast writes flush immediately - so nothing is left in memory when the write window transfers. */
-export function getFlushDeadline(): number | undefined {
-    let dying = current.dying;
-    if (!dying) return undefined;
-    return dying.successorStart + dying.overlap * FLUSH_DEADLINE_FRACTION;
+/** The middle-window alternate port of an active remap. The OTHER process of the takeover lives on
+ *  this port on OUR machine (same disk!), so sources pointing at it are self, never sync targets. */
+export function getTakeoverAltPort(): number | undefined {
+    return current.remap?.altPort;
 }
 
 /** How long to wait between main-port acquisition attempts: tight around the predecessor's