sliftutils 1.7.55 → 1.7.57

package/storage/remoteStorage/blobStore.ts

@@ -126,6 +126,8 @@ export type BlobSourceSpec = {
     validWindow: [number, number];
     route?: [number, number];
     noFullSync?: boolean;
+    // See ArchivesSource.intermediate
+    intermediate?: boolean;
     // Only called for sources that don't match an existing live slot
     create: () => IArchives;
 };
@@ -138,7 +140,7 @@ function newScanTally(): ScanTally {
 }
 function formatScanTally(tally: ScanTally, total: number): string {
     let pct = (n: number) => `${Math.round(n / Math.max(total, 1) * 1000) / 10}%`;
-    return `${tally.new} new paths (${pct(tally.new)}), ${tally.updated} newer writes (${pct(tally.updated)}), ${tally.tombstone} deletions (${pct(tally.tombstone)}), ${tally.unchanged} unchanged (${pct(tally.unchanged)}), ${tally.filtered} outside window/route (${pct(tally.filtered)})`;
+    return `${tally.new} new paths (${pct(tally.new)}), ${tally.updated} newer writes (${pct(tally.updated)}), ${tally.tombstone} deletions (${pct(tally.tombstone)}), ${tally.unchanged} unchanged (${pct(tally.unchanged)}), ${tally.filtered} outside route (${pct(tally.filtered)})`;
 }
 
 function newSourceState(): SourceState {
@@ -359,7 +361,7 @@ export class BlobStore implements IBucketStore {
                 continue;
             }
             let source = spec.create();
-            this.sources.push({ source, url: spec.url, validWindow: spec.validWindow, route: spec.route, noFullSync: spec.noFullSync, identity: spec.identity });
+            this.sources.push({ source, url: spec.url, validWindow: spec.validWindow, route: spec.route, noFullSync: spec.noFullSync, intermediate: spec.intermediate, identity: spec.identity });
             this.sourceStates.push(newSourceState());
             this.sourceFileCounts.push(0);
             this.sourceByteCounts.push(0);
@@ -525,14 +527,23 @@ export class BlobStore implements IBucketStore {
         let state = this.sourceStates[sourceIndex];
         // Read live for every pass, not captured - updateSources can change it while loops run
         let noFullSync = () => this.sources[sourceIndex].noFullSync;
+        let intermediate = this.sources[sourceIndex].intermediate;
         let listing: Map<string, number> | undefined;
         while (!this.stopped.stop && !state.stopped.stop) {
+            if (this.isDeadIntermediate(sourceIndex)) {
+                console.log(`Not scanning sync source ${source.getDebugName()} (store ${this.folder}): it is an intermediate whose window already ended`);
+                break;
+            }
             try {
                 let config = await source.getConfig();
                 state.supportsChangesAfter = !!config.supportsChangesAfter;
                 listing = await this.scanSource(sourceIndex);
                 break;
             } catch (e) {
+                if (intermediate) {
+                    console.error(`Initial scan of intermediate sync source ${source.getDebugName()} (store ${this.folder}) failed; not retrying (intermediates are temporary switchover ports): ${(e as Error).stack ?? e}`);
+                    break;
+                }
                 console.error(`Initial scan of sync source ${source.getDebugName()} failed, retrying:`, e);
                 await delay(SCAN_RETRY_DELAY);
             }
@@ -540,6 +551,8 @@ export class BlobStore implements IBucketStore {
         state.scanComplete = true;
         state.initialScan.resolve(undefined);
         if (this.stopped.stop || state.stopped.stop) return;
+        // An intermediate that never produced a listing is dead or dying - polling and copying from it would only log errors
+        if (intermediate && !listing) return;
         if (listing) {
             await this.reconcileSource(sourceIndex, listing);
         }
@@ -552,16 +565,19 @@ export class BlobStore implements IBucketStore {
         }
         if (state.supportsChangesAfter) {
             runInfinitePoll(CHANGES_POLL_INTERVAL, async () => {
+                if (this.isDeadIntermediate(sourceIndex)) return;
                 await this.pollChanges(sourceIndex);
                 if (!noFullSync()) await this.copySourceFiles(sourceIndex);
             }, state.stopped);
             // Change polls only show what the source HAS, never what it's missing, so pushes run on the full-rescan cadence (findInfo on an index-backed source is cheap)
             runInfinitePoll(FULL_RESCAN_INTERVAL, async () => {
+                if (this.isDeadIntermediate(sourceIndex)) return;
                 let files = await source.findInfo("");
                 await this.reconcileSource(sourceIndex, new Map(files.map(x => [x.path, x.createTime])));
             }, state.stopped);
         } else {
             runInfinitePoll(FULL_RESCAN_UNINDEXED_INTERVAL, async () => {
+                if (this.isDeadIntermediate(sourceIndex)) return;
                 let rescan = await this.scanSource(sourceIndex);
                 await this.reconcileSource(sourceIndex, rescan);
                 if (!noFullSync()) await this.copySourceFiles(sourceIndex);
@@ -569,9 +585,15 @@ export class BlobStore implements IBucketStore {
         }
     }
 
+    // An intermediate is a deploy switchover's temporary alternate port: once its window is past, the port is gone for good, so scanning it (or retrying a failed scan) can never succeed - it would just log errors forever
+    private isDeadIntermediate(sourceIndex: number): boolean {
+        let { intermediate, validWindow } = this.sources[sourceIndex];
+        return !!intermediate && validWindow[1] <= Date.now();
+    }
+
     // Full metadata scan (size, writeTime, path) of one source, applied to the index. Returns the source's listing (path -> write time), which reconcileSource uses for the push direction.
     private async scanSource(sourceIndex: number): Promise<Map<string, number>> {
-        let { source, validWindow, route } = this.sources[sourceIndex];
+        let { source, route } = this.sources[sourceIndex];
         let state = this.sourceStates[sourceIndex];
         let scanStart = Date.now();
         let activity: SyncActivity = { type: "metadataScan", sourceDebugName: source.getDebugName(), startTime: scanStart };
@@ -614,9 +636,8 @@ export class BlobStore implements IBucketStore {
                 removedFromIndex++;
                 continue;
             }
-            // Counted only when the source SHOULD hold the entry (its window/route match) - these are what the reconcile pass pushes to it
+            // Counted only when the source SHOULD hold the entry (its route matches) - these are what the reconcile pass pushes to it (which also ignores the valid window: synchronization moves existing values, the window only routes fresh writes)
             if (entry.size === 0 || key === ROUTING_FILE) continue;
-            if (entry.writeTime < validWindow[0] || entry.writeTime > validWindow[1]) continue;
             if (!routeContains(route, getRoute(key))) continue;
             missingOnSource++;
         }
@@ -689,10 +710,8 @@ export class BlobStore implements IBucketStore {
             // The routing config is NEVER pulled from other sources - it only ever arrives as an explicit, version-validated write, and is only ever read off our own disk. Route and valid-window filters can't possibly apply to it either: it is the file DEFINING them, so filtering it would mean certain sources could never have their routing config updated, ever.
             if (sourceIndex !== 0) return "filtered";
         } else {
-            let { validWindow, route } = this.sources[sourceIndex];
-            let [windowStart, windowEnd] = validWindow;
-            if (file.createTime < windowStart || file.createTime > windowEnd) return "filtered";
-            // A partially-overlapping shard's listing includes keys outside our route; ignore them
+            // The valid window is deliberately NOT applied here: it decides where WRITES route, but a scan is us asking a source what it already holds - existing values synchronize regardless of the window (the same reasoning that lets synchronization ignore the immutable flag). Only the route filters: a partially-overlapping shard's listing legitimately includes keys that aren't ours.
+            let { route } = this.sources[sourceIndex];
             if (!routeContains(route, getRoute(file.path))) return "filtered";
         }
         let existing = this.mem.get(file.path);

package/storage/remoteStorage/createArchives.d.ts

@@ -86,7 +86,7 @@ export declare class ArchivesChain implements IArchives {
         getNextData(): Promise<Buffer | undefined>;
     }): Promise<void>;
     getURL(path: string): Promise<string>;
-    /** Every URL that could serve this path, in source order. Empty when no public source covers the path's route. */
+    /** Every URL that could serve this path: public sources matching both the path's route and the current valid window. The first is the write node's (first matching source in config order, see runWrite - the one guaranteed current); the rest are ranked fastest-first by measured latency. Empty when none qualify. */
     getURLs(path: string): Promise<string[]>;
     dispose(): void;
 }

package/storage/remoteStorage/createArchives.ts

@@ -541,6 +541,8 @@ export class ArchivesChain implements IArchives {
     public async waitingForAccess(): Promise<{ link: string; machineId: string; ip: string } | undefined> {
         let state = await this.getState();
         for (let source of state.sources) {
+            // A source whose window has passed is never read or written again, so its access state is irrelevant - and asking a dead intermediate would just hang or throw. Future windows DO matter: access should be granted before their window starts.
+            if (source.config.validWindow[1] <= Date.now()) continue;
             if (source.api instanceof ArchivesRemote) {
                 let waiting = await source.api.waitingForAccess();
                 if (waiting) return waiting;
@@ -800,19 +802,22 @@ export class ArchivesChain implements IArchives {
         return urls[0];
     }
 
-    /** Every URL that could serve this path, in source order. Empty when no public source covers the path's route. */
+    /** Every URL that could serve this path: public sources matching both the path's route and the current valid window. The first is the write node's (first matching source in config order, see runWrite - the one guaranteed current); the rest are ranked fastest-first by measured latency. Empty when none qualify. */
     public async getURLs(path: string): Promise<string[]> {
         let state = await this.getState();
         let route = getRoute(path);
-        let providers: IArchives[] = [];
+        let candidates: { source: SourceWrapper; provider: IArchives }[] = [];
         for (let source of state.sources) {
             if (!(source.config.public ?? true)) continue;
             if (!routeContains(source.config.route, route)) continue;
+            if (!configWindowCurrent(source.config)) continue;
             let provider = source.url || source.api;
             if (!provider) continue;
-            providers.push(provider);
+            candidates.push({ source, provider });
         }
-        let urls = await Promise.all(providers.map(provider => provider.getURL(path)));
+        let rest = candidates.slice(1);
+        sort(rest, x => x.source.getLatency());
+        let urls = await Promise.all([...candidates.slice(0, 1), ...rest].map(x => x.provider.getURL(path)));
         return [...new Set(urls)];
     }