sliftutils 1.7.40 → 1.7.42

package/storage/remoteStorage/sourceWrapper.d.ts

@@ -21,6 +21,8 @@ export declare class SourceWrapper {
     }): Promise<SourceWrapper>;
     getDebugName(): string;
     isConnected(): boolean;
+    /** A source whose window has passed is never read from or written to (see the window checks in ArchivesChain), and an intermediate only exists for the minutes of a deploy switchover - on either, being unreachable is expected, not a problem to report. */
+    private isConnectionProblemWorthReporting;
     /** Call after a request failed while isConnected() was false: starts (if not already running) the background reconnect loop. Never blocks - the failed request still throws. */
     noteFailure(): void;
     private reconnectLoop;

package/storage/remoteStorage/sourceWrapper.ts

@@ -101,11 +101,19 @@ export class SourceWrapper {
         return this.remote.isConnected();
     }
 
+    /** A source whose window has passed is never read from or written to (see the window checks in ArchivesChain), and an intermediate only exists for the minutes of a deploy switchover - on either, being unreachable is expected, not a problem to report. */
+    private isConnectionProblemWorthReporting(): boolean {
+        if (this.config.intermediate) return false;
+        return this.config.validWindow[1] > Date.now();
+    }
+
     /** Call after a request failed while isConnected() was false: starts (if not already running) the background reconnect loop. Never blocks - the failed request still throws. */
     public noteFailure(): void {
         if (!this.background || this.disposed || this.reconnectRunning) return;
         if (this.isConnected()) return;
-        console.error(`Cannot connect to storage ${this.getDebugName()}`);
+        if (this.isConnectionProblemWorthReporting()) {
+            console.error(`Cannot connect to storage ${this.getDebugName()}`);
+        }
         this.reconnectRunning = true;
         void this.reconnectLoop();
     }
@@ -126,7 +134,9 @@ export class SourceWrapper {
             } catch (e) {
                 // Even a failing call (e.g. access denied) proves the connection is back
                 if (this.isConnected()) break;
-                console.warn(`Cannot connect to storage ${this.getDebugName()}, retrying in ${Math.round(retryDelay / 1000)}s. ${(e as Error).stack ?? e}`);
+                if (this.isConnectionProblemWorthReporting()) {
+                    console.warn(`Cannot connect to storage ${this.getDebugName()}, retrying in ${Math.round(retryDelay / 1000)}s. ${(e as Error).stack ?? e}`);
+                }
             }
             retryDelay = Math.min(RETRY_MAX_DELAY, retryDelay * RETRY_GROWTH);
         }
@@ -182,6 +192,8 @@ export class SourceWrapper {
     public startPinging(): void {
         const remote = this.remote;
         if (!remote || this.pingTimer || this.disposed) return;
+        // Latency only decides which shard a variable-shard write materializes into, and a source whose window has passed never receives writes - so measuring it is pointless traffic
+        if (this.config.validWindow[1] <= Date.now()) return;
         let measure = async () => {
             let start = Date.now();
             try {

package/storage/remoteStorage/storageServerState.ts

@@ -756,40 +756,14 @@ async function propagateRoutingConfig(loaded: LoadedBucket, next: RemoteConfig,
     }
 }
 
-/** Every bucket that exists on this server's disk, as account/bucketName keys. */
-async function getAllBucketKeys(): Promise<string[]> {
-    let root = path.join(getStorageServerConfig().folder, "buckets2");
-    let accounts: string[];
-    try {
-        accounts = await fs.promises.readdir(root);
-    } catch {
-        return [];
-    }
-    let keys: string[] = [];
-    for (let account of accounts) {
-        let names: string[];
-        try {
-            names = await fs.promises.readdir(path.join(root, account));
-        } catch {
-            continue;
-        }
-        for (let bucketName of names) {
-            keys.push(`${account}/${bucketName}`);
-        }
-    }
-    return keys;
-}
-
 async function maintainIntermediates(): Promise<void> {
     let { domain, port } = getStorageServerConfig();
     let takeover = getTakeoverIntermediate();
-    // Every bucket on disk, not just the ones already in memory: a successor starts with nothing loaded, and a bucket whose config we never touch would keep sending its writes to the process that is about to die.
-    for (let key of await getAllBucketKeys()) {
-        let slash = key.indexOf("/");
-        let loaded = await getLoadedBucket(key.slice(0, slash), key.slice(slash + 1)).catch((e: Error) => {
-            console.error(`Loading bucket ${key} to maintain its switchover windows failed: ${e.stack ?? e}`);
-            return undefined;
-        });
+    // ONLY buckets already in memory. A switchover must never load a bucket: loading starts its synchronization, and buckets nothing has touched (legacy ones especially) have no writes to hand off in the first place. One that does get used mid-switchover loads on that access, and the next pass gives it its window.
+    for (let key of [...buckets.keys()]) {
+        let loadedPromise = buckets.get(key);
+        if (!loadedPromise) continue;
+        let loaded = await loadedPromise.catch(() => undefined);
         if (!loaded) continue;
         let expired = expireIntermediateSources(loaded.routing, Date.now());
         if (JSON.stringify(expired) !== JSON.stringify(loaded.routing)) {