sliftutils 1.7.85 → 1.7.87

package/storage/remoteStorage/blobStore.ts

@@ -5,7 +5,7 @@ import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import {
     IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, FindConfig, HostedConfig, SourceConfig, assertValidLastModified,
-    windowAcceptsWrites, SyncActivity, FULL_ROUTE, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, copyArchiveFile,
+    windowAcceptsWrites, windowsAcceptWrites, SyncActivity, FULL_ROUTE, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, copyArchiveFile,
 } from "../IArchives";
 import { ArchivesDisk, applyFindInfoShape } from "../ArchivesDisk";
 import { ArchivesBackblaze } from "../backblaze";
@@ -166,11 +166,11 @@ export type BlobSourceSpec = {
     identity: string;
     // See ArchivesSource.url
     url: string;
-    validWindow: [number, number];
+    validWindows: [number, number][];
     route?: [number, number];
     noFullSync?: boolean;
     // See ArchivesSource.intermediate
-    intermediate?: boolean;
+    intermediate?: string;
     // See ArchivesSource.sourceConfig (absent for the base disk source)
     sourceConfig?: SourceConfig;
     // Only called for sources that don't match an existing live slot
@@ -470,7 +470,7 @@ export class BlobStore implements IBucketStore {
             indexSize: this.mem.size,
             sources: this.sources.map((x, i) => ({
                 debugName: x.source.getDebugName(),
-                validWindow: x.validWindow,
+                validWindows: x.validWindows,
                 route: x.route,
                 noFullSync: x.noFullSync,
                 supportsChangesAfter: this.sourceStates[i].supportsChangesAfter,
@@ -530,14 +530,14 @@ export class BlobStore implements IBucketStore {
         if (entries) {
             this.entries = entries;
         }
-        let setWindow = (i: number, window: [number, number]) => {
-            let old = this.sources[i].validWindow;
-            if (old[0] === window[0] && old[1] === window[1]) return;
-            console.log(`Valid window changed for ${this.sources[i].source.getDebugName()} (store ${this.folder}): [${old.join(", ")}] -> [${window.join(", ")}]`);
-            this.sources[i].validWindow = window;
+        let setWindows = (i: number, windows: [number, number][]) => {
+            let old = this.sources[i].validWindows;
+            if (JSON.stringify(old) === JSON.stringify(windows)) return;
+            console.log(`Valid windows changed for ${this.sources[i].source.getDebugName()} (store ${this.folder}): ${JSON.stringify(old)} -> ${JSON.stringify(windows)}`);
+            this.sources[i].validWindows = windows;
         };
-        setWindow(0, specs[0].validWindow);
-        // Live slots pair with specs by identity, in order (the same endpoint can appear several times, e.g. one entry per window)
+        setWindows(0, specs[0].validWindows);
+        // Live slots pair with specs by identity. Each endpoint is now ONE spec carrying all its windows, so this is a 1:1 pairing; any leftover duplicate slots from before (an endpoint that used to be split into several slots) go unmatched below and are collapsed away.
         let liveByIdentity = new Map<string, number[]>();
         let originalLength = this.sources.length;
         for (let i = 1; i < originalLength; i++) {
@@ -556,7 +556,7 @@ export class BlobStore implements IBucketStore {
             let slot = liveByIdentity.get(spec.identity)?.shift();
             if (slot !== undefined) {
                 matched.add(slot);
-                setWindow(slot, spec.validWindow);
+                setWindows(slot, spec.validWindows);
                 let existing = this.sources[slot];
                 if (JSON.stringify(existing.route) !== JSON.stringify(spec.route)) {
                     console.log(`Route changed for ${existing.source.getDebugName()} (store ${this.folder}): ${JSON.stringify(existing.route)} -> ${JSON.stringify(spec.route)}`);
@@ -567,7 +567,7 @@ export class BlobStore implements IBucketStore {
                 continue;
             }
             let source = spec.create();
-            this.sources.push({ source, url: spec.url, validWindow: spec.validWindow, route: spec.route, noFullSync: spec.noFullSync, intermediate: spec.intermediate, sourceConfig: spec.sourceConfig, identity: spec.identity });
+            this.sources.push({ source, url: spec.url, validWindows: spec.validWindows, route: spec.route, noFullSync: spec.noFullSync, intermediate: spec.intermediate, sourceConfig: spec.sourceConfig, identity: spec.identity });
             this.sourceStates.push(newSourceState());
             this.sourceFileCounts.push(0);
             this.sourceByteCounts.push(0);
@@ -580,7 +580,7 @@ export class BlobStore implements IBucketStore {
             if (!this.isLive(i) || matched.has(i)) continue;
             this.removeSource(i);
         }
-        let deadline = this.sources[0].validWindow[1] - WINDOW_END_FLUSH_MARGIN;
+        let deadline = this.baseWriteWindowEnd() - WINDOW_END_FLUSH_MARGIN;
         let recapped = 0;
         for (let entry of this.overlay.values()) {
             if (entry.fastFlushAt <= deadline && entry.slowFlushAt <= deadline) continue;
@@ -921,10 +921,10 @@ export class BlobStore implements IBucketStore {
         }
     }
 
-    // An intermediate is a deploy switchover's temporary alternate port: once its window is past, the port is gone for good, so scanning it (or retrying a failed scan) can never succeed - it would just log errors forever
+    // An intermediate is a deploy switchover's temporary alternate port: once its window is past, the port is gone for good, so scanning it (or retrying a failed scan) can never succeed - it would just log errors forever. (Dead only once EVERY window has ended - an intermediate normally has just one.)
     private isDeadIntermediate(sourceIndex: number): boolean {
-        let { intermediate, validWindow } = this.sources[sourceIndex];
-        return !!intermediate && validWindow[1] <= Date.now();
+        let { intermediate, validWindows } = this.sources[sourceIndex];
+        return !!intermediate && validWindows.every(w => w[1] <= Date.now());
     }
 
     // Full metadata scan (size, writeTime, path) of one source, applied to the index. Returns the source's listing (path -> write time), which reconcileSource uses for the push direction.
@@ -987,9 +987,9 @@ export class BlobStore implements IBucketStore {
 
     // The push direction of synchronization: everything we know that the source is missing (or holds an older copy of) is written to it — including deletions, as tombstone writes. This is what heals a source whose background writes failed (e.g. it was down): the next scan sees what's missing and re-sends it. A failing file is skipped, not fatal (immutable targets are handled by forceSetImmutable, and one unreadable value must not stop the rest of the pass) - only a run of consecutive failures (the source itself is down) aborts until the next scan cycle.
     private async reconcileSource(sourceIndex: number, listing: Map<string, number>): Promise<void> {
-        let { source, validWindow, route } = this.sources[sourceIndex];
+        let { source, validWindows, route } = this.sources[sourceIndex];
         let state = this.sourceStates[sourceIndex];
-        let acceptsWrites = windowAcceptsWrites(validWindow);
+        let acceptsWrites = windowsAcceptWrites(validWindows);
         let targetSourcesListIndex = this.sourcesListIndexOfSlot(sourceIndex);
         let pushed = 0;
         let failed = 0;
@@ -1253,7 +1253,7 @@ export class BlobStore implements IBucketStore {
                 writeDelay = DEFAULT_FAST_WRITE_DELAY;
             }
             // The delay never extends past our own valid window's end (minus the margin, so the writes are on disk before the next window's source takes over - a deploy switchover is just this too, since its remap ends our window). Past that point fast writes write through immediately.
-            let deadline = this.sources[0].validWindow[1] - WINDOW_END_FLUSH_MARGIN;
+            let deadline = this.baseWriteWindowEnd() - WINDOW_END_FLUSH_MARGIN;
             if (writeDelay > 0 && Date.now() < deadline) {
                 // Own disk + our own storage servers (type-"remote") flush within MAX_REMOTE_FAST_BUFFER; the non-"remote" sources (backblaze) wait the full writeDelay
                 let fastFlushAt = Math.min(Date.now() + Math.min(writeDelay, MAX_REMOTE_FAST_BUFFER), deadline);
@@ -1266,11 +1266,21 @@ export class BlobStore implements IBucketStore {
         await this.writeToSources(key, data, writeTime);
     }
 
+    // The end of our own (base disk) write window that CONTAINS now - the deadline fast writes must flush before, so the next window's source has the data on handoff. The LATEST end among covering windows (overlapping windows hand off at the last one). No window contains now (an inert store, or a moment between our windows) -> 0, i.e. flush through immediately.
+    private baseWriteWindowEnd(): number {
+        let now = Date.now();
+        let end = 0;
+        for (let w of this.sources[0].validWindows) {
+            if (w[0] <= now && now < w[1]) end = Math.max(end, w[1]);
+        }
+        return end;
+    }
+
     private getWritableSources(config?: { ignoreWindow?: boolean }): number[] {
         let writable: number[] = [];
         for (let i = 0; i < this.sources.length; i++) {
             if (!this.isLive(i)) continue;
-            if (!config?.ignoreWindow && !windowAcceptsWrites(this.sources[i].validWindow)) continue;
+            if (!config?.ignoreWindow && !windowsAcceptWrites(this.sources[i].validWindows)) continue;
             writable.push(i);
         }
         return writable;
@@ -1406,7 +1416,7 @@ export class BlobStore implements IBucketStore {
             for (let i = 0; i < this.sources.length; i++) {
                 if (!this.isLive(i)) continue;
                 let sourceEntry = this.sources[i];
-                if (!windowAcceptsWrites(sourceEntry.validWindow)) continue;
+                if (!windowsAcceptWrites(sourceEntry.validWindows)) continue;
                 let source = sourceEntry.source;
                 if (!(source instanceof ArchivesBackblaze)) continue;
                 void source.del(key).catch((e: Error) => {