sliftutils 1.7.56 → 1.7.58

package/storage/remoteStorage/blobStore.ts

@@ -140,7 +140,7 @@ function newScanTally(): ScanTally {
 }
 function formatScanTally(tally: ScanTally, total: number): string {
     let pct = (n: number) => `${Math.round(n / Math.max(total, 1) * 1000) / 10}%`;
-    return `${tally.new} new paths (${pct(tally.new)}), ${tally.updated} newer writes (${pct(tally.updated)}), ${tally.tombstone} deletions (${pct(tally.tombstone)}), ${tally.unchanged} unchanged (${pct(tally.unchanged)}), ${tally.filtered} outside window/route (${pct(tally.filtered)})`;
+    return `${tally.new} new paths (${pct(tally.new)}), ${tally.updated} newer writes (${pct(tally.updated)}), ${tally.tombstone} deletions (${pct(tally.tombstone)}), ${tally.unchanged} unchanged (${pct(tally.unchanged)}), ${tally.filtered} outside route (${pct(tally.filtered)})`;
 }
 
 function newSourceState(): SourceState {
@@ -593,7 +593,7 @@ export class BlobStore implements IBucketStore {
 
     // Full metadata scan (size, writeTime, path) of one source, applied to the index. Returns the source's listing (path -> write time), which reconcileSource uses for the push direction.
     private async scanSource(sourceIndex: number): Promise<Map<string, number>> {
-        let { source, validWindow, route } = this.sources[sourceIndex];
+        let { source, route } = this.sources[sourceIndex];
         let state = this.sourceStates[sourceIndex];
         let scanStart = Date.now();
         let activity: SyncActivity = { type: "metadataScan", sourceDebugName: source.getDebugName(), startTime: scanStart };
@@ -636,9 +636,8 @@ export class BlobStore implements IBucketStore {
                 removedFromIndex++;
                 continue;
             }
-            // Counted only when the source SHOULD hold the entry (its window/route match) - these are what the reconcile pass pushes to it
+            // Counted only when the source SHOULD hold the entry (its route matches) - these are what the reconcile pass pushes to it (which also ignores the valid window: synchronization moves existing values, the window only routes fresh writes)
             if (entry.size === 0 || key === ROUTING_FILE) continue;
-            if (entry.writeTime < validWindow[0] || entry.writeTime > validWindow[1]) continue;
             if (!routeContains(route, getRoute(key))) continue;
             missingOnSource++;
         }
@@ -711,10 +710,8 @@ export class BlobStore implements IBucketStore {
             // The routing config is NEVER pulled from other sources - it only ever arrives as an explicit, version-validated write, and is only ever read off our own disk. Route and valid-window filters can't possibly apply to it either: it is the file DEFINING them, so filtering it would mean certain sources could never have their routing config updated, ever.
             if (sourceIndex !== 0) return "filtered";
         } else {
-            let { validWindow, route } = this.sources[sourceIndex];
-            let [windowStart, windowEnd] = validWindow;
-            if (file.createTime < windowStart || file.createTime > windowEnd) return "filtered";
-            // A partially-overlapping shard's listing includes keys outside our route; ignore them
+            // The valid window is deliberately NOT applied here: it decides where WRITES route, but a scan is us asking a source what it already holds - existing values synchronize regardless of the window (the same reasoning that lets synchronization ignore the immutable flag). Only the route filters: a partially-overlapping shard's listing legitimately includes keys that aren't ours.
+            let { route } = this.sources[sourceIndex];
             if (!routeContains(route, getRoute(file.path))) return "filtered";
         }
         let existing = this.mem.get(file.path);

package/storage/remoteStorage/createArchives.d.ts

@@ -14,6 +14,7 @@ export declare class ArchivesChain implements IArchives {
     private configured;
     private activeConfig;
     private statePromise;
+    private latestState;
     private initRetryDelay;
     private initRetryTimer;
     private pollTimer;
@@ -86,8 +87,10 @@ export declare class ArchivesChain implements IArchives {
         getNextData(): Promise<Buffer | undefined>;
     }): Promise<void>;
     getURL(path: string): Promise<string>;
-    /** Every URL that could serve this path, in source order: public sources matching both the path's route and the current valid window. Empty when none qualify. */
+    /** Every URL that could serve this path: public sources matching both the path's route and the current valid window. The first is the write node's (first matching source in config order, see runWrite - the one guaranteed current); the rest are ranked fastest-first by measured latency. Empty when none qualify. */
     getURLs(path: string): Promise<string[]>;
+    /** getURLs, but after the one await (initialization) the returned function is synchronous: everything underneath - route hashing, window checks, latencies, URL building - is synchronous, and the closure always reads the newest adopted config, so it stays correct across config refreshes. */
+    getGetURLs(): Promise<(path: string) => string[]>;
     dispose(): void;
 }
 export declare function createArchives(config: RemoteConfig | RemoteConfigBase, options?: ArchivesChainOptions): ArchivesChain;

package/storage/remoteStorage/createArchives.ts

@@ -8,7 +8,7 @@ import {
 import {
     ROUTING_FILE, getConfigVersion, parseHostedUrl, parseBackblazeUrl,
     normalizeRemoteConfig, normalizeSource, serializeRemoteConfig,
-    getRoute, routeContains, parseVariableRoute, getBucketBaseUrl,
+    getRoute, routeContains, parseVariableRoute, getBucketBaseUrl, buildFileUrl,
 } from "./remoteConfig";
 import { ArchivesUrl } from "./ArchivesUrl";
 import { resolveIntermediateSources } from "./intermediateSources";
@@ -100,6 +100,8 @@ export class ArchivesChain implements IArchives {
     private configured: RemoteConfig;
     private activeConfig: RemoteConfig;
     private statePromise: Promise<ChainState> | undefined;
+    // The resolved state, for synchronous access (getGetURLs) - always the newest adopted config, where statePromise may briefly lag during a rebuild
+    private latestState: ChainState | undefined;
     private initRetryDelay = RETRY_START_DELAY;
     private initRetryTimer: ReturnType<typeof setTimeout> | undefined;
     private pollTimer: ReturnType<typeof setInterval> | undefined;
@@ -249,7 +251,9 @@ export class ArchivesChain implements IArchives {
             }
             this.activeConfig = active;
             this.startConfigPoll();
-            return { config: active, sources };
+            let state: ChainState = { config: active, sources };
+            this.latestState = state;
+            return state;
         } finally {
             for (let fetch of fetches) {
                 fetch.probe.dispose();
@@ -366,7 +370,9 @@ export class ArchivesChain implements IArchives {
             }
         }
         this.activeConfig = latest;
-        this.statePromise = Promise.resolve({ config: latest, sources });
+        let newState: ChainState = { config: latest, sources };
+        this.latestState = newState;
+        this.statePromise = Promise.resolve(newState);
     }
 
     private lastAvailabilityRecheck = 0;
@@ -802,21 +808,29 @@ export class ArchivesChain implements IArchives {
         return urls[0];
     }
 
-    /** Every URL that could serve this path, in source order: public sources matching both the path's route and the current valid window. Empty when none qualify. */
+    /** Every URL that could serve this path: public sources matching both the path's route and the current valid window. The first is the write node's (first matching source in config order, see runWrite - the one guaranteed current); the rest are ranked fastest-first by measured latency. Empty when none qualify. */
     public async getURLs(path: string): Promise<string[]> {
-        let state = await this.getState();
-        let route = getRoute(path);
-        let providers: IArchives[] = [];
-        for (let source of state.sources) {
-            if (!(source.config.public ?? true)) continue;
-            if (!routeContains(source.config.route, route)) continue;
-            if (!configWindowCurrent(source.config)) continue;
-            let provider = source.url || source.api;
-            if (!provider) continue;
-            providers.push(provider);
-        }
-        let urls = await Promise.all(providers.map(provider => provider.getURL(path)));
-        return [...new Set(urls)];
+        return (await this.getGetURLs())(path);
+    }
+
+    /** getURLs, but after the one await (initialization) the returned function is synchronous: everything underneath - route hashing, window checks, latencies, URL building - is synchronous, and the closure always reads the newest adopted config, so it stays correct across config refreshes. */
+    public async getGetURLs(): Promise<(path: string) => string[]> {
+        let initialState = await this.getState();
+        return (path: string) => {
+            let state = this.latestState || initialState;
+            let route = getRoute(path);
+            let sources: SourceWrapper[] = [];
+            for (let source of state.sources) {
+                if (!(source.config.public ?? true)) continue;
+                if (!routeContains(source.config.route, route)) continue;
+                if (!configWindowCurrent(source.config)) continue;
+                sources.push(source);
+            }
+            let rest = sources.slice(1);
+            sort(rest, x => x.getLatency());
+            let urls = [...sources.slice(0, 1), ...rest].map(x => buildFileUrl(getBucketBaseUrl(x.config.url), path));
+            return [...new Set(urls)];
+        };
     }
 
     public dispose(): void {