sliftutils 1.7.91 → 1.7.92

package/index.d.ts

@@ -2257,6 +2257,7 @@ declare module "sliftutils/storage/IArchives" {
     };
     export declare const STORAGE_WRONG_VALID_WINDOW = "REMOTE_STORAGE_WRONG_VALID_WINDOW_a7c1f04e";
     export declare const STORAGE_WRONG_ROUTE = "REMOTE_STORAGE_WRONG_ROUTE_c94d2e17";
+    export declare const STORAGE_NOT_CONFIGURED = "REMOTE_STORAGE_NOT_CONFIGURED_e51b7d92";
     export declare const FULL_ROUTE: [number, number];
     export declare const VARIABLE_SHARD = "VARIABLE_SHARD_f0234jfah08fgyhfgyssdds83nmp";
     export declare function windowAcceptsWrites(validWindow: [number, number] | undefined): boolean;
@@ -3303,6 +3304,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
         /** What this store is configured to be. It owns this: the routing config is a file IN the store, so the store reads it, applies it to itself, and re-applies it whenever the file changes - by our own write, or by a peer's copy arriving through synchronization. */
         storeConfig: StoreConfig;
         private appliedRoutingVersion;
+        private appliedRouting;
         init: {
             (): Promise<void>;
             reset(): void;
@@ -3489,6 +3491,8 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
          * reason it aimed here is that its config was stale.
          */
         private assertWriteTarget;
+        /** Exactly why this store has no configuration entries - which of the three possible reasons it is, with the values that decided it, because "not configured" alone is undiagnosable. */
+        private unconfiguredDetail;
         /**
          * Whether a routing config may be written here. Two rules, and this is the one place either is
          * applied - a config only ever enters the system through a write, so a config that got in is a

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.91",
+    "version": "1.7.92",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/IArchives.d.ts

@@ -163,6 +163,7 @@ export type ArchivesSource = {
 };
 export declare const STORAGE_WRONG_VALID_WINDOW = "REMOTE_STORAGE_WRONG_VALID_WINDOW_a7c1f04e";
 export declare const STORAGE_WRONG_ROUTE = "REMOTE_STORAGE_WRONG_ROUTE_c94d2e17";
+export declare const STORAGE_NOT_CONFIGURED = "REMOTE_STORAGE_NOT_CONFIGURED_e51b7d92";
 export declare const FULL_ROUTE: [number, number];
 export declare const VARIABLE_SHARD = "VARIABLE_SHARD_f0234jfah08fgyhfgyssdds83nmp";
 export declare function windowAcceptsWrites(validWindow: [number, number] | undefined): boolean;

package/storage/IArchives.ts

@@ -218,6 +218,9 @@ export const STORAGE_WRONG_VALID_WINDOW = "REMOTE_STORAGE_WRONG_VALID_WINDOW_a7c
 // Error marker a server includes when a freshly-stamped write's key routes outside the shards this server handles (the client's config disagrees with the server's - clients re-resolve once)
 export const STORAGE_WRONG_ROUTE = "REMOTE_STORAGE_WRONG_ROUTE_c94d2e17";
 
+// Error marker a server includes when a write reaches a store whose own routing config has NO entry for it on that server (or that has no routing config at all) - accepting the write would store data nothing scans or reconciles. Clients re-resolve once, exactly like the markers above: the usual cause is the client running a different config than the store.
+export const STORAGE_NOT_CONFIGURED = "REMOTE_STORAGE_NOT_CONFIGURED_e51b7d92";
+
 export const FULL_ROUTE: [number, number] = [0, 1];
 
 // A key containing this sentinel doesn't have a fixed shard: setVariableShard picks the (lowest latency, up) write shard, appends "_<value in the shard's route>" directly after the sentinel, and returns the materialized key. getRoute treats that suffix as a complete route override.

package/storage/remoteStorage/blobStore.d.ts

@@ -67,6 +67,7 @@ export declare class BlobStore {
     /** What this store is configured to be. It owns this: the routing config is a file IN the store, so the store reads it, applies it to itself, and re-applies it whenever the file changes - by our own write, or by a peer's copy arriving through synchronization. */
     storeConfig: StoreConfig;
     private appliedRoutingVersion;
+    private appliedRouting;
     init: {
         (): Promise<void>;
         reset(): void;
@@ -253,6 +254,8 @@ export declare class BlobStore {
      * reason it aimed here is that its config was stale.
      */
     private assertWriteTarget;
+    /** Exactly why this store has no configuration entries - which of the three possible reasons it is, with the values that decided it, because "not configured" alone is undiagnosable. */
+    private unconfiguredDetail;
     /**
      * Whether a routing config may be written here. Two rules, and this is the one place either is
      * applied - a config only ever enters the system through a write, so a config that got in is a

package/storage/remoteStorage/blobStore.ts

@@ -3,7 +3,7 @@ import { lazy } from "socket-function/src/caching";
 import { timeInMinute, sort } from "socket-function/src/misc";
 import {
     IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, FindConfig, HostedConfig, RemoteConfig, SourceConfig, assertValidLastModified,
-    windowsAcceptWrites, SyncActivity, FULL_ROUTE, FULL_VALID_WINDOW, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE,
+    windowsAcceptWrites, SyncActivity, FULL_ROUTE, FULL_VALID_WINDOW, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, STORAGE_NOT_CONFIGURED,
 } from "../IArchives";
 import { ArchivesDisk, applyFindInfoShape } from "../ArchivesDisk";
 import { ROUTING_FILE, getRoute, routeContains, routeIntersection, parseRoutingData, assertValidRemoteConfig, getConfigVersion, sourceIdentity, sourcePersistentUrl } from "./remoteConfig";
@@ -122,6 +122,8 @@ export class BlobStore {
     public storeConfig: StoreConfig;
     // The routing config this store last applied, so a write of an identical (or older) one is not re-applied
     private appliedRoutingVersion = -1;
+    // ...and the config itself, so a rejection can say exactly what the store IS running instead of guessing
+    private appliedRouting: RemoteConfig | undefined;
 
     // #region Main interface
 
@@ -161,9 +163,15 @@ export class BlobStore {
         if (routing && !entries.length) {
             console.warn(`The routing config in store ${this.folder} (version ${getConfigVersion(routing)}) does not name this store (${JSON.stringify(this.storeName)}) as one of ours, so it stays a disk-only store`);
         }
+        let previousVersion = this.appliedRoutingVersion;
+        let newVersion = routing && getConfigVersion(routing) || -1;
+        if (newVersion !== previousVersion) {
+            console.log(`Store ${JSON.stringify(this.storeName)} (folder ${this.folder}) adopted routing config version ${previousVersion} -> ${newVersion}: ${entries.length} entr${entries.length === 1 && "y" || "ies"} of ours ${JSON.stringify(entries.map(x => ({ validWindow: x.validWindow, route: x.route || FULL_ROUTE })))}, ${peers.length} peer(s)`);
+        }
         this.storeConfig.update(entries);
         this.updateSources(this.planSources(peers));
-        this.appliedRoutingVersion = routing && getConfigVersion(routing) || -1;
+        this.appliedRoutingVersion = newVersion;
+        this.appliedRouting = routing;
         if (routing) {
             this.config?.onRoutingApplied?.(routing);
         }
@@ -790,8 +798,9 @@ export class BlobStore {
      */
     private assertWriteTarget(key: string, route: number, lastModified: number | undefined): void {
         if (!this.storeConfig.all().length) {
-            logWrongTargetRejection(`Rejecting write of ${JSON.stringify(key)} (store ${this.folder}): this server has no entry in the bucket's routing config, so nothing would ever scan or reconcile what it stored`);
-            throw new Error(`${STORAGE_WRONG_VALID_WINDOW} This server is not in the routing config for this bucket, so it cannot accept writes (store ${this.folder}). Re-resolve the currently valid source and retry.`);
+            let detail = this.unconfiguredDetail();
+            logWrongTargetRejection(`Rejecting write of ${JSON.stringify(key)}: ${detail}`);
+            throw new Error(`${STORAGE_NOT_CONFIGURED} Cannot write ${JSON.stringify(key)}: ${detail} Data written to a store with no configuration entry would never be scanned or reconciled, so accepting it would silently lose it. Re-resolve the currently valid source and retry - or, if this store is genuinely meant to take this write, write the bucket's routing config with an entry naming it on this server.`);
         }
         if (!lastModified) {
             this.assertFreshWriteTarget(key, Date.now(), route);
@@ -803,6 +812,21 @@ export class BlobStore {
         throw new Error(`${STORAGE_WRONG_ROUTE} This store does not handle route ${route} (key ${JSON.stringify(key)}, our routes: ${JSON.stringify(this.storeConfig.all().map(x => x.route || FULL_ROUTE))}, store ${this.folder}). Re-resolve the source for this key and retry.`);
     }
 
+    /** Exactly why this store has no configuration entries - which of the three possible reasons it is, with the values that decided it, because "not configured" alone is undiagnosable. */
+    private unconfiguredDetail(): string {
+        let routing = this.appliedRouting;
+        let identity = `store ${JSON.stringify(this.storeName)} (folder ${this.folder})`;
+        if (!routing) {
+            return `${identity} has no routing config at all - nothing has ever written ${JSON.stringify(ROUTING_FILE)} into it.`;
+        }
+        let objects = routing.sources.filter(x => typeof x !== "string") as SourceConfig[];
+        let named = objects.filter(x => x.name === this.storeName);
+        if (!named.length) {
+            return `${identity} is running routing config version ${getConfigVersion(routing)}, which has no entry named ${JSON.stringify(this.storeName)} (its entries are named ${JSON.stringify(objects.map(x => x.name))}).`;
+        }
+        return `${identity} is running routing config version ${getConfigVersion(routing)}, whose ${named.length} entr${named.length === 1 && "y" || "ies"} named ${JSON.stringify(this.storeName)} point(s) at ${JSON.stringify(named.map(x => x.url))} - none of which is THIS server, so the config says this store lives elsewhere.`;
+    }
+
     /**
      * Whether a routing config may be written here. Two rules, and this is the one place either is
      * applied - a config only ever enters the system through a write, so a config that got in is a
@@ -826,7 +850,7 @@ export class BlobStore {
         let timeValid = this.storeConfig.all().filter(x => writeTime >= x.validWindow[0] && writeTime < x.validWindow[1]);
         if (!timeValid.length) {
             logWrongTargetRejection(`Rejecting fresh write of ${JSON.stringify(key)} (store ${this.folder}): writeTime ${writeTime} (${new Date(writeTime).toISOString()}) is outside all our valid windows ${JSON.stringify(this.storeConfig.all().map(x => x.validWindow))} (a switchover moved the write target)`);
-            throw new Error(`${STORAGE_WRONG_VALID_WINDOW} This store is not a valid write target at ${writeTime} (our valid windows: ${JSON.stringify(this.storeConfig.all().map(x => x.validWindow))}, store ${this.folder}). Re-resolve the currently valid source and retry.`);
+            throw new Error(`${STORAGE_WRONG_VALID_WINDOW} Cannot write ${JSON.stringify(key)}: its write time ${writeTime} (${new Date(writeTime).toISOString()}) is outside every valid window of store ${JSON.stringify(this.storeName)} (windows: ${JSON.stringify(this.storeConfig.all().map(x => x.validWindow))}, folder ${this.folder}) - a switchover moved the write target. Re-resolve the currently valid source and retry.`);
         }
         if (!timeValid.some(x => routeContains(x.route, route))) {
             logWrongTargetRejection(`Rejecting fresh write of ${JSON.stringify(key)} (store ${this.folder}): route ${route} is outside our routes ${JSON.stringify(timeValid.map(x => x.route || FULL_ROUTE))} at writeTime ${writeTime} (the client's shard config is stale)`);

package/storage/remoteStorage/createArchives.ts

@@ -3,7 +3,7 @@ import { delay } from "socket-function/src/batching";
 import {
     IArchives, RemoteConfig, RemoteConfigBase, SourceConfig,
     ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig, STORAGE_WRONG_VALID_WINDOW,
-    STORAGE_WRONG_ROUTE, FULL_ROUTE, VARIABLE_SHARD, LARGE_SET_THRESHOLD, bufferChunkStream,
+    STORAGE_WRONG_ROUTE, STORAGE_NOT_CONFIGURED, FULL_ROUTE, VARIABLE_SHARD, LARGE_SET_THRESHOLD, bufferChunkStream,
 } from "../IArchives";
 import { copyArchiveFile } from "../archiveHelpers";
 import {
@@ -451,11 +451,11 @@ export class ArchivesChain implements IArchives {
                         errors.push(message);
                         continue;
                     }
-                    // A wrong-window/route rejection means OUR config is stale, so every source we'd fall back to is judged by the same stale config - refresh it and restart the pass instead
-                    if ((message.includes(STORAGE_WRONG_VALID_WINDOW) || message.includes(STORAGE_WRONG_ROUTE)) && !retriedWrongTarget) {
+                    // A wrong-window/route/unconfigured rejection means OUR config disagrees with the server's, so every source we'd fall back to is judged by the same stale config - refresh it and restart the pass instead
+                    if ((message.includes(STORAGE_WRONG_VALID_WINDOW) || message.includes(STORAGE_WRONG_ROUTE) || message.includes(STORAGE_NOT_CONFIGURED)) && !retriedWrongTarget) {
                         retriedWrongTarget = true;
                         wrongTarget = true;
-                        await this.prepareWrongTargetRetry(state, message.includes(STORAGE_WRONG_VALID_WINDOW) && "window" || "route");
+                        await this.prepareWrongTargetRetry(state, message.includes(STORAGE_WRONG_VALID_WINDOW) && "window" || message.includes(STORAGE_WRONG_ROUTE) && "route" || "unconfigured");
                         break;
                     }
                     if (source.isConnected()) throw e;
@@ -481,6 +481,7 @@ export class ArchivesChain implements IArchives {
     private async runPrimary<T>(config: { write?: boolean; route?: number; timeout?: SmartTimeout }, run: (archives: IArchives, sourceUrl: string) => Promise<T>): Promise<T> {
         let retriedWrongWindow = false;
         let retriedWrongRoute = false;
+        let retriedNotConfigured = false;
         let deadline = Date.now() + PRIMARY_RETRY_TIMEOUT;
         let attempt = 0;
         while (true) {
@@ -509,6 +510,11 @@ export class ArchivesChain implements IArchives {
                     await this.prepareWrongTargetRetry(state, "route");
                     continue;
                 }
+                if (message.includes(STORAGE_NOT_CONFIGURED) && !retriedNotConfigured) {
+                    retriedNotConfigured = true;
+                    await this.prepareWrongTargetRetry(state, "unconfigured");
+                    continue;
+                }
                 if (target && !target.isConnected()) target.noteFailure();
                 if (!config.write && target && (attempt === 1 || attempt % 3 === 0)) {
                     // The reason we try the HTTP request: httpsRequest has better DNS retrying capabilities than our WebSocket server (we have more control over it), so just using it can fix some DNS issues, which can propagate to fix the WebSocket connection. It is still the primary source's own data, so the no-fallback semantics hold - and ArchivesUrl has no setup cost, so making one on the spot is fine.
@@ -570,7 +576,7 @@ export class ArchivesChain implements IArchives {
     }
 
     private lastConfigRefresh = 0;
-    private async prepareWrongTargetRetry(state: ChainState, kind: "window" | "route"): Promise<void> {
+    private async prepareWrongTargetRetry(state: ChainState, kind: "window" | "route" | "unconfigured"): Promise<void> {
         if (kind === "window") {
             let now = Date.now();
             let nearBoundary = state.sources.some(source => source.config.validWindow.some(t => t > 0 && t < Number.MAX_SAFE_INTEGER && Math.abs(t - now) <= WRONG_TARGET_BOUNDARY_WINDOW));
@@ -582,7 +588,8 @@ export class ArchivesChain implements IArchives {
         }
         if (Date.now() - this.lastConfigRefresh < CONFIG_REFRESH_THROTTLE) return;
         this.lastConfigRefresh = Date.now();
-        console.log(`Write rejected by ${this.getDebugName()} (wrong ${kind}): our config is stale (likely a deploy switchover); refreshing it and retrying`);
+        let reason = kind === "window" && "wrong valid window" || kind === "route" && "wrong route" || "the store has no configuration entry on that server";
+        console.log(`Write rejected by ${this.getDebugName()} (${reason}): our config disagrees with the server's (likely stale, or a deploy switchover); refreshing it and retrying`);
         await this.refreshActiveConfig();
     }