sliftutils 1.7.117 → 1.7.119

package/index.d.ts

@@ -4217,7 +4217,8 @@ declare module "sliftutils/storage/remoteStorage/remoteConfig" {
     export declare function sourceIdentity(sourceConfig: SourceConfig | undefined): string;
     /** What an index entry records as the holder of its bytes (see ArchivesSource.url), so it must name the endpoint FOREVER. An intermediate is a switchover's temporary alternate port onto another source, and that port is gone for good once its window passes - so it is recorded as the source it was split out of, which holds the same bucket and outlives it. */
     export declare function sourcePersistentUrl(sourceConfig: SourceConfig | undefined, folder: string): string;
-    export declare function parseRoutingData(data: Buffer): RemoteConfig;
+    /** Reads a stored routing config. NEVER throws: this runs on every READ of a stored config, and a torn/corrupt file must not brick the paths that would fix it - above all writeRoutingConfig, where throwing while reading the OLD config blocks the write of the NEW one forever. Unreadable data is logged and read as undefined - the same as the file not existing. Judging a config on its way IN is the writer's job (see assertValidRemoteConfig at the write entry points), where rejecting bad data with a throw is correct. */
+    export declare function parseRoutingData(data: Buffer): RemoteConfig | undefined;
     export declare function serializeRemoteConfig(config: RemoteConfig): Buffer;
 
 }
@@ -4299,6 +4300,8 @@ declare module "sliftutils/storage/remoteStorage/sourceWrapper" {
         seedLatency(ms: number): void;
         /** Median of the recent pings (API or URL-form, whichever this source measures), plus DISCONNECTED_LATENCY_PENALTY while the source is disconnected - so a down source still sorts and can still be picked, just after every connected one. Sources with no measurements yet sort last (Infinity), except our own in-process server, which is the best possible target (0). */
         getLatency(): number;
+        /** writeBlocked from missing backblaze credentials is re-checked on every write attempt: the secret load can fail TRANSIENTLY (early startup, a file-read hiccup), and one failed load must never permanently stop this process from writing to the bucket. The other writeBlocked reasons (read-only mode, browsers) are static properties of the process, so there is nothing to re-check. */
+        recheckWriteBlocked(): Promise<void>;
         /** Writes always go through the API, so a permission error throws to the caller on every write (and access granted in the meantime is picked up automatically). */
         write<T>(run: (archives: IArchives) => Promise<T>): Promise<T>;
         dispose(): void;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.117",
+    "version": "1.7.119",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/remoteStorage/blobStore.ts

@@ -213,12 +213,8 @@ export class BlobStore {
     private async readRoutingConfig(): Promise<RemoteConfig | undefined> {
         let data = await this.ownDisk.get(ROUTING_FILE);
         if (!data || !data.length) return undefined;
-        try {
-            return parseRoutingData(data);
-        } catch (e) {
-            logStorageError(`Ignoring the routing config in store ${this.folder}: it could not be parsed: ${(e as Error).stack ?? e}`);
-            return undefined;
-        }
+        // Unparseable reads as "no config" (parseRoutingData logs it) - a store must never be unable to start because its own routing file is torn
+        return parseRoutingData(data);
     }
 
     /** The version of the routing config this store is running, so a copy found on a peer is only taken when it is genuinely newer. -1 means it has none. */
@@ -975,6 +971,9 @@ export class BlobStore {
      */
     private assertRoutingConfigWritable(data: Buffer): void {
         let routing = parseRoutingData(data);
+        if (!routing) {
+            throw new Error(`Routing config write rejected - not a parseable config. Refusing to write ${ROUTING_FILE} to store ${this.folder}: the data is not a valid { version?, sources: [...] } JSON config (${data.length} bytes)`);
+        }
         assertValidRemoteConfig(routing);
         let incoming = getConfigVersion(routing);
         let current = this.routingVersion();

package/storage/remoteStorage/chainStartup.ts

@@ -394,14 +394,23 @@ export async function writeRoutingToAllStores(config: { configured: RemoteConfig
     let routingData = serializeRemoteConfig(configured);
     let routingWriteTime = Math.round(Date.now());
     let targets: SourceWrapper[] = [];
+    let skipped: SourceWrapper[] = [];
     let seen = new Set<string>();
     for (let source of sources) {
         let key = `${source.config.url}|${source.config.name}`;
         if (seen.has(key)) continue;
         seen.add(key);
+        // writeBlocked is a capability of this process (read-only mode, a browser, no backblaze credentials) - not connectivity, so skipping is not a pre-check race. Counting these as failures put the retry loop on its 5-minute cadence FOREVER over writes this process cannot make; the stores get the config from the writers that can (servers, node clients, peers pulling it from each other). The recheck first: the backblaze-credentials block can be a TRANSIENT secret-load failure at startup, and skipping on the stale verdict would otherwise make it permanent (see SourceWrapper.recheckWriteBlocked).
+        if (source.writeBlocked) {
+            await source.recheckWriteBlocked();
+        }
+        if (source.writeBlocked || !source.api) {
+            skipped.push(source);
+            continue;
+        }
         targets.push(source);
     }
-    console.log(`Writing routing config version ${getConfigVersion(configured)} for ${debugName} to all ${targets.length} stores (write time ${formatDateTimeDetailed(routingWriteTime)}): ${targets.map(x => `${x.config.url} (store ${JSON.stringify(x.config.name)})`).join(", ")}`);
+    console.log(`Writing routing config version ${getConfigVersion(configured)} for ${debugName} to all ${targets.length} writable stores (write time ${formatDateTimeDetailed(routingWriteTime)}): ${targets.map(x => `${x.config.url} (store ${JSON.stringify(x.config.name)})`).join(", ") || "none"}${skipped.length && `. Skipping ${skipped.length} not writable from this process: ${skipped.map(x => `${x.config.url} (store ${JSON.stringify(x.config.name)}, ${x.writeBlocked || "no API access"})`).join(", ")}` || ""}`);
     let failures: string[] = [];
     await Promise.all(targets.map(async source => {
         try {

package/storage/remoteStorage/createArchives.ts

@@ -563,7 +563,11 @@ export class ArchivesChain implements IArchives {
 
     private async setRoutingConfig(data: Buffer, config?: { lastModified?: number }): Promise<string> {
         // Checked before the first node is written to, not just by each node as it arrives: every server would reject it anyway, and finding that out one node at a time is how a config write ends up half-applied
-        assertValidRemoteConfig(parseRoutingData(data));
+        let parsedConfig = parseRoutingData(data);
+        if (!parsedConfig) {
+            throw new Error(`Routing config write rejected - not a parseable config. The data is not a valid { version?, sources: [...] } JSON config (${data.length} bytes, for ${this.getDebugName()})`);
+        }
+        assertValidRemoteConfig(parsedConfig);
         let state = await this.state.getState();
         let writeTime = Math.round(config?.lastModified || Date.now());
         let written: string[] = [];