sliftutils 1.7.97 → 1.7.99

package/index.d.ts

@@ -3296,6 +3296,8 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
             onIndexChanged?: ((key: string) => void) | undefined;
             /** Called every time this store applies a routing config to itself (startup, an operator's write, a peer's copy arriving) - the store is the one that knows when a config landed, and the server arms window-boundary scans from it. */
             onRoutingApplied?: ((routing: RemoteConfig) => void) | undefined;
+            /** Asks the client whose request created this store what routing config it intended for our name. Only used when init finds NO configuration in our folder: a store only ever exists because a config names it, so the requester has that config - asking for it lazily is the same information as passing the config on every call, without the per-call kilobytes. */
+            requestRoutingConfig?: (() => Promise<RemoteConfig | undefined>) | undefined;
             onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined;
             resolveSourceUrl?: ((url: string) => IArchives) | undefined;
         } | undefined);
@@ -3601,6 +3603,7 @@ declare module "sliftutils/storage/remoteStorage/chainStartup" {
             /** See ArchivesChainOptions.directConnect. */
             directConnect?: boolean;
         });
+        private untrackConfig;
         /** The newest adopted state, synchronously - undefined until the first init finishes. */
         latest(): ChainState | undefined;
         getState(): Promise<ChainState>;
@@ -4071,11 +4074,25 @@ declare module "sliftutils/storage/remoteStorage/sourcesList" {
 }
 
 declare module "sliftutils/storage/remoteStorage/storageClientController" {
+    import { RemoteConfig } from "../IArchives";
     /** Subscribe to server-pushed routing change notifications. Returns the unsubscribe function. */
     export declare function onServerRoutingChanged(listener: () => void): () => void;
+    /** One chain's configs, as getRoutingConfigForName answers from them: the initial in-code config, and a getter for the synchronized one (adopted from the stored routing files, so it changes over time). */
+    type TrackedChainConfig = {
+        configured: RemoteConfig;
+        active: () => RemoteConfig;
+    };
+    /** Every chain tracks its configs here (see ChainStateManager), so a server can ask what config was intended for a store name. Returns the untrack function. */
+    export declare function trackChainConfig(entry: TrackedChainConfig): () => void;
     export declare const StorageClientController: import("socket-function/SocketFunctionTypes").SocketRegistered<{
         routingConfigChanged: () => Promise<void>;
+        getRoutingConfigForName: (config: {
+            account: string;
+            bucketName: string;
+            name: string;
+        }) => Promise<RemoteConfig | undefined>;
     }>;
+    export {};
 
 }
 
@@ -4317,7 +4334,7 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
     import { RemoteConfig, SourceConfig, IArchives, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
     import { BucketDiskInfo } from "./bucketDisk";
     import { BucketWriteStats } from "./accessStats";
-    export declare function getStore(account: string, bucketName: string, name: string): BlobStore;
+    export declare function getStore(account: string, bucketName: string, name: string, callerNodeId?: string): BlobStore;
     /** The store serving a request: the one the client's selected entry NAMES. Account, name, and bucket ARE the folder, so this is a direct lookup, never a search of what exists - and a name this server has never seen is CREATED, never rejected, because asking for a name is the instruction to have that store (one name, one folder, one index; it configures itself once the routing config lands in it). Nothing else about the request is compared - not the window, not the route, not the flags - which is the whole point of naming it: a client a config version behind on some flag still reaches the right store. */
     export declare function findBucketStore(account: string, bucketName: string, sourceConfig: SourceConfig | undefined): BlobStore;
     /** The stores of a bucket as the DISK records them (a bucket is nothing more than the store folders sharing its name), opened - so the ones that weren't running yet start synchronizing. Empty when the bucket does not exist here. */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.97",
+    "version": "1.7.99",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/remoteStorage/blobStore.d.ts

@@ -59,6 +59,8 @@ export declare class BlobStore {
         onIndexChanged?: ((key: string) => void) | undefined;
         /** Called every time this store applies a routing config to itself (startup, an operator's write, a peer's copy arriving) - the store is the one that knows when a config landed, and the server arms window-boundary scans from it. */
         onRoutingApplied?: ((routing: RemoteConfig) => void) | undefined;
+        /** Asks the client whose request created this store what routing config it intended for our name. Only used when init finds NO configuration in our folder: a store only ever exists because a config names it, so the requester has that config - asking for it lazily is the same information as passing the config on every call, without the per-call kilobytes. */
+        requestRoutingConfig?: (() => Promise<RemoteConfig | undefined>) | undefined;
         onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined;
         resolveSourceUrl?: ((url: string) => IArchives) | undefined;
     } | undefined);

package/storage/remoteStorage/blobStore.ts

@@ -6,7 +6,7 @@ import {
     windowsAcceptWrites, SyncActivity, FULL_ROUTE, FULL_VALID_WINDOW, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, STORAGE_NOT_CONFIGURED,
 } from "../IArchives";
 import { ArchivesDisk, applyFindInfoShape } from "../ArchivesDisk";
-import { ROUTING_FILE, getRoute, routeContains, routeIntersection, parseRoutingData, assertValidRemoteConfig, getConfigVersion, sourceIdentity, sourcePersistentUrl } from "./remoteConfig";
+import { ROUTING_FILE, getRoute, routeContains, routeIntersection, parseRoutingData, serializeRemoteConfig, assertValidRemoteConfig, getConfigVersion, sourceIdentity, sourcePersistentUrl } from "./remoteConfig";
 import { selectEntryAt } from "./storePlan";
 import { sourceWriteDelay } from "./ArchivesDelayed";
 import { LogMap } from "../LogMap";
@@ -102,6 +102,8 @@ export class BlobStore {
             onIndexChanged?: (key: string) => void;
             /** Called every time this store applies a routing config to itself (startup, an operator's write, a peer's copy arriving) - the store is the one that knows when a config landed, and the server arms window-boundary scans from it. */
             onRoutingApplied?: (routing: RemoteConfig) => void;
+            /** Asks the client whose request created this store what routing config it intended for our name. Only used when init finds NO configuration in our folder: a store only ever exists because a config names it, so the requester has that config - asking for it lazily is the same information as passing the config on every call, without the per-call kilobytes. */
+            requestRoutingConfig?: () => Promise<RemoteConfig | undefined>;
             // Every accepted write ("original") and every write that actually reached the sources ("flushed"). Fast writes coalesce, so the two counts differ.
             onWriteCounted?: (kind: "original" | "flushed", bytes: number) => void;
             // Resolves a persisted source URL (see ArchivesSource.url) to a cached IArchives, so entries whose holder is no longer configured can still be read
@@ -130,6 +132,22 @@ export class BlobStore {
     public init = lazy(async () => {
         // Configure first: the config decides what the sources ARE, and everything below is per source. With no config at all that is just our own disk, which is enough to serve and to be written to.
         await this.applyRoutingConfig();
+        // No configuration in our folder at all - which happens when this store was just CREATED by a request, before any config write reached it. The requester knows exactly what config it intended for our name, so ask it.
+        if (!this.storeConfig.all().length && this.config?.requestRoutingConfig) {
+            try {
+                let provided = await this.config.requestRoutingConfig();
+                if (provided) {
+                    console.log(`Store ${JSON.stringify(this.storeName)} (folder ${this.folder}) initialized with no configuration; the requesting client provided routing config version ${getConfigVersion(provided)} - adopting it`);
+                    // Straight onto the disk, not through set(): set waits for init, which is what is running right now
+                    await this.ownDisk.set(ROUTING_FILE, Buffer.from(serializeRemoteConfig(provided)), { lastModified: Date.now() });
+                    await this.applyRoutingConfig();
+                } else {
+                    console.warn(`Store ${JSON.stringify(this.storeName)} (folder ${this.folder}) initialized with no configuration, and the requesting client had no routing config naming it either - running unconfigured`);
+                }
+            } catch (e) {
+                console.error(`Asking the requesting client for the routing config of store ${JSON.stringify(this.storeName)} (folder ${this.folder}) failed - running unconfigured: ${(e as Error).stack ?? e}`);
+            }
+        }
         for (let i = 0; i < this.sources.length; i++) {
             await this.registerSlot(i);
         }

package/storage/remoteStorage/chainStartup.d.ts

@@ -26,6 +26,7 @@ export declare class ChainStateManager {
         /** See ArchivesChainOptions.directConnect. */
         directConnect?: boolean;
     });
+    private untrackConfig;
     /** The newest adopted state, synchronously - undefined until the first init finishes. */
     latest(): ChainState | undefined;
     getState(): Promise<ChainState>;

package/storage/remoteStorage/chainStartup.ts

@@ -3,7 +3,7 @@ import { RemoteConfig, SourceConfig } from "../IArchives";
 import { ROUTING_FILE, getConfigVersion, serializeRemoteConfig, normalizeRemoteConfig, normalizeSource } from "./remoteConfig";
 import { SourceWrapper, RETRY_START_DELAY, RETRY_MAX_DELAY, RETRY_GROWTH } from "./sourceWrapper";
 import { resolveIntermediateSources } from "./intermediateSources";
-import { onServerRoutingChanged } from "./storageClientController";
+import { onServerRoutingChanged, trackChainConfig } from "./storageClientController";
 
 // The LIFECYCLE of an ArchivesChain's state, and nothing else: initializing it (probing every configured source, deciding which routing config to run, writing ours out when it is the newest), the retry when initialization fails, the poll that adopts newer configs, the availability recheck, and the loop that keeps re-writing our config. None of this is request dispatch - the chain (createArchives.ts) asks the ChainStateManager for the current state and dispatches over it.
 
@@ -50,8 +50,12 @@ export class ChainStateManager {
         this.unsubscribeRoutingPush = onServerRoutingChanged(() => {
             void this.refreshActiveConfig().catch((e: Error) => console.error(`Config refresh failed for ${this.config.debugName()}: ${e.stack ?? e}`));
         });
+        // So a server can ask what config we intended for a store name (see StorageClientController.getRoutingConfigForName)
+        this.untrackConfig = trackChainConfig({ configured: config.configured, active: () => this.activeConfig });
     }
 
+    private untrackConfig: () => void;
+
     /** The newest adopted state, synchronously - undefined until the first init finishes. */
     public latest(): ChainState | undefined {
         return this.latestState;
@@ -275,6 +279,7 @@ export class ChainStateManager {
 
     public dispose(): void {
         this.disposed = true;
+        this.untrackConfig();
         this.unsubscribeRoutingPush?.();
         if (this.pollTimer) {
             clearInterval(this.pollTimer);

package/storage/remoteStorage/createArchives.ts

@@ -28,7 +28,7 @@ const PRIMARY_RETRY_DELAY = 2 * 1000;
 const COVERING_RETRY_TIMEOUT = 30 * 1000;
 const COVERING_RETRY_DELAY = 5 * 1000;
 // Smart timeouts: an attempt gets this long to produce anything before we probe getInfo for the file's size (and the probe itself gets the same window)
-const SMART_TIMEOUT_PROBE = 10 * 1000;
+const SMART_TIMEOUT_PROBE = 60 * 1000;
 // Very generous assumed transfer rates - the resulting deadline exists to catch stuck sources, not slow ones
 const SMART_TIMEOUT_DOWNLOAD_BYTES_PER_SECOND = 1024 * 1024;
 const SMART_TIMEOUT_UPLOAD_BYTES_PER_SECOND = 512 * 1024;