sliftutils 1.7.27 → 1.7.29

package/storage/remoteStorage/spec.md

@@ -2,10 +2,6 @@
 
 This is the reasoning behind the remote storage system (createArchives / storageServer / BlobStore). The code documents the mechanics; this documents the ideas that must survive refactors.
 
-## Configuration updates happen only on startup
-
-A client adopts a routing config once, during initialization, and then runs on it. Configs never automatically upgrade mid-run: the 5-minute poll picks up whatever the sources currently hold (so a developer's change spreads), but the version-gated *upgrade* — writing a newer configured version over an older stored one — happens only at startup. This is deliberate: config changes are made by a developer running code, and that developer is present at startup. It also prevents reversion loops — if a server briefly held a newer version and then went away, running clients don't cling to that phantom version; they follow what the surviving sources actually hold, and only a fresh startup with an explicitly newer configured version moves things forward again.
-
 ## Scanning + tombstones + versions make any two sources mergeable
 
 Every store fully rescans its sources' metadata (on startup and periodically), and every write — including deletions — is ordered by last-write time. Deletions are tombstones: an empty file IS a missing file, kept as a size-0 index entry (for a week) so the deletion itself propagates and reconciles like any other write. Because everything is a timestamped write and scans are bidirectional (pull what they have, push what they're missing), any two sources can be merged in any order and converge to the same state: newest write time wins, per file. There is no operation whose loss corrupts the system — a failed background write, a missed delete, a source that was down for a day — the next scan reconciles it.
@@ -16,6 +12,16 @@ Each bucket stores its complete routing config (the full redundancy list) inside
 
 If the client tries to do a write where the valid state is far enough away or the sharding is wrong, it will re-download the routing config throttled, so it only does this at most once every 30 seconds, and retry the request.
 
+Storage routing JSON is only written to if we have write access and it's only written to on startup, it doesn't propagate. that way things don't revert without the developer intentionally rerunning it. 
+
+## Redundancy, sharding, and deployment
+
+For redundancy, we can just have multiple different configurations that will satisfy the same request. The first one is the one that we write to. If that one's down, we don't do writes.
+
+However, there's also sharding, where we have different route ranges that values can hash to. Values can explicitly try to write to a specific value via a special like hash override key. This allows us to sometimes make our rights hit a server which has less latency. In the case that the client writer will accept the fact that we might change the key.
+
+The valid windows allow us to schedule deployments, so the nodes can switch over gracefully.
+
 ## BulkDatabase2 index + our own disk as just another source
 
 Each bucket's store keeps a BulkDatabase2 index of every file (path, write time, size, holding source), served from memory — existence checks and listings are extremely fast, never touching a source. The trick that keeps the index accurate: our own disk is not special-cased, it is simply the first synchronization source. The same scan/reconcile code that synchronizes remote sources also synchronizes the disk with the index, so the index self-heals from the same machinery instead of needing separate consistency logic.

package/storage/remoteStorage/storageClientController.d.ts

@@ -0,0 +1,5 @@
+/** Subscribe to server-pushed routing change notifications. Returns the unsubscribe function. */
+export declare function onServerRoutingChanged(listener: () => void): () => void;
+export declare const StorageClientController: import("socket-function/SocketFunctionTypes").SocketRegistered<{
+    routingConfigChanged: () => Promise<void>;
+}>;

package/storage/remoteStorage/storageClientController.ts

@@ -0,0 +1,35 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+
+// The client half of routing-change broadcasts: the storage server tracks its connected clients
+// (see trackCaller in storageController) and calls routingConfigChanged on every one of them the
+// moment a routing config changes - clients react immediately instead of waiting for a poll.
+
+const listeners = new Set<() => void>();
+
+/** Subscribe to server-pushed routing change notifications. Returns the unsubscribe function. */
+export function onServerRoutingChanged(listener: () => void): () => void {
+    listeners.add(listener);
+    return () => {
+        listeners.delete(listener);
+    };
+}
+
+class StorageClientControllerBase {
+    async routingConfigChanged(): Promise<void> {
+        for (let listener of [...listeners]) {
+            try {
+                listener();
+            } catch (e) {
+                console.error(`Routing change listener failed: ${(e as Error).stack ?? e}`);
+            }
+        }
+    }
+}
+
+export const StorageClientController = SocketFunction.register(
+    "StorageClientController-remoteStorage-2b8f1a6d",
+    new StorageClientControllerBase(),
+    () => ({
+        routingConfigChanged: {},
+    }),
+);

package/storage/remoteStorage/storageController.ts

@@ -14,8 +14,9 @@ import { getTakeoverStamp } from "./deployTakeover";
 import {
     getStorageServerConfig, getTrust, getRequests, getLoadedBucket, writeBucketFile,
     deleteBucketFile, assertWritesAllowed, assertMutable, LoadedBucket,
-    getBucketConfig, listAccountBuckets, ServerBucketInfo,
+    getBucketConfig, listAccountBuckets, ServerBucketInfo, setRoutingChangedBroadcaster,
 } from "./storageServerState";
+import { StorageClientController } from "./storageClientController";
 
 // The remote storage server's API. Authentication uses certs.ts machine identities: a client
 // proves it owns its machine key by signing a timestamped token (bound to this server, so tokens
@@ -105,6 +106,31 @@ function assertValidPath(path: string) {
     }
 }
 
+// Every client pings us (immediately on connect, then continuously), so pings tell us exactly who
+// our currently-connected clients are. Client nodeIds never reconnect (a reconnect is a NEW
+// nodeId), so the disconnect callback fully cleans an entry up.
+const connectedClients = new Set<string>();
+function trackCaller(): void {
+    let nodeId = SocketFunction.getCaller().nodeId;
+    if (connectedClients.has(nodeId)) return;
+    connectedClients.add(nodeId);
+    SocketFunction.onNextDisconnect(nodeId, () => {
+        connectedClients.delete(nodeId);
+    });
+}
+
+// The moment any routing config changes on this server, every connected client is told - clients
+// must never have to wait for a poll to learn the topology changed
+setRoutingChangedBroadcaster(() => {
+    console.log(`Broadcasting routing config change to ${connectedClients.size} connected clients`);
+    for (let nodeId of [...connectedClients]) {
+        void StorageClientController.nodes[nodeId].routingConfigChanged().catch(() => {
+            // The client is gone (or too old to know the controller); the disconnect callback
+            // cleans the registry, nothing to do here
+        });
+    }
+});
+
 function getCallerMachineId(): string {
     let caller = SocketFunction.getCaller();
     let machineId = sessions.get(caller.nodeId);
@@ -155,8 +181,10 @@ async function getBucket(account: string, bucketName: string): Promise<LoadedBuc
 class RemoteStorageControllerBase {
     // Latency measurement (see SourceWrapper's pinging); no auth, it measures the transport. The
     // takeover stamp piggybacks on it so every connected client learns of a deploy takeover
-    // within one ping interval.
+    // within one ping interval. Pings also tell us who our clients ARE - the trackCaller
+    // registry is what routing-change broadcasts push to.
     async ping(): Promise<{ takeover?: string }> {
+        trackCaller();
         return { takeover: getTakeoverStamp() };
     }
 

package/storage/remoteStorage/storageServerState.d.ts

@@ -37,6 +37,7 @@ export type LoadedBucket = {
 };
 export declare function addExtraListenPort(port: number): void;
 export declare function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined>;
+export declare function setRoutingChangedBroadcaster(broadcaster: () => void): void;
 export declare function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void>;
 export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
     lastModified?: number;

package/storage/remoteStorage/storageServerState.ts

@@ -8,7 +8,7 @@ import { BlobStore, IBucketStore, DEFAULT_FAST_WRITE_DELAY, WINDOW_END_FLUSH_MAR
 import {
     RemoteConfig, HostedConfig, BackblazeConfig, IArchives, ArchivesSource, ArchiveFileInfo, ArchivesConfig,
     ArchivesSyncStatus,
-    WRITE_PAST_WINDOW_GRACE, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
+    STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
 } from "../IArchives";
 import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
 import { applyDeployRemap, getTakeoverAltPort, getOwnWindowEndClip, onTakeoverEvent } from "./deployTakeover";
@@ -563,6 +563,18 @@ export function getLoadedBucket(account: string, bucketName: string): Promise<Lo
     return loaded;
 }
 
+// Called whenever any bucket's routing config is applied (created, updated in place, or rebuilt).
+// storageController wires this to its connected-client broadcast, so every client is pushed the
+// change immediately - never waiting for a poll. (A callback, not an import - storageController
+// imports us, so importing it back would be a cycle.)
+let routingChangedBroadcaster: (() => void) | undefined;
+export function setRoutingChangedBroadcaster(broadcaster: () => void): void {
+    routingChangedBroadcaster = broadcaster;
+}
+function notifyRoutingChanged(): void {
+    routingChangedBroadcaster?.();
+}
+
 // Routing reloads are serialized per bucket, so concurrent writes/syncs can't rebuild the same
 // store twice in parallel
 const routingReloads = new Map<string, Promise<void>>();
@@ -610,6 +622,7 @@ async function checkRoutingChanged(account: string, bucketName: string, config?:
         buckets.set(key, Promise.resolve(updated));
         scheduleWindowBoundaryRebuild(updated);
         scheduleBoundaryScans(updated);
+        notifyRoutingChanged();
         return;
     }
     console.log(`Rebuilding the store for bucket ${key} (${reason})`);
@@ -617,6 +630,7 @@ async function checkRoutingChanged(account: string, bucketName: string, config?:
         await loaded.store.dispose();
     }
     buckets.set(key, Promise.resolve(buildBucket(account, bucketName, routing, plan)));
+    notifyRoutingChanged();
 }
 
 // Per-write options are evaluated at the WRITE's time (and the key's route), not the current
@@ -657,6 +671,7 @@ async function writeRoutingConfig(account: string, bucketName: string, data: Buf
     if (!loaded) {
         await new ArchivesDisk(getBucketFolder(account, bucketName)).set(ROUTING_FILE, data, { lastModified: config?.lastModified });
         buckets.set(key, Promise.resolve(buildBucket(account, bucketName, incoming)));
+        notifyRoutingChanged();
         console.log(`Created bucket ${key}`);
         return;
     }
@@ -696,9 +711,9 @@ export async function writeBucketFile(account: string, bucketName: string, fileP
     // target from a stale view - it must re-resolve and retry at the correct source.
     let route = getRoute(filePath);
     if (!config?.lastModified && loaded.selfEntries.length) {
-        let timeValid = loaded.selfEntries.filter(x => writeTime >= x.validWindow[0] - WRITE_PAST_WINDOW_GRACE && writeTime <= x.validWindow[1] + WRITE_PAST_WINDOW_GRACE);
+        let timeValid = loaded.selfEntries.filter(x => writeTime >= x.validWindow[0] && writeTime < x.validWindow[1]);
         if (!timeValid.length) {
-            logWrongTargetRejection(`Rejecting fresh write of ${JSON.stringify(filePath)} to bucket ${account}/${bucketName}: writeTime ${writeTime} (${new Date(writeTime).toISOString()}) is outside all our valid windows ${JSON.stringify(loaded.selfEntries.map(x => x.validWindow))} even with the ${WRITE_PAST_WINDOW_GRACE}ms grace (a switchover moved the write target)`);
+            logWrongTargetRejection(`Rejecting fresh write of ${JSON.stringify(filePath)} to bucket ${account}/${bucketName}: writeTime ${writeTime} (${new Date(writeTime).toISOString()}) is outside all our valid windows ${JSON.stringify(loaded.selfEntries.map(x => x.validWindow))} (a switchover moved the write target)`);
             throw new Error(`${STORAGE_WRONG_VALID_WINDOW} This server is not a valid write target at ${writeTime} for bucket ${account}/${bucketName} (our valid windows: ${JSON.stringify(loaded.selfEntries.map(x => x.validWindow))}). Re-resolve the currently valid source and retry.`);
         }
         if (!timeValid.some(x => routeContains(x.route, route))) {