sliftutils 1.7.22 → 1.7.24

package/storage/remoteStorage/sourceWrapper.ts

@@ -53,7 +53,7 @@ export class SourceWrapper {
         let wrapper = new SourceWrapper(config, options?.background !== false);
         if (config.type === "backblaze") {
             if (await hasBackblazeCreds()) {
-                wrapper.api = new ArchivesBackblaze({ bucketName: parseBackblazeUrl(config.url).bucketName, public: config.public, immutable: config.immutable });
+                wrapper.api = new ArchivesBackblaze({ bucketName: parseBackblazeUrl(config.url).bucketName, public: config.public, immutable: config.immutable, allowedOrigins: config.allowedOrigins });
             } else if (isNode()) {
                 wrapper.writeBlocked = `No backblaze credentials for ${config.url}. Provide backblaze.json.applicationKeyId and backblaze.json.applicationKey via getSecret to enable API access.`;
             } else {
@@ -93,7 +93,7 @@ export class SourceWrapper {
             let endText = end === Number.MAX_SAFE_INTEGER && "forever" || new Date(end).toISOString();
             parts.push(`validWindow [${startText}, ${endText}]`);
         }
-        return `source/${this.config.url}${parts.length && ` (${parts.join(", ")})` || ""}`;
+        return `source ${this.config.url}${parts.length && ` (${parts.join(", ")})` || ""}`;
     }
 
     public isConnected(): boolean {

package/storage/remoteStorage/spec.md

@@ -10,10 +10,12 @@ A client adopts a routing config once, during initialization, and then runs on i
 
 Every store fully rescans its sources' metadata (on startup and periodically), and every write — including deletions — is ordered by last-write time. Deletions are tombstones: an empty file IS a missing file, kept as a size-0 index entry (for a week) so the deletion itself propagates and reconciles like any other write. Because everything is a timestamped write and scans are bidirectional (pull what they have, push what they're missing), any two sources can be merged in any order and converge to the same state: newest write time wins, per file. There is no operation whose loss corrupts the system — a failed background write, a missed delete, a source that was down for a day — the next scan reconciles it.
 
-## The full source list is duplicated into every source
+## storage/storagerouting.json Has the routing config and is duplicated on every node. 
 
 Each bucket stores its complete routing config (the full redundancy list) inside itself, at storage/storagerouting.json, on every source. Discovery is therefore trivial: as long as ONE node is up, a client reading it gets the full overview of the intended sources. And because clients re-discover on every startup (and re-read every 5 minutes), a developer can change the configuration in one place and clients rapidly accept it — no redeploy, no coordinated restart of the fleet.
 
+If the client tries to do a write where the valid state is far enough away or the sharding is wrong, it will re-download the routing config throttled, so it only does this at most once every 30 seconds, and retry the request.
+
 ## BulkDatabase2 index + our own disk as just another source
 
 Each bucket's store keeps a BulkDatabase2 index of every file (path, write time, size, holding source), served from memory — existence checks and listings are extremely fast, never touching a source. The trick that keeps the index accurate: our own disk is not special-cased, it is simply the first synchronization source. The same scan/reconcile code that synchronizes remote sources also synchronizes the disk with the index, so the index self-heals from the same machinery instead of needing separate consistency logic.
@@ -29,3 +31,19 @@ Clients always write to the same node — the first source whose valid window is
 ## Trust instead of API keys
 
 Machines authenticate with their certs.ts identity (proving ownership of their machine key with a signed, server-bound token), and access is granted per account to specific machineIds. No API keys are minted, copied into configs, or passed around — granting a machine access is one command on the storage machine, and revoking it is removing the trust record. The only API keys left in the system are the ones third parties force on us: backblaze and cloudflare, both resolved through getSecret.
+
+## storage/storagerouting.json is a special file
+
+    This is our special file that stores the routing information. We write it directly to each node. The client side tries to keep an updated version of these (that's mentioned earlier And is how the client can keep up to date even if the client's code isn't up to date, As long as at least one of the sources is still alive).
+
+    IMPORTANTLY! This special file is never synchronized between different storage nodes. It's only directly written to, and it's only read off of our disk. It can be written to on any node. We don't take into account the valid state window or the shard. We still do take into account the write time, though, the latest write time wins. We only allow writes if version >= the previous version.
+
+## fast writes
+
+    We support a flag that does fast writes, which will cause us to batch all the writes in memory, Returning from the set call immediately. This allows you to do many writes to the same file with very little disk I/O. This uses a configurable delay amount. You could set it to zero and then we just won't delay it at all, and we'll flush everything to the disk immediately away. 
+
+    The deploy system will tell us if it's intending to switch over a source, which we use to create a virtual valid state window in the middle that uses a different port. 
+
+    We look at the valid state windows and we make sure we never delay past the valid state window. In fact, if we are within five minutes of being invalid due to the valid state window, we flush the writes immediately. That way, when the next valid window starts running, the writes will be already on disk. 
+
+    We also do scans when we are coming up to the valid state window. If we are going to be the new valid state window, both before, a little bit after, and farther after. This helps the switch over be smoother so we get all of the trailing writes. These scans use our ability to ask for the changes since a certain time. We do the scan on the right write node. The write node is always the first node in order (with a valid state window and matching the route hash). Which might result in us having to scan multiple nodes if we require multiple nodes to fill the full route hash window. 

package/storage/remoteStorage/storageServerState.d.ts

@@ -42,7 +42,6 @@ export declare function writeBucketFile(account: string, bucketName: string, fil
 }): Promise<void>;
 export declare function getBucketConfig(bucket: LoadedBucket): ArchivesConfig;
 export declare function rebuildAllLoadedBuckets(): Promise<void>;
-export declare function rescanAllLoadedBucketDisks(): Promise<void>;
 export type ServerBucketInfo = {
     bucketName: string;
     active: boolean;

package/storage/remoteStorage/storageServerState.ts

@@ -4,14 +4,14 @@ import { getFileStorageNested2 } from "../FileFolderAPI";
 import { TransactionStorage } from "../TransactionStorage";
 import { JSONStorage } from "../JSONStorage";
 import { ArchivesDisk } from "../ArchivesDisk";
-import { BlobStore, IBucketStore } from "./blobStore";
+import { BlobStore, IBucketStore, DEFAULT_FAST_WRITE_DELAY, WINDOW_END_FLUSH_MARGIN } from "./blobStore";
 import {
     RemoteConfig, HostedConfig, IArchives, ArchivesSource, ArchiveFileInfo, ArchivesConfig,
     ArchivesSyncStatus, FULL_VALID_WINDOW,
     WRITE_PAST_WINDOW_GRACE, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
 } from "../IArchives";
 import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
-import { applyDeployRemap, getFlushDeadline, getTakeoverAltPort, onTakeoverEvent } from "./deployTakeover";
+import { applyDeployRemap, getTakeoverAltPort, onTakeoverEvent } from "./deployTakeover";
 import { createApiArchives } from "./createArchives";
 import type { IStorage } from "../IStorage";
 import type { AccessRequest, TrustRecord } from "./storageController";
@@ -233,6 +233,150 @@ function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
     (timer as { unref?: () => void }).unref?.();
 }
 
+// Extra scans around each of OUR valid-window starts (a deploy switchover is just this too - its
+// remap adds valid windows), catching last-moment writes the previous window's owner accepted near
+// the boundary. By the final scan the previous owner has flushed everything (fast writes never
+// delay past a window's end), so nothing is missed.
+const BOUNDARY_SCAN_OFFSETS = [-30 * 1000, 2 * 1000, 30 * 1000];
+// How far back a boundary scan asks the previous owner for changes: anything older has already
+// arrived through normal synchronization, and this covers the longest a write can sit unflushed
+const BOUNDARY_SCAN_LOOKBACK = DEFAULT_FAST_WRITE_DELAY + WINDOW_END_FLUSH_MARGIN;
+
+// Each (bucket, boundary, offset) is scheduled exactly once, surviving store rebuilds - the timer
+// re-resolves the bucket (and recomputes owners from the then-current routing) when it fires
+const scheduledBoundaryScans = new Set<string>();
+const boundaryScansRunning = new Set<string>();
+
+function scheduleBoundaryScans(loaded: LoadedBucket): void {
+    let key = `${loaded.account}/${loaded.bucketName}`;
+    let now = Date.now();
+    let starts = new Set<number>();
+    for (let self of loaded.selfEntries) {
+        let start = self.validWindow[0];
+        if (start > now && start < Number.MAX_SAFE_INTEGER) {
+            starts.add(start);
+        }
+    }
+    for (let start of starts) {
+        for (let offset of BOUNDARY_SCAN_OFFSETS) {
+            let at = start + offset;
+            if (at <= now) continue;
+            let scheduleKey = `${key}|${start}|${offset}`;
+            if (scheduledBoundaryScans.has(scheduleKey)) continue;
+            scheduledBoundaryScans.add(scheduleKey);
+            let arm = () => {
+                let timer = setTimeout(() => {
+                    // Timer length is capped (far-future boundaries), so re-arm until the time
+                    // actually arrives
+                    if (Date.now() < at) {
+                        arm();
+                        return;
+                    }
+                    void runBoundaryScan(key, start, offset).catch((e: Error) => console.error(`Boundary scan for bucket ${key} failed: ${e.stack ?? e}`));
+                }, Math.min(at - Date.now(), MAX_REBUILD_TIMER_DELAY));
+                (timer as { unref?: () => void }).unref?.();
+            };
+            arm();
+        }
+    }
+}
+
+async function runBoundaryScan(bucketKey: string, windowStart: number, offset: number): Promise<void> {
+    let label = `bucket ${bucketKey}, window starting ${new Date(windowStart).toISOString()}, offset ${offset / 1000}s`;
+    if (boundaryScansRunning.has(bucketKey)) {
+        // Boundary scans never run in parallel; a slow one simply swallows later scheduled points
+        console.log(`Skipping boundary scan (${label}): the previous boundary scan is still running`);
+        return;
+    }
+    boundaryScansRunning.add(bucketKey);
+    try {
+        let loaded = await buckets.get(bucketKey);
+        if (!loaded) return;
+        let store = loaded.store;
+        if (!(store instanceof BlobStore)) return;
+        // Owners are recomputed at fire time - the routing (or the takeover remap) may have
+        // changed since this scan was scheduled
+        let effective = applyDeployRemap(loaded.routing);
+        let selfIndexes = findSelfIndexes(effective, loaded.account, loaded.bucketName);
+        let selfIndexSet = new Set(selfIndexes);
+        let selves = selfIndexes.map(i => effective.sources[i] as HostedConfig).filter(x => x.validWindow[0] === windowStart);
+        if (!selves.length) return;
+        let prevTime = windowStart - 1;
+        let needDiskScan = false;
+        // sourceIndex -> the slice of our route that source owned in the previous window
+        let remoteOwners = new Map<number, [number, number]>();
+        for (let self of selves) {
+            let selfRoute = self.route || FULL_ROUTE;
+            let idx = effective.sources.indexOf(self);
+            // These scans only run on the write node - the first source valid at the new window's
+            // start for our route. An earlier entry fully covering our route means writes go there,
+            // and IT does the boundary scans.
+            let shadowed = false;
+            for (let j = 0; j < idx; j++) {
+                let other = effective.sources[j];
+                if (typeof other === "string") continue;
+                let [ws, we] = other.validWindow;
+                if (!(ws <= windowStart && windowStart < we)) continue;
+                let r = other.route || FULL_ROUTE;
+                if (r[0] <= selfRoute[0] && selfRoute[1] <= r[1]) {
+                    shadowed = true;
+                    break;
+                }
+            }
+            if (shadowed) continue;
+            // The previous window's owners: walking the config in order (write-target order),
+            // each source claims whatever slice of our route no earlier source already owns
+            let uncovered: [number, number][] = [[selfRoute[0], selfRoute[1]]];
+            for (let j = 0; j < effective.sources.length && uncovered.length; j++) {
+                let other = effective.sources[j];
+                if (typeof other === "string") continue;
+                let [ws, we] = other.validWindow;
+                if (!(ws <= prevTime && prevTime < we)) continue;
+                let r = other.route || FULL_ROUTE;
+                let remaining: [number, number][] = [];
+                let claimed: [number, number] | undefined;
+                for (let u of uncovered) {
+                    let inter = routeIntersection(u, r);
+                    if (!inter) {
+                        remaining.push(u);
+                        continue;
+                    }
+                    claimed = claimed && [Math.min(claimed[0], inter[0]), Math.max(claimed[1], inter[1])] as [number, number] || inter;
+                    if (u[0] < inter[0]) remaining.push([u[0], inter[0]]);
+                    if (inter[1] < u[1]) remaining.push([inter[1], u[1]]);
+                }
+                uncovered = remaining;
+                if (!claimed) continue;
+                if (selfIndexSet.has(j)) {
+                    // The previous owner shares our storage (same machine + bucket, e.g. the other
+                    // process of a deploy switchover), so a disk rescan sees its writes
+                    needDiskScan = true;
+                } else {
+                    let existing = remoteOwners.get(j);
+                    remoteOwners.set(j, existing && [Math.min(existing[0], claimed[0]), Math.max(existing[1], claimed[1])] as [number, number] || claimed);
+                }
+            }
+        }
+        if (!needDiskScan && !remoteOwners.size) return;
+        console.log(`Boundary scan (${label}): diskScan=${needDiskScan}, remote previous-window owners: ${remoteOwners.size}`);
+        if (needDiskScan) {
+            await store.rescanBase();
+        }
+        let since = windowStart - BOUNDARY_SCAN_LOOKBACK;
+        for (let [sourceIndex, route] of remoteOwners) {
+            let ownerSource = effective.sources[sourceIndex];
+            if (typeof ownerSource === "string") continue;
+            try {
+                await store.boundaryScanRemote(createApiArchives(ownerSource), { since, route });
+            } catch (e) {
+                console.error(`Boundary scan (${label}) of previous-window owner (source index ${sourceIndex}) failed: ${(e as Error).stack ?? e}`);
+            }
+        }
+    } finally {
+        boundaryScansRunning.delete(bucketKey);
+    }
+}
+
 function buildBucket(account: string, bucketName: string, routing: RemoteConfig): LoadedBucket {
     let folder = getBucketFolder(account, bucketName);
     // The deploy-takeover remap is an INTERPRETATION overlay: everything behavioral (self entries,
@@ -288,11 +432,11 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig)
                 void scheduleRoutingReload(account, bucketName);
             },
             readerDiskLimit: self?.readerDiskLimit,
-            getFlushDeadline,
         });
     }
     let loaded: LoadedBucket = { account, bucketName, routing, routingJSON: JSON.stringify(routing), selfEntries, self, store };
     scheduleWindowBoundaryRebuild(loaded);
+    scheduleBoundaryScans(loaded);
     return loaded;
 }
 
@@ -339,9 +483,9 @@ async function checkRoutingChanged(account: string, bucketName: string, config?:
     let key = `${account}/${bucketName}`;
     let loaded = await buckets.get(key);
     if (!loaded) return;
-    // get() cache-reads the file onto our disk when a remote source holds it, so the bucket still
-    // exists after a restart
-    let data = await loaded.store.get(ROUTING_FILE);
+    // The routing config is only ever read off our own disk - it is never pulled from another
+    // source (it reaches us exclusively as a version-validated write, which lands on disk first)
+    let data = await new ArchivesDisk(getBucketFolder(account, bucketName)).get(ROUTING_FILE);
     if (!data) return;
     let routing = parseRoutingData(data);
     if (!config?.force && JSON.stringify(routing) === loaded.routingJSON) return;
@@ -468,28 +612,12 @@ export async function rebuildAllLoadedBuckets(): Promise<void> {
     }
 }
 
-export async function rescanAllLoadedBucketDisks(): Promise<void> {
-    console.log(`Deploy switchover disk rescan starting (${buckets.size} loaded buckets) - catching files the other process wrote`);
-    for (let loadedPromise of [...buckets.values()]) {
-        let loaded = await loadedPromise.catch(() => undefined);
-        if (!loaded) continue;
-        let store = loaded.store;
-        if (!(store instanceof BlobStore)) continue;
-        try {
-            await store.rescanBase();
-        } catch (e) {
-            console.error(`Deploy switchover disk rescan failed for bucket ${loaded.account}/${loaded.bucketName}: ${(e as Error).stack ?? e}`);
-        }
-    }
-}
-
 onTakeoverEvent(event => {
     if (event === "remapChanged") {
+        // The rebuild applies the remapped windows, and scheduleBoundaryScans (part of the
+        // rebuild) schedules the handoff scans around the new windows' starts
         void rebuildAllLoadedBuckets();
     }
-    if (event === "diskScan") {
-        void rescanAllLoadedBucketDisks();
-    }
 });
 
 export type ServerBucketInfo = {
@@ -564,7 +692,7 @@ class ArchivesLocalBucket implements IArchives {
     }
 
     public getDebugName() {
-        return `localBucket/${this.account}/${this.bucketName}`;
+        return `localBucket account ${this.account} bucket ${this.bucketName}`;
     }
     public async get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
         let result = await this.get2(fileName, config);