sliftutils 1.7.48 → 1.7.49

package/storage/remoteStorage/remoteConfig.d.ts

@@ -1,11 +1,13 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } from "../IArchives";
+import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ChangesAfterConfig } from "../IArchives";
 export declare const ROUTING_FILE = "storage/storagerouting.json";
 /** The variable-shard route override embedded in the key ("<sentinel>_<value>", see VARIABLE_SHARD), or undefined when the key has no sentinel or the sentinel has no value yet. */
 export declare function parseVariableRoute(key: string): number | undefined;
 /** Where a key routes in [0, 1). A materialized variable-shard suffix completely overrides the hash. */
 export declare function getRoute(key: string): number;
+/** The in-memory getChangesAfter2 emulation, for backends without a native change feed: a full listing filtered down to files written after config.time whose keys route into config.routes. */
+export declare function filterChanges(files: ArchiveFileInfo[], config: ChangesAfterConfig): ArchiveFileInfo[];
 export declare function routeContains(route: [number, number] | undefined, value: number): boolean;
 export declare function routesOverlap(a: [number, number] | undefined, b: [number, number] | undefined): boolean;
 /** The overlap of two route ranges, or undefined when they don't overlap. */

package/storage/remoteStorage/remoteConfig.ts

@@ -2,7 +2,7 @@ import { sort } from "socket-function/src/misc";
 import { getBufferInt } from "socket-function/src/bits";
 import { setFlag } from "socket-function/require/compileFlags";
 import jsSha256 from "js-sha256";
-import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, FULL_VALID_WINDOW, FULL_ROUTE, VARIABLE_SHARD } from "../IArchives";
+import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ChangesAfterConfig, FULL_VALID_WINDOW, FULL_ROUTE, VARIABLE_SHARD } from "../IArchives";
 
 setFlag(require, "js-sha256", "allowclient", true);
 
@@ -31,6 +31,16 @@ export function getRoute(key: string): number {
     return hash % ROUTE_PRECISION / ROUTE_PRECISION;
 }
 
+/** The in-memory getChangesAfter2 emulation, for backends without a native change feed: a full listing filtered down to files written after config.time whose keys route into config.routes. */
+export function filterChanges(files: ArchiveFileInfo[], config: ChangesAfterConfig): ArchiveFileInfo[] {
+    return files.filter(file => {
+        if (file.createTime <= config.time) return false;
+        let routes = config.routes;
+        if (routes && !routes.some(route => routeContains(route, getRoute(file.path)))) return false;
+        return true;
+    });
+}
+
 // Route ranges are [start, end) - inclusive start, exclusive end
 export function routeContains(route: [number, number] | undefined, value: number): boolean {
     if (!route) return true;

package/storage/remoteStorage/sourcesList.d.ts

@@ -0,0 +1,15 @@
+export declare class SourcesList {
+    private filePath;
+    constructor(filePath: string);
+    private urls;
+    private indexes;
+    private endsClean;
+    private lastReload;
+    private appendQueue;
+    private load;
+    getUrl(sourcesListIndex: number): string | undefined;
+    /** For a sourcesListIndex beyond our in-memory list (another process appended since we read): re-reads the file, at most once per RELOAD_THROTTLE. Returning undefined can therefore mean "throttled", not "definitely absent" - never treat it as proof the index is bogus. */
+    getUrlReloading(sourcesListIndex: number): Promise<string | undefined>;
+    /** The sourcesListIndex of the url, appending it if it is new. Appends are serialized within this process; before each append the file is re-read, so appends by other processes are picked up instead of duplicated. */
+    ensure(url: string): Promise<number>;
+}

package/storage/remoteStorage/sourcesList.ts

@@ -0,0 +1,69 @@
+import fs from "fs";
+
+const RELOAD_THROTTLE = 5 * 1000;
+
+// The persistent identity behind IndexEntry.sourcesListIndex: an append-only file of source URLs / disk folder paths, one per line, where a URL's 0-based line number IS the sourcesListIndex the index persists. Because the file is only ever appended to, a line number means the same URL forever - across restarts and across processes sharing the folder. A torn append (crash mid-write) leaves an unterminated partial line; the next writer terminates it before appending, so the garbage line permanently occupies its index (harmless - it never matches a real URL) and every reader agrees on the numbering.
+export class SourcesList {
+    constructor(private filePath: string) { }
+    private urls: string[] = [];
+    private indexes = new Map<string, number>();
+    private endsClean = true;
+    private lastReload = 0;
+    private appendQueue = Promise.resolve();
+
+    private async load(): Promise<void> {
+        let content = "";
+        try {
+            content = await fs.promises.readFile(this.filePath, "utf8");
+        } catch (e) {
+            if ((e as { code?: string }).code !== "ENOENT") throw e;
+        }
+        this.endsClean = !content || content.endsWith("\n");
+        let lines = content.split("\n");
+        if (lines[lines.length - 1] === "") {
+            lines.pop();
+        }
+        this.urls = lines;
+        this.indexes.clear();
+        for (let i = 0; i < lines.length; i++) {
+            if (!this.indexes.has(lines[i])) {
+                this.indexes.set(lines[i], i);
+            }
+        }
+    }
+
+    public getUrl(sourcesListIndex: number): string | undefined {
+        return this.urls[sourcesListIndex];
+    }
+
+    /** For a sourcesListIndex beyond our in-memory list (another process appended since we read): re-reads the file, at most once per RELOAD_THROTTLE. Returning undefined can therefore mean "throttled", not "definitely absent" - never treat it as proof the index is bogus. */
+    public async getUrlReloading(sourcesListIndex: number): Promise<string | undefined> {
+        if (sourcesListIndex < this.urls.length) return this.urls[sourcesListIndex];
+        if (Date.now() - this.lastReload < RELOAD_THROTTLE) return undefined;
+        this.lastReload = Date.now();
+        await this.load();
+        return this.urls[sourcesListIndex];
+    }
+
+    /** The sourcesListIndex of the url, appending it if it is new. Appends are serialized within this process; before each append the file is re-read, so appends by other processes are picked up instead of duplicated. */
+    public ensure(url: string): Promise<number> {
+        if (url.includes("\n")) {
+            throw new Error(`Source URLs cannot contain newlines (they are stored one per line): ${JSON.stringify(url)}`);
+        }
+        let result = this.appendQueue.then(async () => {
+            await this.load();
+            let existing = this.indexes.get(url);
+            if (existing !== undefined) return existing;
+            let prefix = this.endsClean && "" || "\n";
+            await fs.promises.appendFile(this.filePath, prefix + url + "\n");
+            this.endsClean = true;
+            let index = this.urls.length;
+            this.urls.push(url);
+            this.indexes.set(url, index);
+            console.log(`Registered new source ${JSON.stringify(url)} as sourcesListIndex ${index} in ${this.filePath}`);
+            return index;
+        });
+        this.appendQueue = result.then(() => { }, () => { });
+        return result;
+    }
+}

package/storage/remoteStorage/storageController.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig } from "../IArchives";
 import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
 export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
@@ -70,7 +70,7 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         writeTime: number;
         size: number;
     } | undefined>;
-    set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number) => Promise<void>;
+    set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean) => Promise<void>;
     del: (account: string, bucketName: string, path: string) => Promise<void>;
     getInfo: (account: string, bucketName: string, path: string) => Promise<{
         writeTime: number;
@@ -80,7 +80,7 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         shallow?: boolean;
         type?: "files" | "folders";
     }) => Promise<ArchiveFileInfo[]>;
-    getChangesAfter: (account: string, bucketName: string, time: number) => Promise<ArchiveFileInfo[]>;
+    getChangesAfter2: (account: string, bucketName: string, config: ChangesAfterConfig) => Promise<ArchiveFileInfo[]>;
     getArchivesConfig: (account: string, bucketName: string) => Promise<ArchivesConfig>;
     listBuckets: (account: string) => Promise<ServerBucketInfo[]>;
     getActiveBucket: (account: string, bucketName: string) => Promise<ActiveBucketInfo | string>;

package/storage/remoteStorage/storageController.ts

@@ -6,7 +6,7 @@ import { performLocalCall } from "socket-function/src/callManager";
 import { RequireController } from "socket-function/require/RequireController";
 import { timeInMinute } from "socket-function/src/misc";
 import { getCommonName, getPublicIdentifier, getOwnMachineId, verify, verifyMachineIdForPublicKey } from "../../misc/https/certs";
-import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, IMMUTABLE_CACHE_TIME } from "../IArchives";
+import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, IMMUTABLE_CACHE_TIME } from "../IArchives";
 import { ROUTING_FILE } from "./remoteConfig";
 import {
     getStorageServerConfig, getTrust, getRequests, getLoadedBucket, writeBucketFile,
@@ -294,10 +294,10 @@ class RemoteStorageControllerBase {
         if (!bucket) return undefined;
         return await bucket.store.get2(path, { range });
     }
-    async set(account: string, bucketName: string, path: string, data: Buffer, lastModified?: number): Promise<void> {
+    async set(account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean): Promise<void> {
         assertValidName(bucketName, "bucket name");
         assertValidPath(path);
-        await writeBucketFile(account, bucketName, path, Buffer.from(data), { lastModified });
+        await writeBucketFile(account, bucketName, path, Buffer.from(data), { lastModified, forceSetImmutable });
     }
     async del(account: string, bucketName: string, path: string): Promise<void> {
         assertValidName(bucketName, "bucket name");
@@ -315,13 +315,10 @@ class RemoteStorageControllerBase {
         if (!bucket) return [];
         return await bucket.store.findInfo(prefix, config);
     }
-    async getChangesAfter(account: string, bucketName: string, time: number): Promise<ArchiveFileInfo[]> {
+    async getChangesAfter2(account: string, bucketName: string, config: ChangesAfterConfig): Promise<ArchiveFileInfo[]> {
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return [];
-        if (!bucket.store.getChangesAfter) {
-            throw new Error(`Bucket ${account}/${bucketName} does not support getChangesAfter (rawDisk buckets have no index)`);
-        }
-        return await bucket.store.getChangesAfter(time);
+        return await bucket.store.getChangesAfter2(config);
     }
     async getArchivesConfig(account: string, bucketName: string): Promise<ArchivesConfig> {
         let bucket = await getBucket(account, bucketName);
@@ -523,7 +520,7 @@ export const RemoteStorageController = SocketFunction.register(
         del: { hooks: [accountAccess] },
         getInfo: { hooks: [accountAccess] },
         findInfo: { hooks: [accountAccess] },
-        getChangesAfter: { hooks: [accountAccess] },
+        getChangesAfter2: { hooks: [accountAccess] },
         getArchivesConfig: { hooks: [accountAccess] },
         getIndexInfo: { hooks: [accountAccess] },
         listBuckets: { hooks: [accountAccess] },

package/storage/remoteStorage/storageServerState.d.ts

@@ -46,10 +46,13 @@ export type LoadedBucket = {
 };
 export declare function addExtraListenPort(port: number): void;
 export declare function removeExtraListenPort(port: number): void;
+/** A cached IArchives for a persisted source identity: a routing URL (hosted/backblaze) or a disk folder path - the form BlobStore's sources list stores. Configuration (valid windows, routes) decides WHEN a source should be used; for reading bytes the index says a source holds, the URL alone is enough - even for sources no longer in any config. */
+export declare function resolveSourceArchives(url: string): IArchives;
 export declare function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined>;
 export declare function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void>;
 export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
     lastModified?: number;
+    forceSetImmutable?: boolean;
 }): Promise<void>;
 export declare function getBucketConfig(bucket: LoadedBucket): ArchivesConfig;
 /** Which buckets this process currently has loaded - what a deploy successor asks its predecessor for, so it activates exactly the buckets that are actually in use. */

package/storage/remoteStorage/storageServerState.ts

@@ -9,10 +9,10 @@ import { ArchivesDisk } from "../ArchivesDisk";
 import { BlobStore, IBucketStore, DEFAULT_FAST_WRITE_DELAY, WINDOW_END_FLUSH_MARGIN } from "./blobStore";
 import {
     RemoteConfig, HostedConfig, BackblazeConfig, IArchives, ArchivesSource, ArchiveFileInfo, ArchivesConfig,
-    ArchivesSyncStatus,
+    ArchivesSyncStatus, ChangesAfterConfig, SetConfig,
     STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
 } from "../IArchives";
-import { ROUTING_FILE, parseRoutingData, serializeRemoteConfig, parseHostedUrl, replaceHostedUrlPort, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
+import { ROUTING_FILE, parseRoutingData, serializeRemoteConfig, parseHostedUrl, replaceHostedUrlPort, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection, normalizeSource } from "./remoteConfig";
 import { injectIntermediateSource, expireIntermediateSources, getIntermediateSources, findSplitUrl, nextIntermediateVersion, INTERMEDIATE_EXPIRE_GRACE } from "./intermediateSources";
 import { getTakeoverIntermediate } from "./deployTakeover";
 import { createApiArchives, listServerActiveBucketKeys } from "./createArchives";
@@ -424,6 +424,21 @@ type StorePlan = {
     structureKey: string;
 };
 
+const resolvedSourceArchives = new Map<string, IArchives>();
+/** A cached IArchives for a persisted source identity: a routing URL (hosted/backblaze) or a disk folder path - the form BlobStore's sources list stores. Configuration (valid windows, routes) decides WHEN a source should be used; for reading bytes the index says a source holds, the URL alone is enough - even for sources no longer in any config. */
+export function resolveSourceArchives(url: string): IArchives {
+    let existing = resolvedSourceArchives.get(url);
+    if (existing) return existing;
+    let archives: IArchives;
+    if (url.startsWith("https://")) {
+        archives = createApiArchives(normalizeSource(url));
+    } else {
+        archives = new ArchivesDisk(url);
+    }
+    resolvedSourceArchives.set(url, archives);
+    return archives;
+}
+
 function sourceIdentity(sourceConfig: HostedConfig | BackblazeConfig | undefined): string {
     if (!sourceConfig) return "disk";
     return JSON.stringify({ ...sourceConfig, validWindow: undefined, route: undefined });
@@ -491,6 +506,7 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig,
         }
         let sources: ArchivesSource[] = plan.sourceSpecs.map(spec => ({
             source: spec.sourceConfig && createApiArchives(spec.sourceConfig) || new ArchivesDisk(folder),
+            url: spec.sourceConfig?.url || folder,
             validWindow: spec.validWindow,
             route: spec.route,
             noFullSync: spec.noFullSync,
@@ -503,6 +519,7 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig,
             },
             readerDiskLimit: plan.readerDiskLimit,
             onWriteCounted: (kind, bytes) => countBucketWrite(`${account}/${bucketName}`, kind, bytes),
+            resolveSourceUrl: resolveSourceArchives,
         });
     }
     let loaded: LoadedBucket = { account, bucketName, routing, routingJSON: JSON.stringify(routing), selfEntries, self, store, structureKey: plan.structureKey };
@@ -584,10 +601,11 @@ async function checkRoutingChanged(account: string, bucketName: string, config?:
         loaded.store.updateSources(plan.sourceSpecs.map(spec => {
             const sourceConfig = spec.sourceConfig;
             if (!sourceConfig) {
-                return { identity: sourceIdentity(undefined), validWindow: spec.validWindow, create: (): IArchives => new ArchivesDisk(getBucketFolder(account, bucketName)) };
+                return { identity: sourceIdentity(undefined), url: getBucketFolder(account, bucketName), validWindow: spec.validWindow, create: (): IArchives => new ArchivesDisk(getBucketFolder(account, bucketName)) };
             }
             return {
                 identity: sourceIdentity(sourceConfig),
+                url: sourceConfig.url,
                 validWindow: spec.validWindow,
                 route: spec.route,
                 noFullSync: spec.noFullSync,
@@ -666,7 +684,7 @@ async function writeRoutingConfig(account: string, bucketName: string, data: Buf
     await scheduleRoutingReload(account, bucketName);
 }
 
-export async function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
+export async function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: { lastModified?: number; forceSetImmutable?: boolean }): Promise<void> {
     assertWritesAllowed();
     if (filePath === ROUTING_FILE) {
         let key = `${account}/${bucketName}`;
@@ -691,14 +709,24 @@ export async function writeBucketFile(account: string, bucketName: string, fileP
             throw new Error(`${STORAGE_WRONG_ROUTE} This server does not handle route ${route} (key ${JSON.stringify(filePath)}) for bucket ${account}/${bucketName} (our routes at this time: ${JSON.stringify(timeValid.map(x => x.route || FULL_ROUTE))}). Re-resolve the source for this key and retry.`);
         }
     }
-    await assertMutable(loaded, filePath, writeTime);
+    if (config?.forceSetImmutable) {
+        if (!config.lastModified) {
+            throw new Error(`forceSetImmutable requires lastModified (synchronization writes are ordered by their write time), writing ${JSON.stringify(filePath)} to bucket ${account}/${bucketName}`);
+        }
+        // Immutability wins: an existing path is kept instead of the push throwing (see SetConfig.forceSetImmutable)
+        let self = selectEntryAt(loaded.selfEntries, writeTime, route);
+        if (self?.immutable && await loaded.store.getInfo(filePath)) return;
+    } else {
+        await assertMutable(loaded, filePath, writeTime);
+    }
     await loaded.store.set(filePath, data, { ...getWriteConfig(loaded, writeTime, route), lastModified: writeTime });
 }
 
 export function getBucketConfig(bucket: LoadedBucket): ArchivesConfig {
     let progress = bucket.store.getSyncProgress?.();
     return {
-        supportsChangesAfter: !!bucket.store.getChangesAfter,
+        // Native change-feed support = the store keeps an index (rawDisk buckets emulate getChangesAfter2 with a full listing) - clients use this to pick their scan cadence
+        supportsChangesAfter: bucket.store instanceof BlobStore,
         remoteConfig: bucket.routing,
         index: progress?.index,
         indexSources: progress?.sources,
@@ -1033,7 +1061,7 @@ class ArchivesLocalBucket implements IArchives {
         if (!bucket) return undefined;
         return await bucket.store.get2(fileName, config);
     }
-    public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
+    public async set(fileName: string, data: Buffer, config?: SetConfig): Promise<string> {
         await writeBucketFile(this.account, this.bucketName, fileName, data, config);
         return fileName;
     }
@@ -1087,13 +1115,10 @@ class ArchivesLocalBucket implements IArchives {
     public async hasWriteAccess(): Promise<boolean> {
         return true;
     }
-    public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
+    public async getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]> {
         let bucket = await this.getBucket();
         if (!bucket) return [];
-        if (!bucket.store.getChangesAfter) {
-            throw new Error(`Bucket ${this.account}/${this.bucketName} does not support getChangesAfter (rawDisk buckets have no index)`);
-        }
-        return await bucket.store.getChangesAfter(time);
+        return await bucket.store.getChangesAfter2(config);
     }
     public async getSyncStatus(): Promise<ArchivesSyncStatus> {
         let bucket = await this.getBucket();