sliftutils 1.7.48 → 1.7.50

package/index.d.ts

@@ -858,7 +858,7 @@ declare module "sliftutils/render-utils/observer" {
 declare module "sliftutils/storage/ArchivesDisk" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesConfig } from "./IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, SetConfig } from "./IArchives";
     export declare class ArchivesDisk implements IArchives {
         private folder;
         constructor(folder: string);
@@ -874,11 +874,10 @@ declare module "sliftutils/storage/ArchivesDisk" {
         };
         getDebugName(): string;
         getConfig(): Promise<ArchivesConfig>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         hasWriteAccess(): Promise<boolean>;
         private filePath;
-        set(key: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(key: string, data: Buffer, config?: SetConfig): Promise<string>;
         del(key: string): Promise<void>;
         get(key: string, config?: {
             range?: {
@@ -911,11 +910,12 @@ declare module "sliftutils/storage/ArchivesDisk" {
         private collectFiles;
         setLargeFile(config: {
             path: string;
+            lastModified?: number;
             getNextData(): Promise<Buffer | undefined>;
         }): Promise<void>;
         startLargeUpload(): Promise<string>;
         appendLargeUpload(id: string, data: Buffer): Promise<void>;
-        finishLargeUpload(id: string, key: string): Promise<void>;
+        finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void>;
         cancelLargeUpload(id: string): Promise<void>;
         getURL(path: string): Promise<string>;
     }
@@ -2149,6 +2149,18 @@ declare module "sliftutils/storage/IArchives" {
         allowedOrigins?: string[];
     };
     export declare const FULL_VALID_WINDOW: [number, number];
+    export type ChangesAfterConfig = {
+        time: number;
+        /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
+        routes?: [number, number][];
+    };
+    export type SetConfig = {
+        lastModified?: number;
+        /** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
+        forceSetImmutable?: boolean;
+        /** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
+        noChecks?: boolean;
+    };
     export type ArchiveFileInfo = {
         path: string;
         createTime: number;
@@ -2180,6 +2192,8 @@ declare module "sliftutils/storage/IArchives" {
     };
     export type ArchivesSource = {
         source: IArchives;
+        /** The persistent identity of the endpoint: its routing URL (hosted/backblaze), or the disk folder path for the base disk source. The store persists this (via its append-only sources list) as IndexEntry.sourcesListIndex, so it must mean the same endpoint forever. */
+        url: string;
         validWindow: [number, number];
         route?: [number, number];
         noFullSync?: boolean;
@@ -2190,6 +2204,19 @@ declare module "sliftutils/storage/IArchives" {
     export declare const FULL_ROUTE: [number, number];
     export declare const VARIABLE_SHARD = "VARIABLE_SHARD_f0234jfah08fgyhfgyssdds83nmp";
     export declare function windowAcceptsWrites(validWindow: [number, number] | undefined): boolean;
+    /** Copies one file between two archives. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. size/writeTime usually come from the caller's metadata scan; when either is omitted, getInfo fills them in. Returns the copied file's info, or undefined when the source doesn't have the file. */
+    export declare function copyArchiveFile(config: {
+        from: IArchives;
+        to: IArchives;
+        path: string;
+        size?: number;
+        writeTime?: number;
+        forceSetImmutable?: boolean;
+        noChecks?: boolean;
+    }): Promise<{
+        writeTime: number;
+        size: number;
+    } | undefined>;
     export type ArchivesSyncSourceStatus = {
         debugName: string;
         validWindow: [number, number];
@@ -2233,13 +2260,12 @@ declare module "sliftutils/storage/IArchives" {
          * VARIABLE_SHARD, where the shard value is materialized into the key (picked by shard latency,
          * see ArchivesChain) and the caller needs the returned key to ever read the value back.
          */
-        set(fileName: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
         del(fileName: string): Promise<void>;
-        /** Streams a file too large to hold in memory. getNextData returns undefined when done. */
+        /** Streams a file too large to hold in memory. getNextData returns undefined when done. lastModified stamps the finished file like set's (synchronized copies need it to keep write ordering); backends that stamp their own times (backblaze) accept and ignore it. */
         setLargeFile(config: {
             path: string;
+            lastModified?: number;
             getNextData(): Promise<Buffer | undefined>;
         }): Promise<void>;
         /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. */
@@ -2260,10 +2286,13 @@ declare module "sliftutils/storage/IArchives" {
         /** The bucket's configuration, which tells whether the optional functions are supported. */
         getConfig(): Promise<ArchivesConfig>;
         /**
-         * All files changed after the given time. Only exists when getConfig().supportsChangesAfter;
-         * backed by an index, so it is fast (unlike a full findInfo scan). Deletions are not reported.
+         * All files changed after config.time, optionally restricted to keys routing into one of
+         * config.routes (used by scanning, so partially-overlapping shards only receive their slice).
+         * When getConfig().supportsChangesAfter, this is backed by an index (fast, and deletions ARE
+         * reported, as size-0 tombstone entries). Every other backend emulates it: a full findInfo
+         * listing filtered in memory - correct, but no cheaper than the listing itself.
          */
-        getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         /** Synchronization introspection, for backends that synchronize from sources (see BlobStore). */
         getSyncStatus?(): Promise<ArchivesSyncStatus>;
     }
@@ -2526,7 +2555,7 @@ declare module "sliftutils/storage/TransactionStorage" {
 declare module "sliftutils/storage/backblaze" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchivesConfig } from "./IArchives";
+    import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, SetConfig } from "./IArchives";
     export declare class ArchivesBackblaze implements IArchives {
         private config;
         constructor(config: {
@@ -2565,12 +2594,11 @@ declare module "sliftutils/storage/backblaze" {
         } | undefined>;
         getConfig(): Promise<ArchivesConfig>;
         hasWriteAccess(): Promise<boolean>;
-        set(fileName: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
         del(fileName: string): Promise<void>;
         setLargeFile(config: {
             path: string;
+            lastModified?: number;
             getNextData(): Promise<Buffer | undefined>;
         }): Promise<void>;
         getInfo(fileName: string): Promise<{
@@ -2581,6 +2609,7 @@ declare module "sliftutils/storage/backblaze" {
             shallow?: boolean;
             type: "files" | "folders";
         }): Promise<string[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         findInfo(prefix: string, config?: {
             shallow?: boolean;
             type: "files" | "folders";
@@ -2841,7 +2870,7 @@ declare module "sliftutils/storage/remoteFileStorage" {
 declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, SetConfig } from "../IArchives";
     export type ArchivesRemoteConfig = {
         url: string;
         waitForAccess?: boolean;
@@ -2896,9 +2925,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
             writeTime: number;
             size: number;
         } | undefined>;
-        set(fileName: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
         del(fileName: string): Promise<void>;
         getInfo(fileName: string): Promise<{
             writeTime: number;
@@ -2912,11 +2939,12 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
             shallow?: boolean;
             type: "files" | "folders";
         }): Promise<string[]>;
-        getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getConfig(): Promise<ArchivesConfig>;
         getSyncStatus(): Promise<ArchivesSyncStatus>;
         setLargeFile(config: {
             path: string;
+            lastModified?: number;
             getNextData(): Promise<Buffer | undefined>;
         }): Promise<void>;
         getURL(path: string): Promise<string>;
@@ -2927,7 +2955,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
 declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig } from "../IArchives";
     export declare class ArchivesUrl implements IArchives {
         private base;
         constructor(base: string);
@@ -2969,6 +2997,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
             shallow?: boolean;
             type: "files" | "folders";
         }): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getURL(path: string): Promise<string>;
         getConfig(): Promise<ArchivesConfig>;
         hasWriteAccess(): Promise<boolean>;
@@ -2984,7 +3013,7 @@ declare module "sliftutils/storage/remoteStorage/accessPage" {
 declare module "sliftutils/storage/remoteStorage/blobStore" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, SyncActivity } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, SyncActivity } from "../IArchives";
     export declare const DEFAULT_FAST_WRITE_DELAY: number;
     export declare const WINDOW_END_FLUSH_MARGIN: number;
     export type WriteConfig = {
@@ -3019,7 +3048,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
             shallow?: boolean;
             type?: "files" | "folders";
         }): Promise<ArchiveFileInfo[]>;
-        getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getSyncStatus?(): Promise<ArchivesSyncStatus>;
         getSyncProgress?(): {
             index: {
@@ -3045,11 +3074,12 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
         }>;
         startLargeUpload(): Promise<string>;
         appendLargeUpload(id: string, data: Buffer): Promise<void>;
-        finishLargeUpload(id: string, key: string): Promise<void>;
+        finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void>;
         cancelLargeUpload(id: string): Promise<void>;
     };
     export type BlobSourceSpec = {
         identity: string;
+        url: string;
         validWindow: [number, number];
         route?: [number, number];
         noFullSync?: boolean;
@@ -3063,6 +3093,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
             onIndexChanged?: ((key: string) => void) | undefined;
             readerDiskLimit?: number | undefined;
             onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined;
+            resolveSourceUrl?: ((url: string) => IArchives) | undefined;
         } | undefined);
         private stopped;
         private index;
@@ -3076,7 +3107,14 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
         private overlay;
         private sourceStates;
         private syncStarted;
+        private sourcesList;
+        private slotSourcesListIndexes;
+        private slotRegistrations;
         private isLive;
+        private registerSlot;
+        private sourcesListIndexOfSlot;
+        private slotForSourcesListIndex;
+        private getEntryHolder;
         init: {
             (): Promise<void>;
             reset(): void;
@@ -3125,7 +3163,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
         private runSourceSync;
         private scanSource;
         private reconcileSource;
-        private applyScanned;
+        private updateScanIndex;
         private pollChanges;
         private copySourceFiles;
         private waitForRequiredScans;
@@ -3160,12 +3198,12 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
             shallow?: boolean;
             type?: "files" | "folders";
         }): Promise<ArchiveFileInfo[]>;
-        getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getSyncStatus(): Promise<ArchivesSyncStatus>;
         private getDiskSource;
         startLargeUpload(): Promise<string>;
         appendLargeUpload(id: string, data: Buffer): Promise<void>;
-        finishLargeUpload(id: string, key: string): Promise<void>;
+        finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void>;
         cancelLargeUpload(id: string): Promise<void>;
         private flushOverlay;
         private evicting;
@@ -3183,7 +3221,7 @@ declare module "sliftutils/storage/remoteStorage/cliArgs" {
 declare module "sliftutils/storage/remoteStorage/createArchives" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+    import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, SetConfig } from "../IArchives";
     import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
     /** The address, port, account, and bucket name a bucket routing URL addresses. Throws when the URL isn't a hosted bucket routing URL (https://host:port/file/<account>/<bucketName>/storage/storagerouting.json). */
     export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteConfig";
@@ -3260,18 +3298,17 @@ declare module "sliftutils/storage/remoteStorage/createArchives" {
             shallow?: boolean;
             type: "files" | "folders";
         }): Promise<ArchiveFileInfo[]>;
-        getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getSyncStatus(): Promise<ArchivesSyncStatus>;
         getConfig(): Promise<ArchivesConfig>;
         hasWriteAccess(): Promise<boolean>;
-        set(fileName: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
         private setRoutingConfig;
         del(fileName: string): Promise<void>;
         private setVariableShard;
         setLargeFile(config: {
             path: string;
+            lastModified?: number;
             getNextData(): Promise<Buffer | undefined>;
         }): Promise<void>;
         getURL(path: string): Promise<string>;
@@ -3376,12 +3413,14 @@ declare module "sliftutils/storage/remoteStorage/productionEnv" {
 declare module "sliftutils/storage/remoteStorage/remoteConfig" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } from "../IArchives";
+    import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ChangesAfterConfig } from "../IArchives";
     export declare const ROUTING_FILE = "storage/storagerouting.json";
     /** The variable-shard route override embedded in the key ("<sentinel>_<value>", see VARIABLE_SHARD), or undefined when the key has no sentinel or the sentinel has no value yet. */
     export declare function parseVariableRoute(key: string): number | undefined;
     /** Where a key routes in [0, 1). A materialized variable-shard suffix completely overrides the hash. */
     export declare function getRoute(key: string): number;
+    /** The in-memory getChangesAfter2 emulation, for backends without a native change feed: a full listing filtered down to files written after config.time whose keys route into config.routes. */
+    export declare function filterChanges(files: ArchiveFileInfo[], config: ChangesAfterConfig): ArchiveFileInfo[];
     export declare function routeContains(route: [number, number] | undefined, value: number): boolean;
     export declare function routesOverlap(a: [number, number] | undefined, b: [number, number] | undefined): boolean;
     /** The overlap of two route ranges, or undefined when they don't overlap. */
@@ -3460,6 +3499,25 @@ declare module "sliftutils/storage/remoteStorage/sourceWrapper" {
 
 }
 
+declare module "sliftutils/storage/remoteStorage/sourcesList" {
+    export declare class SourcesList {
+        private filePath;
+        constructor(filePath: string);
+        private urls;
+        private indexes;
+        private endsClean;
+        private lastReload;
+        private appendQueue;
+        private load;
+        getUrl(sourcesListIndex: number): string | undefined;
+        /** For a sourcesListIndex beyond our in-memory list (another process appended since we read): re-reads the file, at most once per RELOAD_THROTTLE. Returning undefined can therefore mean "throttled", not "definitely absent" - never treat it as proof the index is bogus. */
+        getUrlReloading(sourcesListIndex: number): Promise<string | undefined>;
+        /** The sourcesListIndex of the url, appending it if it is new. Appends are serialized within this process; before each append the file is re-read, so appends by other processes are picked up instead of duplicated. */
+        ensure(url: string): Promise<number>;
+    }
+
+}
+
 declare module "sliftutils/storage/remoteStorage/storageClientController" {
     /** Subscribe to server-pushed routing change notifications. Returns the unsubscribe function. */
     export declare function onServerRoutingChanged(listener: () => void): () => void;
@@ -3472,7 +3530,7 @@ declare module "sliftutils/storage/remoteStorage/storageClientController" {
 declare module "sliftutils/storage/remoteStorage/storageController" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+    import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig } from "../IArchives";
     import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
     export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
     export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
@@ -3542,7 +3600,7 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
             writeTime: number;
             size: number;
         } | undefined>;
-        set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number) => Promise<void>;
+        set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean) => Promise<void>;
         del: (account: string, bucketName: string, path: string) => Promise<void>;
         getInfo: (account: string, bucketName: string, path: string) => Promise<{
             writeTime: number;
@@ -3552,7 +3610,7 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
             shallow?: boolean;
             type?: "files" | "folders";
         }) => Promise<ArchiveFileInfo[]>;
-        getChangesAfter: (account: string, bucketName: string, time: number) => Promise<ArchiveFileInfo[]>;
+        getChangesAfter2: (account: string, bucketName: string, config: ChangesAfterConfig) => Promise<ArchiveFileInfo[]>;
         getArchivesConfig: (account: string, bucketName: string) => Promise<ArchivesConfig>;
         listBuckets: (account: string) => Promise<ServerBucketInfo[]>;
         getActiveBucket: (account: string, bucketName: string) => Promise<ActiveBucketInfo | string>;
@@ -3570,7 +3628,7 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
             }[];
         } | undefined>;
         getSyncStatus: (account: string, bucketName: string) => Promise<ArchivesSyncStatus>;
-        startLargeFile: (account: string, bucketName: string, path: string) => Promise<string>;
+        startLargeFile: (account: string, bucketName: string, path: string, lastModified?: number) => Promise<string>;
         uploadPart: (uploadId: string, data: Buffer) => Promise<void>;
         finishLargeFile: (uploadId: string) => Promise<void>;
         cancelLargeFile: (uploadId: string) => Promise<void>;
@@ -3647,10 +3705,13 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
     };
     export declare function addExtraListenPort(port: number): void;
     export declare function removeExtraListenPort(port: number): void;
+    /** A cached IArchives for a persisted source identity: a routing URL (hosted/backblaze) or a disk folder path - the form BlobStore's sources list stores. Configuration (valid windows, routes) decides WHEN a source should be used; for reading bytes the index says a source holds, the URL alone is enough - even for sources no longer in any config. */
+    export declare function resolveSourceArchives(url: string): IArchives;
     export declare function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined>;
     export declare function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void>;
     export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
         lastModified?: number;
+        forceSetImmutable?: boolean;
     }): Promise<void>;
     export declare function getBucketConfig(bucket: LoadedBucket): ArchivesConfig;
     /** Which buckets this process currently has loaded - what a deploy successor asks its predecessor for, so it activates exactly the buckets that are actually in use. */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.48",
+    "version": "1.7.50",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/ArchivesDisk.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchiveFileInfo, ArchivesConfig } from "./IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, SetConfig } from "./IArchives";
 export declare class ArchivesDisk implements IArchives {
     private folder;
     constructor(folder: string);
@@ -16,11 +16,10 @@ export declare class ArchivesDisk implements IArchives {
     };
     getDebugName(): string;
     getConfig(): Promise<ArchivesConfig>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     hasWriteAccess(): Promise<boolean>;
     private filePath;
-    set(key: string, data: Buffer, config?: {
-        lastModified?: number;
-    }): Promise<string>;
+    set(key: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(key: string): Promise<void>;
     get(key: string, config?: {
         range?: {
@@ -53,11 +52,12 @@ export declare class ArchivesDisk implements IArchives {
     private collectFiles;
     setLargeFile(config: {
         path: string;
+        lastModified?: number;
         getNextData(): Promise<Buffer | undefined>;
     }): Promise<void>;
     startLargeUpload(): Promise<string>;
     appendLargeUpload(id: string, data: Buffer): Promise<void>;
-    finishLargeUpload(id: string, key: string): Promise<void>;
+    finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void>;
     cancelLargeUpload(id: string): Promise<void>;
     getURL(path: string): Promise<string>;
 }

package/storage/ArchivesDisk.ts

@@ -3,7 +3,8 @@ import path from "path";
 import { lazy } from "socket-function/src/caching";
 import { runInfinitePoll } from "socket-function/src/batching";
 import { sort, binarySearchBasic } from "socket-function/src/misc";
-import { IArchives, ArchiveFileInfo, ArchivesConfig, assertValidLastModified } from "./IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, SetConfig, assertValidLastModified } from "./IArchives";
+import { filterChanges } from "./remoteStorage/remoteConfig";
 
 // The base file-system IArchives: storage is one-to-one with the file system, every key is exactly one real file under <folder>/files, so the file system itself is the index. File handles are cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in serial, so they can't collide with each other or with handle closing. Used as the disk synchronization source of BlobStore (see remoteStorage/blobStore.ts).
 
@@ -124,6 +125,11 @@ export class ArchivesDisk implements IArchives {
         return {};
     }
 
+    public async getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]> {
+        // No native change feed - a full listing filtered in memory (see the scanning note in BlobStore.scanSource for why the listing itself takes no filters)
+        return filterChanges(await this.findInfo(""), config);
+    }
+
     public async hasWriteAccess(): Promise<boolean> {
         return true;
     }
@@ -136,7 +142,8 @@ export class ArchivesDisk implements IArchives {
         return result;
     }
 
-    public async set(key: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
+    // forceSetImmutable is accepted and needs no handling: disk sources are never immutable, and the older-write no-op below already gives synchronization its only-take-the-latest semantics
+    public async set(key: string, data: Buffer, config?: SetConfig): Promise<string> {
         await this.init();
         let lastModified = config?.lastModified;
         if (lastModified) {
@@ -255,7 +262,7 @@ export class ArchivesDisk implements IArchives {
         }
     }
 
-    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
+    public async setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
         let id = await this.startLargeUpload();
         try {
             while (true) {
@@ -263,7 +270,7 @@ export class ArchivesDisk implements IArchives {
                 if (!data) break;
                 await this.appendLargeUpload(id, data);
             }
-            await this.finishLargeUpload(id, config.path);
+            await this.finishLargeUpload(id, config.path, config.lastModified);
         } catch (e) {
             await this.cancelLargeUpload(id);
             throw e;
@@ -286,9 +293,12 @@ export class ArchivesDisk implements IArchives {
             await handle.write(data, 0, data.length);
         });
     }
-    public async finishLargeUpload(id: string, key: string): Promise<void> {
+    public async finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void> {
         let upload = this.largeUploads.get(id);
         if (!upload) throw new Error(`Unknown large upload ${id}`);
+        if (lastModified) {
+            assertValidLastModified(lastModified);
+        }
         const tmpPath = upload.tmpPath;
         this.largeUploads.delete(id);
         let filePath = this.filePath(key);
@@ -304,6 +314,10 @@ export class ArchivesDisk implements IArchives {
                 // Nothing was ever appended, so the upload file was never created
                 await fs.promises.writeFile(filePath, Buffer.alloc(0));
             }
+            // The rename preserves the temp file's mtime, which is just when the last append happened - the logical write time has to be stamped explicitly (it is the metadata scans order everything by)
+            if (lastModified) {
+                await fs.promises.utimes(filePath, new Date(lastModified), new Date(lastModified));
+            }
         });
     }
     public async cancelLargeUpload(id: string): Promise<void> {

package/storage/IArchives.d.ts

@@ -48,6 +48,18 @@ export type BackblazeConfig = CommonConfig & {
     allowedOrigins?: string[];
 };
 export declare const FULL_VALID_WINDOW: [number, number];
+export type ChangesAfterConfig = {
+    time: number;
+    /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
+    routes?: [number, number][];
+};
+export type SetConfig = {
+    lastModified?: number;
+    /** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
+    forceSetImmutable?: boolean;
+    /** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
+    noChecks?: boolean;
+};
 export type ArchiveFileInfo = {
     path: string;
     createTime: number;
@@ -79,6 +91,8 @@ export type ArchivesConfig = {
 };
 export type ArchivesSource = {
     source: IArchives;
+    /** The persistent identity of the endpoint: its routing URL (hosted/backblaze), or the disk folder path for the base disk source. The store persists this (via its append-only sources list) as IndexEntry.sourcesListIndex, so it must mean the same endpoint forever. */
+    url: string;
     validWindow: [number, number];
     route?: [number, number];
     noFullSync?: boolean;
@@ -89,6 +103,19 @@ export declare const STORAGE_WRONG_ROUTE = "REMOTE_STORAGE_WRONG_ROUTE_c94d2e17"
 export declare const FULL_ROUTE: [number, number];
 export declare const VARIABLE_SHARD = "VARIABLE_SHARD_f0234jfah08fgyhfgyssdds83nmp";
 export declare function windowAcceptsWrites(validWindow: [number, number] | undefined): boolean;
+/** Copies one file between two archives. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. size/writeTime usually come from the caller's metadata scan; when either is omitted, getInfo fills them in. Returns the copied file's info, or undefined when the source doesn't have the file. */
+export declare function copyArchiveFile(config: {
+    from: IArchives;
+    to: IArchives;
+    path: string;
+    size?: number;
+    writeTime?: number;
+    forceSetImmutable?: boolean;
+    noChecks?: boolean;
+}): Promise<{
+    writeTime: number;
+    size: number;
+} | undefined>;
 export type ArchivesSyncSourceStatus = {
     debugName: string;
     validWindow: [number, number];
@@ -132,13 +159,12 @@ export interface IArchives {
      * VARIABLE_SHARD, where the shard value is materialized into the key (picked by shard latency,
      * see ArchivesChain) and the caller needs the returned key to ever read the value back.
      */
-    set(fileName: string, data: Buffer, config?: {
-        lastModified?: number;
-    }): Promise<string>;
+    set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(fileName: string): Promise<void>;
-    /** Streams a file too large to hold in memory. getNextData returns undefined when done. */
+    /** Streams a file too large to hold in memory. getNextData returns undefined when done. lastModified stamps the finished file like set's (synchronized copies need it to keep write ordering); backends that stamp their own times (backblaze) accept and ignore it. */
     setLargeFile(config: {
         path: string;
+        lastModified?: number;
         getNextData(): Promise<Buffer | undefined>;
     }): Promise<void>;
     /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. */
@@ -159,10 +185,13 @@ export interface IArchives {
     /** The bucket's configuration, which tells whether the optional functions are supported. */
     getConfig(): Promise<ArchivesConfig>;
     /**
-     * All files changed after the given time. Only exists when getConfig().supportsChangesAfter;
-     * backed by an index, so it is fast (unlike a full findInfo scan). Deletions are not reported.
+     * All files changed after config.time, optionally restricted to keys routing into one of
+     * config.routes (used by scanning, so partially-overlapping shards only receive their slice).
+     * When getConfig().supportsChangesAfter, this is backed by an index (fast, and deletions ARE
+     * reported, as size-0 tombstone entries). Every other backend emulates it: a full findInfo
+     * listing filtered in memory - correct, but no cheaper than the listing itself.
      */
-    getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     /** Synchronization introspection, for backends that synchronize from sources (see BlobStore). */
     getSyncStatus?(): Promise<ArchivesSyncStatus>;
 }