sliftutils 1.7.48 → 1.7.49

package/index.d.ts

@@ -858,7 +858,7 @@ declare module "sliftutils/render-utils/observer" {
 declare module "sliftutils/storage/ArchivesDisk" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesConfig } from "./IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, SetConfig } from "./IArchives";
     export declare class ArchivesDisk implements IArchives {
         private folder;
         constructor(folder: string);
@@ -874,11 +874,10 @@ declare module "sliftutils/storage/ArchivesDisk" {
         };
         getDebugName(): string;
         getConfig(): Promise<ArchivesConfig>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         hasWriteAccess(): Promise<boolean>;
         private filePath;
-        set(key: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(key: string, data: Buffer, config?: SetConfig): Promise<string>;
         del(key: string): Promise<void>;
         get(key: string, config?: {
             range?: {
@@ -2149,6 +2148,18 @@ declare module "sliftutils/storage/IArchives" {
         allowedOrigins?: string[];
     };
     export declare const FULL_VALID_WINDOW: [number, number];
+    export type ChangesAfterConfig = {
+        time: number;
+        /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
+        routes?: [number, number][];
+    };
+    export type SetConfig = {
+        lastModified?: number;
+        /** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
+        forceSetImmutable?: boolean;
+        /** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
+        noChecks?: boolean;
+    };
     export type ArchiveFileInfo = {
         path: string;
         createTime: number;
@@ -2180,6 +2191,8 @@ declare module "sliftutils/storage/IArchives" {
     };
     export type ArchivesSource = {
         source: IArchives;
+        /** The persistent identity of the endpoint: its routing URL (hosted/backblaze), or the disk folder path for the base disk source. The store persists this (via its append-only sources list) as IndexEntry.sourcesListIndex, so it must mean the same endpoint forever. */
+        url: string;
         validWindow: [number, number];
         route?: [number, number];
         noFullSync?: boolean;
@@ -2233,9 +2246,7 @@ declare module "sliftutils/storage/IArchives" {
          * VARIABLE_SHARD, where the shard value is materialized into the key (picked by shard latency,
          * see ArchivesChain) and the caller needs the returned key to ever read the value back.
          */
-        set(fileName: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
         del(fileName: string): Promise<void>;
         /** Streams a file too large to hold in memory. getNextData returns undefined when done. */
         setLargeFile(config: {
@@ -2260,10 +2271,13 @@ declare module "sliftutils/storage/IArchives" {
         /** The bucket's configuration, which tells whether the optional functions are supported. */
         getConfig(): Promise<ArchivesConfig>;
         /**
-         * All files changed after the given time. Only exists when getConfig().supportsChangesAfter;
-         * backed by an index, so it is fast (unlike a full findInfo scan). Deletions are not reported.
+         * All files changed after config.time, optionally restricted to keys routing into one of
+         * config.routes (used by scanning, so partially-overlapping shards only receive their slice).
+         * When getConfig().supportsChangesAfter, this is backed by an index (fast, and deletions ARE
+         * reported, as size-0 tombstone entries). Every other backend emulates it: a full findInfo
+         * listing filtered in memory - correct, but no cheaper than the listing itself.
          */
-        getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         /** Synchronization introspection, for backends that synchronize from sources (see BlobStore). */
         getSyncStatus?(): Promise<ArchivesSyncStatus>;
     }
@@ -2526,7 +2540,7 @@ declare module "sliftutils/storage/TransactionStorage" {
 declare module "sliftutils/storage/backblaze" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchivesConfig } from "./IArchives";
+    import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, SetConfig } from "./IArchives";
     export declare class ArchivesBackblaze implements IArchives {
         private config;
         constructor(config: {
@@ -2565,9 +2579,7 @@ declare module "sliftutils/storage/backblaze" {
         } | undefined>;
         getConfig(): Promise<ArchivesConfig>;
         hasWriteAccess(): Promise<boolean>;
-        set(fileName: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
         del(fileName: string): Promise<void>;
         setLargeFile(config: {
             path: string;
@@ -2581,6 +2593,7 @@ declare module "sliftutils/storage/backblaze" {
             shallow?: boolean;
             type: "files" | "folders";
         }): Promise<string[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         findInfo(prefix: string, config?: {
             shallow?: boolean;
             type: "files" | "folders";
@@ -2841,7 +2854,7 @@ declare module "sliftutils/storage/remoteFileStorage" {
 declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, SetConfig } from "../IArchives";
     export type ArchivesRemoteConfig = {
         url: string;
         waitForAccess?: boolean;
@@ -2896,9 +2909,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
             writeTime: number;
             size: number;
         } | undefined>;
-        set(fileName: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
         del(fileName: string): Promise<void>;
         getInfo(fileName: string): Promise<{
             writeTime: number;
@@ -2912,7 +2923,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
             shallow?: boolean;
             type: "files" | "folders";
         }): Promise<string[]>;
-        getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getConfig(): Promise<ArchivesConfig>;
         getSyncStatus(): Promise<ArchivesSyncStatus>;
         setLargeFile(config: {
@@ -2927,7 +2938,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
 declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig } from "../IArchives";
     export declare class ArchivesUrl implements IArchives {
         private base;
         constructor(base: string);
@@ -2969,6 +2980,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
             shallow?: boolean;
             type: "files" | "folders";
         }): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getURL(path: string): Promise<string>;
         getConfig(): Promise<ArchivesConfig>;
         hasWriteAccess(): Promise<boolean>;
@@ -2984,7 +2996,7 @@ declare module "sliftutils/storage/remoteStorage/accessPage" {
 declare module "sliftutils/storage/remoteStorage/blobStore" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, SyncActivity } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, SyncActivity } from "../IArchives";
     export declare const DEFAULT_FAST_WRITE_DELAY: number;
     export declare const WINDOW_END_FLUSH_MARGIN: number;
     export type WriteConfig = {
@@ -3019,7 +3031,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
             shallow?: boolean;
             type?: "files" | "folders";
         }): Promise<ArchiveFileInfo[]>;
-        getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getSyncStatus?(): Promise<ArchivesSyncStatus>;
         getSyncProgress?(): {
             index: {
@@ -3050,6 +3062,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
     };
     export type BlobSourceSpec = {
         identity: string;
+        url: string;
         validWindow: [number, number];
         route?: [number, number];
         noFullSync?: boolean;
@@ -3063,6 +3076,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
             onIndexChanged?: ((key: string) => void) | undefined;
             readerDiskLimit?: number | undefined;
             onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined;
+            resolveSourceUrl?: ((url: string) => IArchives) | undefined;
         } | undefined);
         private stopped;
         private index;
@@ -3076,7 +3090,14 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
         private overlay;
         private sourceStates;
         private syncStarted;
+        private sourcesList;
+        private slotSourcesListIndexes;
+        private slotRegistrations;
         private isLive;
+        private registerSlot;
+        private sourcesListIndexOfSlot;
+        private slotForSourcesListIndex;
+        private getEntryHolder;
         init: {
             (): Promise<void>;
             reset(): void;
@@ -3125,7 +3146,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
         private runSourceSync;
         private scanSource;
         private reconcileSource;
-        private applyScanned;
+        private updateScanIndex;
         private pollChanges;
         private copySourceFiles;
         private waitForRequiredScans;
@@ -3160,7 +3181,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
             shallow?: boolean;
             type?: "files" | "folders";
         }): Promise<ArchiveFileInfo[]>;
-        getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getSyncStatus(): Promise<ArchivesSyncStatus>;
         private getDiskSource;
         startLargeUpload(): Promise<string>;
@@ -3183,7 +3204,7 @@ declare module "sliftutils/storage/remoteStorage/cliArgs" {
 declare module "sliftutils/storage/remoteStorage/createArchives" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+    import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, SetConfig } from "../IArchives";
     import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
     /** The address, port, account, and bucket name a bucket routing URL addresses. Throws when the URL isn't a hosted bucket routing URL (https://host:port/file/<account>/<bucketName>/storage/storagerouting.json). */
     export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteConfig";
@@ -3260,13 +3281,11 @@ declare module "sliftutils/storage/remoteStorage/createArchives" {
             shallow?: boolean;
             type: "files" | "folders";
         }): Promise<ArchiveFileInfo[]>;
-        getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+        getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
         getSyncStatus(): Promise<ArchivesSyncStatus>;
         getConfig(): Promise<ArchivesConfig>;
         hasWriteAccess(): Promise<boolean>;
-        set(fileName: string, data: Buffer, config?: {
-            lastModified?: number;
-        }): Promise<string>;
+        set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
         private setRoutingConfig;
         del(fileName: string): Promise<void>;
         private setVariableShard;
@@ -3376,12 +3395,14 @@ declare module "sliftutils/storage/remoteStorage/productionEnv" {
 declare module "sliftutils/storage/remoteStorage/remoteConfig" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } from "../IArchives";
+    import { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ChangesAfterConfig } from "../IArchives";
     export declare const ROUTING_FILE = "storage/storagerouting.json";
     /** The variable-shard route override embedded in the key ("<sentinel>_<value>", see VARIABLE_SHARD), or undefined when the key has no sentinel or the sentinel has no value yet. */
     export declare function parseVariableRoute(key: string): number | undefined;
     /** Where a key routes in [0, 1). A materialized variable-shard suffix completely overrides the hash. */
     export declare function getRoute(key: string): number;
+    /** The in-memory getChangesAfter2 emulation, for backends without a native change feed: a full listing filtered down to files written after config.time whose keys route into config.routes. */
+    export declare function filterChanges(files: ArchiveFileInfo[], config: ChangesAfterConfig): ArchiveFileInfo[];
     export declare function routeContains(route: [number, number] | undefined, value: number): boolean;
     export declare function routesOverlap(a: [number, number] | undefined, b: [number, number] | undefined): boolean;
     /** The overlap of two route ranges, or undefined when they don't overlap. */
@@ -3460,6 +3481,25 @@ declare module "sliftutils/storage/remoteStorage/sourceWrapper" {
 
 }
 
+declare module "sliftutils/storage/remoteStorage/sourcesList" {
+    export declare class SourcesList {
+        private filePath;
+        constructor(filePath: string);
+        private urls;
+        private indexes;
+        private endsClean;
+        private lastReload;
+        private appendQueue;
+        private load;
+        getUrl(sourcesListIndex: number): string | undefined;
+        /** For a sourcesListIndex beyond our in-memory list (another process appended since we read): re-reads the file, at most once per RELOAD_THROTTLE. Returning undefined can therefore mean "throttled", not "definitely absent" - never treat it as proof the index is bogus. */
+        getUrlReloading(sourcesListIndex: number): Promise<string | undefined>;
+        /** The sourcesListIndex of the url, appending it if it is new. Appends are serialized within this process; before each append the file is re-read, so appends by other processes are picked up instead of duplicated. */
+        ensure(url: string): Promise<number>;
+    }
+
+}
+
 declare module "sliftutils/storage/remoteStorage/storageClientController" {
     /** Subscribe to server-pushed routing change notifications. Returns the unsubscribe function. */
     export declare function onServerRoutingChanged(listener: () => void): () => void;
@@ -3472,7 +3512,7 @@ declare module "sliftutils/storage/remoteStorage/storageClientController" {
 declare module "sliftutils/storage/remoteStorage/storageController" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+    import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig } from "../IArchives";
     import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
     export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
     export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
@@ -3542,7 +3582,7 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
             writeTime: number;
             size: number;
         } | undefined>;
-        set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number) => Promise<void>;
+        set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean) => Promise<void>;
         del: (account: string, bucketName: string, path: string) => Promise<void>;
         getInfo: (account: string, bucketName: string, path: string) => Promise<{
             writeTime: number;
@@ -3552,7 +3592,7 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
             shallow?: boolean;
             type?: "files" | "folders";
         }) => Promise<ArchiveFileInfo[]>;
-        getChangesAfter: (account: string, bucketName: string, time: number) => Promise<ArchiveFileInfo[]>;
+        getChangesAfter2: (account: string, bucketName: string, config: ChangesAfterConfig) => Promise<ArchiveFileInfo[]>;
         getArchivesConfig: (account: string, bucketName: string) => Promise<ArchivesConfig>;
         listBuckets: (account: string) => Promise<ServerBucketInfo[]>;
         getActiveBucket: (account: string, bucketName: string) => Promise<ActiveBucketInfo | string>;
@@ -3647,10 +3687,13 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
     };
     export declare function addExtraListenPort(port: number): void;
     export declare function removeExtraListenPort(port: number): void;
+    /** A cached IArchives for a persisted source identity: a routing URL (hosted/backblaze) or a disk folder path - the form BlobStore's sources list stores. Configuration (valid windows, routes) decides WHEN a source should be used; for reading bytes the index says a source holds, the URL alone is enough - even for sources no longer in any config. */
+    export declare function resolveSourceArchives(url: string): IArchives;
     export declare function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined>;
     export declare function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void>;
     export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
         lastModified?: number;
+        forceSetImmutable?: boolean;
     }): Promise<void>;
     export declare function getBucketConfig(bucket: LoadedBucket): ArchivesConfig;
     /** Which buckets this process currently has loaded - what a deploy successor asks its predecessor for, so it activates exactly the buckets that are actually in use. */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.48",
+    "version": "1.7.49",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/ArchivesDisk.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchiveFileInfo, ArchivesConfig } from "./IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, SetConfig } from "./IArchives";
 export declare class ArchivesDisk implements IArchives {
     private folder;
     constructor(folder: string);
@@ -16,11 +16,10 @@ export declare class ArchivesDisk implements IArchives {
     };
     getDebugName(): string;
     getConfig(): Promise<ArchivesConfig>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     hasWriteAccess(): Promise<boolean>;
     private filePath;
-    set(key: string, data: Buffer, config?: {
-        lastModified?: number;
-    }): Promise<string>;
+    set(key: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(key: string): Promise<void>;
     get(key: string, config?: {
         range?: {

package/storage/ArchivesDisk.ts

@@ -3,7 +3,8 @@ import path from "path";
 import { lazy } from "socket-function/src/caching";
 import { runInfinitePoll } from "socket-function/src/batching";
 import { sort, binarySearchBasic } from "socket-function/src/misc";
-import { IArchives, ArchiveFileInfo, ArchivesConfig, assertValidLastModified } from "./IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, SetConfig, assertValidLastModified } from "./IArchives";
+import { filterChanges } from "./remoteStorage/remoteConfig";
 
 // The base file-system IArchives: storage is one-to-one with the file system, every key is exactly one real file under <folder>/files, so the file system itself is the index. File handles are cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in serial, so they can't collide with each other or with handle closing. Used as the disk synchronization source of BlobStore (see remoteStorage/blobStore.ts).
 
@@ -124,6 +125,11 @@ export class ArchivesDisk implements IArchives {
         return {};
     }
 
+    public async getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]> {
+        // No native change feed - a full listing filtered in memory (see the scanning note in BlobStore.scanSource for why the listing itself takes no filters)
+        return filterChanges(await this.findInfo(""), config);
+    }
+
     public async hasWriteAccess(): Promise<boolean> {
         return true;
     }
@@ -136,7 +142,8 @@ export class ArchivesDisk implements IArchives {
         return result;
     }
 
-    public async set(key: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
+    // forceSetImmutable is accepted and needs no handling: disk sources are never immutable, and the older-write no-op below already gives synchronization its only-take-the-latest semantics
+    public async set(key: string, data: Buffer, config?: SetConfig): Promise<string> {
         await this.init();
         let lastModified = config?.lastModified;
         if (lastModified) {

package/storage/IArchives.d.ts

@@ -48,6 +48,18 @@ export type BackblazeConfig = CommonConfig & {
     allowedOrigins?: string[];
 };
 export declare const FULL_VALID_WINDOW: [number, number];
+export type ChangesAfterConfig = {
+    time: number;
+    /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
+    routes?: [number, number][];
+};
+export type SetConfig = {
+    lastModified?: number;
+    /** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
+    forceSetImmutable?: boolean;
+    /** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
+    noChecks?: boolean;
+};
 export type ArchiveFileInfo = {
     path: string;
     createTime: number;
@@ -79,6 +91,8 @@ export type ArchivesConfig = {
 };
 export type ArchivesSource = {
     source: IArchives;
+    /** The persistent identity of the endpoint: its routing URL (hosted/backblaze), or the disk folder path for the base disk source. The store persists this (via its append-only sources list) as IndexEntry.sourcesListIndex, so it must mean the same endpoint forever. */
+    url: string;
     validWindow: [number, number];
     route?: [number, number];
     noFullSync?: boolean;
@@ -132,9 +146,7 @@ export interface IArchives {
      * VARIABLE_SHARD, where the shard value is materialized into the key (picked by shard latency,
      * see ArchivesChain) and the caller needs the returned key to ever read the value back.
      */
-    set(fileName: string, data: Buffer, config?: {
-        lastModified?: number;
-    }): Promise<string>;
+    set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(fileName: string): Promise<void>;
     /** Streams a file too large to hold in memory. getNextData returns undefined when done. */
     setLargeFile(config: {
@@ -159,10 +171,13 @@ export interface IArchives {
     /** The bucket's configuration, which tells whether the optional functions are supported. */
     getConfig(): Promise<ArchivesConfig>;
     /**
-     * All files changed after the given time. Only exists when getConfig().supportsChangesAfter;
-     * backed by an index, so it is fast (unlike a full findInfo scan). Deletions are not reported.
+     * All files changed after config.time, optionally restricted to keys routing into one of
+     * config.routes (used by scanning, so partially-overlapping shards only receive their slice).
+     * When getConfig().supportsChangesAfter, this is backed by an index (fast, and deletions ARE
+     * reported, as size-0 tombstone entries). Every other backend emulates it: a full findInfo
+     * listing filtered in memory - correct, but no cheaper than the listing itself.
      */
-    getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     /** Synchronization introspection, for backends that synchronize from sources (see BlobStore). */
     getSyncStatus?(): Promise<ArchivesSyncStatus>;
 }

package/storage/IArchives.ts

@@ -58,7 +58,7 @@ export type HostedConfig = CommonConfig & {
     // Fast mode: the server acknowledges writes once they are in memory, flushing to disk after writeDelay (default 5 minutes) and coalescing writes to the same file. A server crash loses writes that haven't flushed yet.
     fast?: boolean;
     writeDelay?: number;
-    // The bucket is served straight from the server's disk, with no index — so no fast writes and no getChangesAfter/getSyncStatus.
+    // The bucket is served straight from the server's disk, with no index — so no fast writes, no getSyncStatus, and getChangesAfter2 falls back to a full listing.
     rawDisk?: boolean;
     // Writes to paths that already exist are disallowed (deletes still work).
     immutable?: boolean;
@@ -86,6 +86,20 @@ export const FULL_VALID_WINDOW: [number, number] = [0, Number.MAX_SAFE_INTEGER];
 
 
 
+export type ChangesAfterConfig = {
+    time: number;
+    /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
+    routes?: [number, number][];
+};
+
+export type SetConfig = {
+    lastModified?: number;
+    /** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
+    forceSetImmutable?: boolean;
+    /** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
+    noChecks?: boolean;
+};
+
 // createTime is a misnomer kept for compatibility — it is really the LAST-WRITE time, same as getInfo's writeTime. Neither Backblaze nor our remote storage tracks a distinct creation date: each write stamps a fresh timestamp on the current version, so both fields are just "when the bytes served by get() were most recently written".
 export type ArchiveFileInfo = { path: string; createTime: number; size: number };
 
@@ -102,7 +116,7 @@ export type SyncActivity = {
 };
 
 export type ArchivesConfig = {
-    // Whether getChangesAfter is implemented (fast change polling, instead of full rescans)
+    // Whether getChangesAfter2 is natively index-backed (fast change polling; every backend still serves it, but the others emulate it with a full listing)
     supportsChangesAfter?: boolean;
     // The bucket's full routing config (ROUTING_FILE). Absent for sources that don't have one (a bare disk source, or a bucket that doesn't exist yet).
     remoteConfig?: RemoteConfig;
@@ -119,6 +133,8 @@ export type ArchivesConfig = {
 // A synchronization source of a BlobStore (which synchronizes an index + local cache from them)
 export type ArchivesSource = {
     source: IArchives;
+    /** The persistent identity of the endpoint: its routing URL (hosted/backblaze), or the disk folder path for the base disk source. The store persists this (via its append-only sources list) as IndexEntry.sourcesListIndex, so it must mean the same endpoint forever. */
+    url: string;
     // From the source's CommonConfig. Values with write times outside the window are ignored when scanning, and once its end is past (see windowAcceptsWrites) the source stops receiving writes entirely - still scanned (it holds the authoritative data for its window), it just stops growing.
     validWindow: [number, number];
     // From the source's CommonConfig (intersected with the owning store's own route): only keys routing into [start, end) are accepted from this source's scans and sent to it in writes/reconciliation. The routing file is exempt - config flows everywhere. Absent = all keys.
@@ -177,7 +193,7 @@ export interface IArchives {
      * VARIABLE_SHARD, where the shard value is materialized into the key (picked by shard latency,
      * see ArchivesChain) and the caller needs the returned key to ever read the value back.
      */
-    set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string>;
+    set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(fileName: string): Promise<void>;
     /** Streams a file too large to hold in memory. getNextData returns undefined when done. */
     setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void>;
@@ -190,10 +206,13 @@ export interface IArchives {
     /** The bucket's configuration, which tells whether the optional functions are supported. */
     getConfig(): Promise<ArchivesConfig>;
     /**
-     * All files changed after the given time. Only exists when getConfig().supportsChangesAfter;
-     * backed by an index, so it is fast (unlike a full findInfo scan). Deletions are not reported.
+     * All files changed after config.time, optionally restricted to keys routing into one of
+     * config.routes (used by scanning, so partially-overlapping shards only receive their slice).
+     * When getConfig().supportsChangesAfter, this is backed by an index (fast, and deletions ARE
+     * reported, as size-0 tombstone entries). Every other backend emulates it: a full findInfo
+     * listing filtered in memory - correct, but no cheaper than the listing itself.
      */
-    getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     /** Synchronization introspection, for backends that synchronize from sources (see BlobStore). */
     getSyncStatus?(): Promise<ArchivesSyncStatus>;
 }

package/storage/backblaze.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchivesConfig } from "./IArchives";
+import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, SetConfig } from "./IArchives";
 export declare class ArchivesBackblaze implements IArchives {
     private config;
     constructor(config: {
@@ -39,9 +39,7 @@ export declare class ArchivesBackblaze implements IArchives {
     } | undefined>;
     getConfig(): Promise<ArchivesConfig>;
     hasWriteAccess(): Promise<boolean>;
-    set(fileName: string, data: Buffer, config?: {
-        lastModified?: number;
-    }): Promise<string>;
+    set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(fileName: string): Promise<void>;
     setLargeFile(config: {
         path: string;
@@ -55,6 +53,7 @@ export declare class ArchivesBackblaze implements IArchives {
         shallow?: boolean;
         type: "files" | "folders";
     }): Promise<string[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     findInfo(prefix: string, config?: {
         shallow?: boolean;
         type: "files" | "folders";