sliftutils 1.7.51 → 1.7.53

package/index.d.ts

@@ -858,7 +858,7 @@ declare module "sliftutils/render-utils/observer" {
 declare module "sliftutils/storage/ArchivesDisk" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, SetConfig } from "./IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, SetConfig } from "./IArchives";
     export declare class ArchivesDisk implements IArchives {
         private folder;
         constructor(folder: string);
@@ -879,18 +879,8 @@ declare module "sliftutils/storage/ArchivesDisk" {
         private filePath;
         set(key: string, data: Buffer, config?: SetConfig): Promise<string>;
         del(key: string): Promise<void>;
-        get(key: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<Buffer | undefined>;
-        get2(key: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<{
+        get(key: string, config?: GetConfig): Promise<Buffer | undefined>;
+        get2(key: string, config?: GetConfig): Promise<{
             data: Buffer;
             writeTime: number;
             size: number;
@@ -2149,6 +2139,14 @@ declare module "sliftutils/storage/IArchives" {
         allowedOrigins?: string[];
     };
     export declare const FULL_VALID_WINDOW: [number, number];
+    export type GetConfig = {
+        range?: {
+            start: number;
+            end: number;
+        };
+        /** Read ONLY from the primary source - the one writes would target - and throw when it cannot serve the read, instead of falling back across the redundant sources. */
+        noFallbacks?: boolean;
+    };
     export type ChangesAfterConfig = {
         time: number;
         /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
@@ -2235,18 +2233,16 @@ declare module "sliftutils/storage/IArchives" {
         getDebugName(): string;
         /** Whether writes would be accepted (credentials exist, the account trusts this machine, etc). Checked without writing anything. */
         hasWriteAccess(): Promise<boolean>;
-        get(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<Buffer | undefined>;
-        get2(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<{
+        /**
+         * Reads automatically fall back across the redundant sources unless config.noFallbacks is set.
+         * A fallback copy can lag the write target, so a caller reading state in order to mutate it
+         * (e.g. x++), where acting on previous state would cause big issues, should pass noFallbacks -
+         * and try/catch the read, handling the catch case (a down primary is retried for a while, then
+         * throws instead of degrading to a stale copy).
+         */
+        get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
+        /** See get for the fallback semantics (and when to pass noFallbacks). */
+        get2(fileName: string, config?: GetConfig): Promise<{
             data: Buffer;
             writeTime: number;
             size: number;
@@ -2555,7 +2551,7 @@ declare module "sliftutils/storage/TransactionStorage" {
 declare module "sliftutils/storage/backblaze" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, SetConfig } from "./IArchives";
+    import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, SetConfig } from "./IArchives";
     export declare class ArchivesBackblaze implements IArchives {
         private config;
         constructor(config: {
@@ -2575,19 +2571,8 @@ declare module "sliftutils/storage/backblaze" {
         private currentReset;
         private last503Reset;
         private apiRetryLogic;
-        get(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-            retryCount?: number;
-        }): Promise<Buffer | undefined>;
-        get2(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<{
+        get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
+        get2(fileName: string, config?: GetConfig): Promise<{
             data: Buffer;
             writeTime: number;
             size: number;
@@ -2870,7 +2855,7 @@ declare module "sliftutils/storage/remoteFileStorage" {
 declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, SetConfig } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SetConfig } from "../IArchives";
     export type ArchivesRemoteConfig = {
         url: string;
         waitForAccess?: boolean;
@@ -2909,18 +2894,8 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
         hasWriteAccess(): Promise<boolean>;
         private registerAccessRequest;
         private call;
-        get(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<Buffer | undefined>;
-        get2(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<{
+        get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
+        get2(fileName: string, config?: GetConfig): Promise<{
             data: Buffer;
             writeTime: number;
             size: number;
@@ -2955,24 +2930,14 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
 declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig } from "../IArchives";
     export declare class ArchivesUrl implements IArchives {
         private base;
         constructor(base: string);
         getDebugName(): string;
         private readOnlyError;
-        get(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<Buffer | undefined>;
-        get2(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<{
+        get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
+        get2(fileName: string, config?: GetConfig): Promise<{
             data: Buffer;
             writeTime: number;
             size: number;
@@ -3013,7 +2978,7 @@ declare module "sliftutils/storage/remoteStorage/accessPage" {
 declare module "sliftutils/storage/remoteStorage/blobStore" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, SyncActivity } from "../IArchives";
+    import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SyncActivity } from "../IArchives";
     export declare const DEFAULT_FAST_WRITE_DELAY: number;
     export declare const WINDOW_END_FLUSH_MARGIN: number;
     export type WriteConfig = {
@@ -3022,18 +2987,8 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
         lastModified?: number;
     };
     export type IBucketStore = {
-        get(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<Buffer | undefined>;
-        get2(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<{
+        get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
+        get2(fileName: string, config?: GetConfig): Promise<{
             data: Buffer;
             writeTime: number;
             size: number;
@@ -3169,18 +3124,8 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
         private waitForRequiredScans;
         private checkMissingKey;
         private getIndexEntry;
-        get(key: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<Buffer | undefined>;
-        get2(key: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<{
+        get(key: string, config?: GetConfig): Promise<Buffer | undefined>;
+        get2(key: string, config?: GetConfig): Promise<{
             data: Buffer;
             writeTime: number;
             size: number;
@@ -3221,7 +3166,7 @@ declare module "sliftutils/storage/remoteStorage/cliArgs" {
 declare module "sliftutils/storage/remoteStorage/createArchives" {
     /// <reference types="node" />
     /// <reference types="node" />
-    import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, SetConfig } from "../IArchives";
+    import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SetConfig } from "../IArchives";
     import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
     /** The address, port, account, and bucket name a bucket routing URL addresses. Throws when the URL isn't a hosted bucket routing URL (https://host:port/file/<account>/<bucketName>/storage/storagerouting.json). */
     export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteConfig";
@@ -3268,18 +3213,8 @@ declare module "sliftutils/storage/remoteStorage/createArchives" {
             machineId: string;
             ip: string;
         } | undefined>;
-        get(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<Buffer | undefined>;
-        get2(fileName: string, config?: {
-            range?: {
-                start: number;
-                end: number;
-            };
-        }): Promise<{
+        get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
+        get2(fileName: string, config?: GetConfig): Promise<{
             data: Buffer;
             writeTime: number;
             size: number;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.51",
+    "version": "1.7.53",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/ArchivesDisk.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, SetConfig } from "./IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, SetConfig } from "./IArchives";
 export declare class ArchivesDisk implements IArchives {
     private folder;
     constructor(folder: string);
@@ -21,18 +21,8 @@ export declare class ArchivesDisk implements IArchives {
     private filePath;
     set(key: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(key: string): Promise<void>;
-    get(key: string, config?: {
-        range?: {
-            start: number;
-            end: number;
-        };
-    }): Promise<Buffer | undefined>;
-    get2(key: string, config?: {
-        range?: {
-            start: number;
-            end: number;
-        };
-    }): Promise<{
+    get(key: string, config?: GetConfig): Promise<Buffer | undefined>;
+    get2(key: string, config?: GetConfig): Promise<{
         data: Buffer;
         writeTime: number;
         size: number;

package/storage/ArchivesDisk.ts

@@ -3,7 +3,7 @@ import path from "path";
 import { lazy } from "socket-function/src/caching";
 import { runInfinitePoll } from "socket-function/src/batching";
 import { sort, binarySearchBasic } from "socket-function/src/misc";
-import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, SetConfig, assertValidLastModified } from "./IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, SetConfig, assertValidLastModified } from "./IArchives";
 import { filterChanges } from "./remoteStorage/remoteConfig";
 
 // The base file-system IArchives: storage is one-to-one with the file system, every key is exactly one real file under <folder>/files, so the file system itself is the index. File handles are cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in serial, so they can't collide with each other or with handle closing. Used as the disk synchronization source of BlobStore (see remoteStorage/blobStore.ts).
@@ -180,12 +180,12 @@ export class ArchivesDisk implements IArchives {
         });
     }
 
-    public async get(key: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
+    public async get(key: string, config?: GetConfig): Promise<Buffer | undefined> {
         let result = await this.get2(key, config);
         return result && result.data || undefined;
     }
 
-    public async get2(key: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
+    public async get2(key: string, config?: GetConfig): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         await this.init();
         let range = config?.range;
         let filePath = this.filePath(key);

package/storage/IArchives.d.ts

@@ -48,6 +48,14 @@ export type BackblazeConfig = CommonConfig & {
     allowedOrigins?: string[];
 };
 export declare const FULL_VALID_WINDOW: [number, number];
+export type GetConfig = {
+    range?: {
+        start: number;
+        end: number;
+    };
+    /** Read ONLY from the primary source - the one writes would target - and throw when it cannot serve the read, instead of falling back across the redundant sources. */
+    noFallbacks?: boolean;
+};
 export type ChangesAfterConfig = {
     time: number;
     /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
@@ -134,18 +142,16 @@ export interface IArchives {
     getDebugName(): string;
     /** Whether writes would be accepted (credentials exist, the account trusts this machine, etc). Checked without writing anything. */
     hasWriteAccess(): Promise<boolean>;
-    get(fileName: string, config?: {
-        range?: {
-            start: number;
-            end: number;
-        };
-    }): Promise<Buffer | undefined>;
-    get2(fileName: string, config?: {
-        range?: {
-            start: number;
-            end: number;
-        };
-    }): Promise<{
+    /**
+     * Reads automatically fall back across the redundant sources unless config.noFallbacks is set.
+     * A fallback copy can lag the write target, so a caller reading state in order to mutate it
+     * (e.g. x++), where acting on previous state would cause big issues, should pass noFallbacks -
+     * and try/catch the read, handling the catch case (a down primary is retried for a while, then
+     * throws instead of degrading to a stale copy).
+     */
+    get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
+    /** See get for the fallback semantics (and when to pass noFallbacks). */
+    get2(fileName: string, config?: GetConfig): Promise<{
         data: Buffer;
         writeTime: number;
         size: number;

package/storage/IArchives.ts

@@ -86,6 +86,12 @@ export const FULL_VALID_WINDOW: [number, number] = [0, Number.MAX_SAFE_INTEGER];
 
 
 
+export type GetConfig = {
+    range?: { start: number; end: number };
+    /** Read ONLY from the primary source - the one writes would target - and throw when it cannot serve the read, instead of falling back across the redundant sources. */
+    noFallbacks?: boolean;
+};
+
 export type ChangesAfterConfig = {
     time: number;
     /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
@@ -231,8 +237,16 @@ export interface IArchives {
     getDebugName(): string;
     /** Whether writes would be accepted (credentials exist, the account trusts this machine, etc). Checked without writing anything. */
     hasWriteAccess(): Promise<boolean>;
-    get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined>;
-    get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined>;
+    /**
+     * Reads automatically fall back across the redundant sources unless config.noFallbacks is set.
+     * A fallback copy can lag the write target, so a caller reading state in order to mutate it
+     * (e.g. x++), where acting on previous state would cause big issues, should pass noFallbacks -
+     * and try/catch the read, handling the catch case (a down primary is retried for a while, then
+     * throws instead of degrading to a stale copy).
+     */
+    get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
+    /** See get for the fallback semantics (and when to pass noFallbacks). */
+    get2(fileName: string, config?: GetConfig): Promise<{ data: Buffer; writeTime: number; size: number } | undefined>;
     /**
      * lastModified stamps the write with that last-write time instead of now. If it is OLDER than
      * the file's current last-write time the write no-ops (so delayed / synchronized writes can

package/storage/backblaze.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, SetConfig } from "./IArchives";
+import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, SetConfig } from "./IArchives";
 export declare class ArchivesBackblaze implements IArchives {
     private config;
     constructor(config: {
@@ -20,19 +20,8 @@ export declare class ArchivesBackblaze implements IArchives {
     private currentReset;
     private last503Reset;
     private apiRetryLogic;
-    get(fileName: string, config?: {
-        range?: {
-            start: number;
-            end: number;
-        };
-        retryCount?: number;
-    }): Promise<Buffer | undefined>;
-    get2(fileName: string, config?: {
-        range?: {
-            start: number;
-            end: number;
-        };
-    }): Promise<{
+    get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
+    get2(fileName: string, config?: GetConfig): Promise<{
         data: Buffer;
         writeTime: number;
         size: number;

package/storage/backblaze.ts

@@ -9,7 +9,7 @@ import debugbreak from "debugbreak";
 import dns from "dns";
 import { getSecret } from "../misc/getSecret";
 import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
-import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, SetConfig, assertValidLastModified, IMMUTABLE_CACHE_TIME } from "./IArchives";
+import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, SetConfig, assertValidLastModified, IMMUTABLE_CACHE_TIME } from "./IArchives";
 import { filterChanges } from "./remoteStorage/remoteConfig";
 
 type BackblazeCreds = {
@@ -622,11 +622,11 @@ export class ArchivesBackblaze implements IArchives {
         }
     }
 
-    public async get(fileName: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
+    public async get(fileName: string, config?: GetConfig): Promise<Buffer | undefined> {
         let result = await this.get2(fileName, config);
         return result && result.data || undefined;
     }
-    public async get2(fileName: string, config?: { range?: { start: number; end: number; } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
+    public async get2(fileName: string, config?: GetConfig): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         let downloading = true;
         try {
             let time = Date.now();