sliftutils 1.7.99 → 1.7.100

package/index.d.ts

@@ -910,7 +910,8 @@ declare module "sliftutils/storage/ArchivesDisk" {
         private collectFiles;
         setLargeFile(config: SetLargeFileConfig): Promise<void>;
         startLargeUpload(): Promise<string>;
-        appendLargeUpload(id: string, data: Buffer): Promise<void>;
+        /** offset makes the write POSITIONAL instead of appending: a retried part lands exactly where the failed attempt would have, so part retries are idempotent. One upload must use either appends or offsets throughout, never both - the cached handle keeps its first flags, and O_APPEND ignores the position argument. */
+        appendLargeUpload(id: string, data: Buffer, offset?: number): Promise<void>;
         finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void>;
         cancelLargeUpload(id: string): Promise<void>;
         getURL(path: string): Promise<string>;
@@ -2169,6 +2170,8 @@ declare module "sliftutils/storage/IArchives" {
         internal?: boolean;
         /** Also return size-0 results (tombstones - an empty file IS a missing file) instead of treating them as absent. Off by default, matching getInfo's flag of the same name. Synchronization passes this so a DELETED file (with its write time) is distinguishable from a file that never existed. */
         includeTombstones?: boolean;
+        /** How many extra times the WHOLE operation is retried after every source in a pass failed - any error counts (the wrong-window/route markers still get their config re-resolve first). Only applies to fallback dispatch (multi-source, not noFallbacks), where it defaults to 3; the noFallbacks/write-node path already retries on its own deadline. Multi-part uploads additionally retry per part regardless of this. */
+        retries?: number;
     };
     export type FindConfig = {
         shallow?: boolean;
@@ -2185,12 +2188,16 @@ declare module "sliftutils/storage/IArchives" {
         noChecks?: boolean;
         /** See SetConfig.fallbacks. */
         fallbacks?: boolean;
+        /** See GetConfig.retries. */
+        retries?: number;
     };
     export type GetInfoConfig = {
         /** Also report size-0 entries (tombstones - an empty file IS a missing file). Off by default, so a deleted key reports undefined, matching get. Synchronization-style callers pass this when they need a deletion's write time (e.g. to compare it against a write they are about to make). */
         includeTombstones?: boolean;
         /** See GetConfig.noFallbacks: answer ONLY from the primary source (the one writes would target) instead of falling back across the redundant sources. */
         noFallbacks?: boolean;
+        /** See GetConfig.retries. */
+        retries?: number;
     };
     export type ChangesAfterConfig = {
         time: number;
@@ -2207,6 +2214,8 @@ declare module "sliftutils/storage/IArchives" {
         internal?: boolean;
         /** Writes normally go ONLY to the write node (the first current-window source covering the key), retrying it even while it is down - consistent, but unavailable when that node is. With fallbacks, the write node is still tried first, but on failure the write lands on the next current-window source covering the key (synchronization moves it to the write node later) - availability at the cost of reads possibly missing the write until it propagates. Single-source archives ignore the flag. */
         fallbacks?: boolean;
+        /** See GetConfig.retries. */
+        retries?: number;
     };
     /** setLargeFile's config: a SetConfig (it IS a set - the same immutability, ordering, internal, and fallbacks rules apply) plus the stream carrying the bytes. */
     export type SetLargeFileConfig = SetConfig & {
@@ -2456,7 +2465,7 @@ declare module "sliftutils/storage/LogMap" {
         time: number;
         changedAt: number;
     };
-    export declare class LogMap<T> {
+    export declare class TransactionFile<T> {
         private filePath;
         constructor(filePath: string);
         private values;
@@ -2622,6 +2631,66 @@ declare module "sliftutils/storage/StorageObservableAsync" {
 
 }
 
+declare module "sliftutils/storage/TransactionFile" {
+    /** A live value: what was stored, when it was written (the caller's ordering), and when we last changed it (ours). */
+    export type LogEntry<T> = {
+        value: T;
+        time: number;
+        changedAt: number;
+    };
+    /** A deleted key: when it was deleted, and when we learned. The value is gone; the time is the point of a tombstone. */
+    export type LogTombstone = {
+        time: number;
+        changedAt: number;
+    };
+    export declare class TransactionFile<T> {
+        private filePath;
+        constructor(filePath: string);
+        private values;
+        private deleted;
+        private logRecords;
+        private pending;
+        private flushTimer;
+        private writeChain;
+        /** Reads the log and replays it into memory. Every other method assumes this has finished. */
+        load: {
+            (): Promise<void>;
+            reset(): void;
+            set(newValue: Promise<void>): void;
+        };
+        /** The live value, or undefined when the key does not exist here (deleted included - a deletion is an absence, see getDeleted for its time). */
+        get(key: string): LogEntry<T> | undefined;
+        /** When the key was deleted, if it was. Absent both here and in get means we have never heard of it. */
+        getDeleted(key: string): LogTombstone | undefined;
+        /** The time the key last changed either way, or 0 if we have never heard of it - what a new write has to beat. */
+        timeOf(key: string): number;
+        /** O(1), and counts only what exists. */
+        get size(): number;
+        get deletedSize(): number;
+        /** Live values only. Live, in insertion order - deleting during iteration is safe (JS skips entries removed before they are reached), which is what the passes that walk everything and prune as they go rely on. */
+        entries(): IterableIterator<[string, LogEntry<T>]>;
+        /** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
+        deletedEntries(): IterableIterator<[string, LogTombstone]>;
+        /** Stores a value as of `time`. Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
+        set(key: string, value: T, time: number): boolean;
+        /** Deletes as of `time`, keeping the tombstone. Returns false when something at least as new is already here. */
+        delete(key: string, time: number): boolean;
+        /** Forgets the key entirely, tombstone included - for a tombstone old enough that nobody needs to hear about the deletion any more, and for an entry that turned out never to have existed. Not a deletion: it leaves nothing behind to propagate. */
+        purge(key: string): void;
+        private applySet;
+        private applyDelete;
+        private append;
+        private scheduleFlush;
+        /** Writes everything pending (rewriting the log first if it has grown too far past what it describes). */
+        flush(): Promise<void>;
+        private directory;
+        private write;
+        private appendPending;
+        private compact;
+    }
+
+}
+
 declare module "sliftutils/storage/TransactionStorage" {
     /// <reference types="node" />
     /// <reference types="node" />
@@ -3241,7 +3310,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
     import { StoreSync } from "./storeSync";
     import { StoreConfig } from "./storeConfig";
     export declare const WINDOW_END_FLUSH_MARGIN: number;
-    /** What we store about a file. Its times are not in here: the index keeps those for every key, deleted ones included (see LogMap). */
+    /** What we store about a file. Its times are not in here: the index keeps those for every key, deleted ones included (see TransactionFile). */
     type IndexValue = {
         size: number;
         sourcesListIndex: number;
@@ -3425,6 +3494,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
         appendLargeUpload(config: {
             id: string;
             data: Buffer;
+            offset?: number;
         }): Promise<void>;
         finishLargeUpload(config: {
             id: string;
@@ -4294,6 +4364,7 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
         uploadPart: (config: {
             uploadId: string;
             data: Buffer;
+            offset?: number;
         }) => Promise<void>;
         finishLargeFile: (config: {
             uploadId: string;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.99",
+    "version": "1.7.100",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/ArchivesDisk.d.ts

@@ -37,7 +37,8 @@ export declare class ArchivesDisk implements IArchives {
     private collectFiles;
     setLargeFile(config: SetLargeFileConfig): Promise<void>;
     startLargeUpload(): Promise<string>;
-    appendLargeUpload(id: string, data: Buffer): Promise<void>;
+    /** offset makes the write POSITIONAL instead of appending: a retried part lands exactly where the failed attempt would have, so part retries are idempotent. One upload must use either appends or offsets throughout, never both - the cached handle keeps its first flags, and O_APPEND ignores the position argument. */
+    appendLargeUpload(id: string, data: Buffer, offset?: number): Promise<void>;
     finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void>;
     cancelLargeUpload(id: string): Promise<void>;
     getURL(path: string): Promise<string>;

package/storage/ArchivesDisk.ts

@@ -322,13 +322,19 @@ export class ArchivesDisk implements IArchives {
         this.largeUploads.set(id, { tmpPath: path.join(this.uploadsDir, `upload_${id}.tmp`) });
         return id;
     }
-    public async appendLargeUpload(id: string, data: Buffer): Promise<void> {
+    /** offset makes the write POSITIONAL instead of appending: a retried part lands exactly where the failed attempt would have, so part retries are idempotent. One upload must use either appends or offsets throughout, never both - the cached handle keeps its first flags, and O_APPEND ignores the position argument. */
+    public async appendLargeUpload(id: string, data: Buffer, offset?: number): Promise<void> {
         let upload = this.largeUploads.get(id);
         if (!upload) throw new Error(`Unknown large upload ${id}`);
         const tmpPath = upload.tmpPath;
         await this.handles.run(tmpPath, async () => {
-            let handle = await this.handles.getHandle(tmpPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_APPEND);
-            await handle.write(data, 0, data.length);
+            if (offset === undefined) {
+                let handle = await this.handles.getHandle(tmpPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_APPEND);
+                await handle.write(data, 0, data.length);
+                return;
+            }
+            let handle = await this.handles.getHandle(tmpPath, fs.constants.O_WRONLY | fs.constants.O_CREAT);
+            await handle.write(data, 0, data.length, offset);
         });
     }
     public async finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void> {

package/storage/IArchives.d.ts

@@ -75,6 +75,8 @@ export type GetConfig = {
     internal?: boolean;
     /** Also return size-0 results (tombstones - an empty file IS a missing file) instead of treating them as absent. Off by default, matching getInfo's flag of the same name. Synchronization passes this so a DELETED file (with its write time) is distinguishable from a file that never existed. */
     includeTombstones?: boolean;
+    /** How many extra times the WHOLE operation is retried after every source in a pass failed - any error counts (the wrong-window/route markers still get their config re-resolve first). Only applies to fallback dispatch (multi-source, not noFallbacks), where it defaults to 3; the noFallbacks/write-node path already retries on its own deadline. Multi-part uploads additionally retry per part regardless of this. */
+    retries?: number;
 };
 export type FindConfig = {
     shallow?: boolean;
@@ -91,12 +93,16 @@ export type DelConfig = {
     noChecks?: boolean;
     /** See SetConfig.fallbacks. */
     fallbacks?: boolean;
+    /** See GetConfig.retries. */
+    retries?: number;
 };
 export type GetInfoConfig = {
     /** Also report size-0 entries (tombstones - an empty file IS a missing file). Off by default, so a deleted key reports undefined, matching get. Synchronization-style callers pass this when they need a deletion's write time (e.g. to compare it against a write they are about to make). */
     includeTombstones?: boolean;
     /** See GetConfig.noFallbacks: answer ONLY from the primary source (the one writes would target) instead of falling back across the redundant sources. */
     noFallbacks?: boolean;
+    /** See GetConfig.retries. */
+    retries?: number;
 };
 export type ChangesAfterConfig = {
     time: number;
@@ -113,6 +119,8 @@ export type SetConfig = {
     internal?: boolean;
     /** Writes normally go ONLY to the write node (the first current-window source covering the key), retrying it even while it is down - consistent, but unavailable when that node is. With fallbacks, the write node is still tried first, but on failure the write lands on the next current-window source covering the key (synchronization moves it to the write node later) - availability at the cost of reads possibly missing the write until it propagates. Single-source archives ignore the flag. */
     fallbacks?: boolean;
+    /** See GetConfig.retries. */
+    retries?: number;
 };
 /** setLargeFile's config: a SetConfig (it IS a set - the same immutability, ordering, internal, and fallbacks rules apply) plus the stream carrying the bytes. */
 export type SetLargeFileConfig = SetConfig & {

package/storage/IArchives.ts

@@ -110,6 +110,8 @@ export type GetConfig = {
     internal?: boolean;
     /** Also return size-0 results (tombstones - an empty file IS a missing file) instead of treating them as absent. Off by default, matching getInfo's flag of the same name. Synchronization passes this so a DELETED file (with its write time) is distinguishable from a file that never existed. */
     includeTombstones?: boolean;
+    /** How many extra times the WHOLE operation is retried after every source in a pass failed - any error counts (the wrong-window/route markers still get their config re-resolve first). Only applies to fallback dispatch (multi-source, not noFallbacks), where it defaults to 3; the noFallbacks/write-node path already retries on its own deadline. Multi-part uploads additionally retry per part regardless of this. */
+    retries?: number;
 };
 
 export type FindConfig = {
@@ -128,6 +130,8 @@ export type DelConfig = {
     noChecks?: boolean;
     /** See SetConfig.fallbacks. */
     fallbacks?: boolean;
+    /** See GetConfig.retries. */
+    retries?: number;
 };
 
 export type GetInfoConfig = {
@@ -135,6 +139,8 @@ export type GetInfoConfig = {
     includeTombstones?: boolean;
     /** See GetConfig.noFallbacks: answer ONLY from the primary source (the one writes would target) instead of falling back across the redundant sources. */
     noFallbacks?: boolean;
+    /** See GetConfig.retries. */
+    retries?: number;
 };
 
 export type ChangesAfterConfig = {
@@ -153,6 +159,8 @@ export type SetConfig = {
     internal?: boolean;
     /** Writes normally go ONLY to the write node (the first current-window source covering the key), retrying it even while it is down - consistent, but unavailable when that node is. With fallbacks, the write node is still tried first, but on failure the write lands on the next current-window source covering the key (synchronization moves it to the write node later) - availability at the cost of reads possibly missing the write until it propagates. Single-source archives ignore the flag. */
     fallbacks?: boolean;
+    /** See GetConfig.retries. */
+    retries?: number;
 };
 
 /** setLargeFile's config: a SetConfig (it IS a set - the same immutability, ordering, internal, and fallbacks rules apply) plus the stream carrying the bytes. */

package/storage/LogMap.d.ts

@@ -9,7 +9,7 @@ export type LogTombstone = {
     time: number;
     changedAt: number;
 };
-export declare class LogMap<T> {
+export declare class TransactionFile<T> {
     private filePath;
     constructor(filePath: string);
     private values;

package/storage/TransactionFile.d.ts

@@ -0,0 +1,56 @@
+/** A live value: what was stored, when it was written (the caller's ordering), and when we last changed it (ours). */
+export type LogEntry<T> = {
+    value: T;
+    time: number;
+    changedAt: number;
+};
+/** A deleted key: when it was deleted, and when we learned. The value is gone; the time is the point of a tombstone. */
+export type LogTombstone = {
+    time: number;
+    changedAt: number;
+};
+export declare class TransactionFile<T> {
+    private filePath;
+    constructor(filePath: string);
+    private values;
+    private deleted;
+    private logRecords;
+    private pending;
+    private flushTimer;
+    private writeChain;
+    /** Reads the log and replays it into memory. Every other method assumes this has finished. */
+    load: {
+        (): Promise<void>;
+        reset(): void;
+        set(newValue: Promise<void>): void;
+    };
+    /** The live value, or undefined when the key does not exist here (deleted included - a deletion is an absence, see getDeleted for its time). */
+    get(key: string): LogEntry<T> | undefined;
+    /** When the key was deleted, if it was. Absent both here and in get means we have never heard of it. */
+    getDeleted(key: string): LogTombstone | undefined;
+    /** The time the key last changed either way, or 0 if we have never heard of it - what a new write has to beat. */
+    timeOf(key: string): number;
+    /** O(1), and counts only what exists. */
+    get size(): number;
+    get deletedSize(): number;
+    /** Live values only. Live, in insertion order - deleting during iteration is safe (JS skips entries removed before they are reached), which is what the passes that walk everything and prune as they go rely on. */
+    entries(): IterableIterator<[string, LogEntry<T>]>;
+    /** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
+    deletedEntries(): IterableIterator<[string, LogTombstone]>;
+    /** Stores a value as of `time`. Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
+    set(key: string, value: T, time: number): boolean;
+    /** Deletes as of `time`, keeping the tombstone. Returns false when something at least as new is already here. */
+    delete(key: string, time: number): boolean;
+    /** Forgets the key entirely, tombstone included - for a tombstone old enough that nobody needs to hear about the deletion any more, and for an entry that turned out never to have existed. Not a deletion: it leaves nothing behind to propagate. */
+    purge(key: string): void;
+    private applySet;
+    private applyDelete;
+    private append;
+    private scheduleFlush;
+    /** Writes everything pending (rewriting the log first if it has grown too far past what it describes). */
+    flush(): Promise<void>;
+    private directory;
+    private write;
+    private appendPending;
+    private compact;
+}

package/storage/{LogMap.ts → TransactionFile.ts}

@@ -18,18 +18,21 @@ export type LogEntry<T> = { value: T; time: number; changedAt: number };
 /** A deleted key: when it was deleted, and when we learned. The value is gone; the time is the point of a tombstone. */
 export type LogTombstone = { time: number; changedAt: number };
 
-// One line of the log. JSON.stringify escapes newlines inside keys and values, so a record can never contain the separator.
+// One line of the log, with single-letter field names because there is one record per set/delete/purge ever made and the field names are most of a record's overhead. JSON.stringify escapes newlines inside keys and values, so a record can never contain the separator.
 type LogRecord<T> = {
+    // key
     k: string;
-    // The caller's time for this change - what ordering is decided by
+    // time, lastModified time of the file, or the delete time, or the purge time
     t: number;
-    // A set carries the value; a delete carries d; a purge carries p (forget the key entirely, tombstone included)
+    // value
     v?: T;
+    // deleted, the file on disk is deleted, preserved when we compact
     d?: 1;
+    // purged, the file on disk is deleted, not preserved when we compact
     p?: 1;
 };
 
-export class LogMap<T> {
+export class TransactionFile<T> {
     constructor(private filePath: string) { }
 
     private values = new Map<string, LogEntry<T>>();

package/storage/backblaze.ts

@@ -629,7 +629,10 @@ export class ArchivesBackblaze implements IArchives {
                 }
             }
 
-            throw err;
+            // EVERYTHING else retries too (bounded by `retries`): backblaze fails a small fraction of requests with ever-new error shapes, and aborting a multi-part upload (or a full sync) on one unmatched error costs the whole transfer. A genuinely permanent error just pays the few retry delays before surfacing.
+            console.warn(`[${context}] Unrecognized error, retrying in 5s (${retries} left): ${err.message}`);
+            await delay(5000);
+            return this.apiRetryLogic(context, fnc, retries - 1);
         }
     }