sliftutils 1.7.88 → 1.7.89

package/storage/LogMap.ts

@@ -0,0 +1,220 @@
+import fs from "fs";
+import path from "path";
+import { lazy } from "socket-function/src/caching";
+
+// A persistent Map that knows about time and about deletions. Every value carries the time it was written, so a write that arrives out of order is simply not applied - the map cannot be made to go backwards. A deleted key keeps its time too, as a tombstone in a second map, so "was this deleted, and when" is as cheap to ask as "does this exist", and expiring old tombstones is a walk of the tombstones alone rather than of everything.
+//
+// On disk it is one append-only log of the sets, deletes and purges that produced both maps, rewritten from them whenever it has grown to several times their size.
+
+// The log is rewritten once it holds this many times more records than there are keys (a log of 3 records per surviving key is mostly history)
+const COMPACT_RATIO = 3;
+// ...but never for a small log, where the rewrite would cost more than the history it removes
+const MIN_COMPACT_RECORDS = 1000;
+// Appends are coalesced for this long. A crash loses at most this much - the callers of this class treat their data as reconstructible - and the alternative is a file write per set.
+const FLUSH_DELAY = 500;
+
+/** A live value: what was stored, when it was written (the caller's ordering), and when we last changed it (ours). */
+export type LogEntry<T> = { value: T; time: number; changedAt: number };
+/** A deleted key: when it was deleted, and when we learned. The value is gone; the time is the point of a tombstone. */
+export type LogTombstone = { time: number; changedAt: number };
+
+// One line of the log. JSON.stringify escapes newlines inside keys and values, so a record can never contain the separator.
+type LogRecord<T> = {
+    k: string;
+    // The caller's time for this change - what ordering is decided by
+    t: number;
+    // A set carries the value; a delete carries d; a purge carries p (forget the key entirely, tombstone included)
+    v?: T;
+    d?: 1;
+    p?: 1;
+};
+
+export class LogMap<T> {
+    constructor(private filePath: string) { }
+
+    private values = new Map<string, LogEntry<T>>();
+    private deleted = new Map<string, LogTombstone>();
+    // Records the log holds (written plus pending) - what compaction is decided against, NOT the number of keys
+    private logRecords = 0;
+    private pending: string[] = [];
+    private flushTimer: ReturnType<typeof setTimeout> | undefined;
+    // Serializes everything that touches the file, so an append can never interleave with a rewrite
+    private writeChain: Promise<void> = Promise.resolve();
+
+    /** Reads the log and replays it into memory. Every other method assumes this has finished. */
+    public load = lazy(async () => {
+        let data: string;
+        try {
+            data = await fs.promises.readFile(this.filePath, "utf8");
+        } catch (e) {
+            if ((e as { code?: string }).code === "ENOENT") return;
+            throw e;
+        }
+        let applied = 0;
+        let damaged = 0;
+        for (let line of data.split("\n")) {
+            if (!line) continue;
+            let record: LogRecord<T> | undefined;
+            try {
+                record = JSON.parse(line) as LogRecord<T>;
+            } catch {
+                // A crash mid-append leaves a torn final line; anything earlier means real damage. Either way the rest of the log is still worth having - this is an index, and scanning rebuilds whatever we lose.
+                damaged++;
+                continue;
+            }
+            applied++;
+            // Replayed through the same ordering rules that produced it, so a log written out of order still lands on the same state
+            if (record.p) {
+                this.values.delete(record.k);
+                this.deleted.delete(record.k);
+            } else if (record.d) {
+                this.applyDelete(record.k, record.t, record.t);
+            } else if (record.v !== undefined) {
+                this.applySet(record.k, record.v, record.t, record.t);
+            }
+        }
+        this.logRecords = applied;
+        if (damaged) {
+            console.error(`${damaged} damaged record(s) in ${this.filePath} were skipped; ${applied} applied, leaving ${this.values.size} values and ${this.deleted.size} tombstones`);
+        }
+    });
+
+    /** The live value, or undefined when the key does not exist here (deleted included - a deletion is an absence, see getDeleted for its time). */
+    public get(key: string): LogEntry<T> | undefined {
+        return this.values.get(key);
+    }
+    /** When the key was deleted, if it was. Absent both here and in get means we have never heard of it. */
+    public getDeleted(key: string): LogTombstone | undefined {
+        return this.deleted.get(key);
+    }
+    /** The time the key last changed either way, or 0 if we have never heard of it - what a new write has to beat. */
+    public timeOf(key: string): number {
+        let live = this.values.get(key);
+        if (live) return live.time;
+        return this.deleted.get(key)?.time || 0;
+    }
+
+    /** O(1), and counts only what exists. */
+    public get size(): number {
+        return this.values.size;
+    }
+    public get deletedSize(): number {
+        return this.deleted.size;
+    }
+    /** Live values only. Live, in insertion order - deleting during iteration is safe (JS skips entries removed before they are reached), which is what the passes that walk everything and prune as they go rely on. */
+    public entries(): IterableIterator<[string, LogEntry<T>]> {
+        return this.values.entries();
+    }
+    /** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
+    public deletedEntries(): IterableIterator<[string, LogTombstone]> {
+        return this.deleted.entries();
+    }
+
+    /** Stores a value as of `time`. Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
+    public set(key: string, value: T, time: number): boolean {
+        if (!this.applySet(key, value, time, Date.now())) return false;
+        this.append({ k: key, t: time, v: value });
+        return true;
+    }
+
+    /** Deletes as of `time`, keeping the tombstone. Returns false when something at least as new is already here. */
+    public delete(key: string, time: number): boolean {
+        if (!this.applyDelete(key, time, Date.now())) return false;
+        this.append({ k: key, t: time, d: 1 });
+        return true;
+    }
+
+    /** Forgets the key entirely, tombstone included - for a tombstone old enough that nobody needs to hear about the deletion any more, and for an entry that turned out never to have existed. Not a deletion: it leaves nothing behind to propagate. */
+    public purge(key: string): void {
+        let had = this.values.delete(key);
+        had = this.deleted.delete(key) || had;
+        if (!had) return;
+        this.append({ k: key, t: Date.now(), p: 1 });
+    }
+
+    private applySet(key: string, value: T, time: number, changedAt: number): boolean {
+        if (time < this.timeOf(key)) return false;
+        this.deleted.delete(key);
+        this.values.set(key, { value, time, changedAt });
+        return true;
+    }
+
+    private applyDelete(key: string, time: number, changedAt: number): boolean {
+        if (time < this.timeOf(key)) return false;
+        this.values.delete(key);
+        this.deleted.set(key, { time, changedAt });
+        return true;
+    }
+
+    private append(record: LogRecord<T>): void {
+        this.pending.push(JSON.stringify(record));
+        this.logRecords++;
+        this.scheduleFlush();
+    }
+
+    private scheduleFlush(): void {
+        if (this.flushTimer !== undefined) return;
+        this.flushTimer = setTimeout(() => {
+            this.flushTimer = undefined;
+            void this.flush().catch((e: Error) => console.error(`Flushing ${this.filePath} failed: ${e.stack ?? e}`));
+        }, FLUSH_DELAY);
+        (this.flushTimer as { unref?: () => void }).unref?.();
+    }
+
+    /** Writes everything pending (rewriting the log first if it has grown too far past what it describes). */
+    public async flush(): Promise<void> {
+        let result = this.writeChain.then(() => this.write());
+        // The chain only exists to serialize writes, so it must not stay rejected: a failure belongs to the caller that asked for this flush, not to every flush after it
+        this.writeChain = result.catch(() => { });
+        await result;
+    }
+
+    private directory = lazy(async () => {
+        await fs.promises.mkdir(path.dirname(this.filePath), { recursive: true });
+    });
+
+    private async write(): Promise<void> {
+        if (this.logRecords > Math.max(MIN_COMPACT_RECORDS, (this.values.size + this.deleted.size) * COMPACT_RATIO)) {
+            await this.compact();
+            return;
+        }
+        await this.appendPending();
+    }
+
+    private async appendPending(): Promise<void> {
+        if (!this.pending.length) return;
+        let records = this.pending;
+        this.pending = [];
+        await this.directory();
+        try {
+            await fs.promises.appendFile(this.filePath, records.join("\n") + "\n");
+        } catch (e) {
+            // Put them back rather than lose them: the values are still in memory, so all that is at risk is having to rebuild them by scanning
+            this.pending = records.concat(this.pending);
+            this.scheduleFlush();
+            throw e;
+        }
+    }
+
+    // Rewrites the log as the current contents of both maps. They are the truth - the log only exists to rebuild them - so the snapshot IS the new log.
+    private async compact(): Promise<void> {
+        let records: string[] = [];
+        for (let [key, entry] of this.values) {
+            records.push(JSON.stringify({ k: key, t: entry.time, v: entry.value } satisfies LogRecord<T>));
+        }
+        for (let [key, tombstone] of this.deleted) {
+            records.push(JSON.stringify({ k: key, t: tombstone.time, d: 1 } satisfies LogRecord<T>));
+        }
+        // Cleared with the snapshot: every pending record is already reflected in the maps, so the snapshot contains it. Anything appended from here on is NOT in the snapshot, and lands in the new log after the rename.
+        this.pending = [];
+        let startedWith = this.logRecords;
+        let tempPath = `${this.filePath}.compacting`;
+        await this.directory();
+        await fs.promises.writeFile(tempPath, records.length && `${records.join("\n")}\n` || "");
+        // Replaces the old log in one step, so a crash leaves either the whole old log or the whole new one
+        await fs.promises.rename(tempPath, this.filePath);
+        this.logRecords = records.length + this.pending.length;
+        console.log(`Rewrote ${this.filePath}: ${startedWith} log records for ${this.values.size} values and ${this.deleted.size} tombstones`);
+        await this.appendPending();
+    }
+}

package/storage/archiveHelpers.d.ts

@@ -0,0 +1,36 @@
+import type { IArchives } from "./IArchives";
+/** Copies one file between two archives. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, or undefined when the source doesn't have the file. */
+export declare function copyArchiveFile(config: {
+    from: IArchives;
+    to: IArchives;
+    path: string;
+    /** The path at the destination - defaults to path (the common case: the same key moving between two archives). */
+    toPath?: string;
+    size?: number;
+    /** Stamps the destination write with this time INSTEAD of now - which makes the copy lose (silently) to anything newer at the destination, tombstones included. That is exactly right for synchronization, which copies the same key between replicas and must preserve its ordering, and exactly wrong for anything else - so omit it unless the destination is another copy of the same key. */
+    writeTime?: number;
+    forceSetImmutable?: boolean;
+    noChecks?: boolean;
+    internal?: boolean;
+    noFallbacks?: boolean;
+}): Promise<{
+    writeTime: number;
+    size: number;
+} | undefined>;
+/**
+ * Moves one file - between two archives, or between two paths of one. When from and to are the SAME
+ * archives instance and it implements move, the backend moves the file itself (backblaze copies
+ * server-side, disk renames, the storage server relocates it node-side) and the bytes never travel
+ * through us. Everything else is the safe fallback: copy, then CONFIRM the destination actually
+ * reports the file (getInfo), and only then delete the source - the one order in which no failure
+ * can lose the file, only at worst leave it in both places. Throws when the source doesn't have the
+ * file, or when the copy cannot be confirmed (the source is then left untouched).
+ */
+export declare function moveArchiveFile(config: {
+    from: IArchives;
+    to: IArchives;
+    path: string;
+    /** The path at the destination - defaults to path (moving between two archives). Required in practice when from and to are the same archives, where the same path would be a no-op. */
+    toPath?: string;
+    noFallbacks?: boolean;
+}): Promise<void>;

package/storage/archiveHelpers.ts

@@ -0,0 +1,103 @@
+// Type-only: IArchives.ts re-exports our functions, so a value import back at it would be a require cycle
+import type { IArchives } from "./IArchives";
+
+// Cross-archive file operations built ON TOP of IArchives (copy, move) - helpers over the interface, not part of it, so they live beside it rather than in it.
+
+const LARGE_COPY_THRESHOLD = 64 * 1024 * 1024;
+const LARGE_COPY_CHUNK = 32 * 1024 * 1024;
+
+/** Copies one file between two archives. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, or undefined when the source doesn't have the file. */
+export async function copyArchiveFile(config: {
+    from: IArchives;
+    to: IArchives;
+    path: string;
+    /** The path at the destination - defaults to path (the common case: the same key moving between two archives). */
+    toPath?: string;
+    // From the caller's metadata scan; getInfo fills it in when omitted
+    size?: number;
+    /** Stamps the destination write with this time INSTEAD of now - which makes the copy lose (silently) to anything newer at the destination, tombstones included. That is exactly right for synchronization, which copies the same key between replicas and must preserve its ordering, and exactly wrong for anything else - so omit it unless the destination is another copy of the same key. */
+    writeTime?: number;
+    forceSetImmutable?: boolean;
+    noChecks?: boolean;
+    internal?: boolean;
+    noFallbacks?: boolean;
+}): Promise<{ writeTime: number; size: number } | undefined> {
+    let { from, to, path } = config;
+    let toPath = config.toPath || path;
+    let size = config.size;
+    if (size === undefined) {
+        let info = await from.getInfo(path, { noFallbacks: config.noFallbacks });
+        if (!info) return undefined;
+        size = info.size;
+    }
+    // Preserving the source's stamp is ONLY for synchronization (copies of the same key between replicas, where the original ordering must survive propagation) - those callers pass writeTime explicitly. A plain copy is a NEW write and is stamped now: stamping it with the source's old time makes it LOSE to any newer write or tombstone at the destination, silently - move a file back to a folder it was deleted from and the copy is dropped, then the caller deletes the source, and the file is gone entirely.
+    let writeTime = config.writeTime || Date.now();
+    if (size <= LARGE_COPY_THRESHOLD) {
+        let result = await from.get2(path, { internal: config.internal, noFallbacks: config.noFallbacks });
+        // Empty counts as absent, never as content to copy: an empty file IS a deletion, set refuses empty buffers, and deletions travel through their own path (del / scan tombstones)
+        if (!result || !result.data || !result.data.length) return undefined;
+        await to.set(toPath, result.data, { lastModified: writeTime, forceSetImmutable: config.forceSetImmutable, noChecks: config.noChecks, internal: config.internal });
+        return { writeTime, size: result.data.length };
+    }
+    // Const so the closure keeps the narrowed type
+    const totalSize = size;
+    let offset = 0;
+    await to.setLargeFile({
+        path: toPath,
+        lastModified: writeTime,
+        forceSetImmutable: config.forceSetImmutable,
+        noChecks: config.noChecks,
+        internal: config.internal,
+        restartStream: () => {
+            offset = 0;
+        },
+        getNextData: async () => {
+            if (offset >= totalSize) return undefined;
+            let end = Math.min(offset + LARGE_COPY_CHUNK, totalSize);
+            let data = await from.get(path, { range: { start: offset, end }, internal: config.internal, noFallbacks: config.noFallbacks });
+            if (!data || !data.length) {
+                throw new Error(`Ranged read of ${JSON.stringify(path)} from ${from.getDebugName()} returned ${data && data.length || "nothing"} at ${offset}-${end} (expected ${end - offset} bytes of a ${totalSize} byte file - it changed or vanished mid-copy)`);
+            }
+            offset += data.length;
+            return data;
+        },
+    });
+    return { writeTime, size: totalSize };
+}
+
+/**
+ * Moves one file - between two archives, or between two paths of one. When from and to are the SAME
+ * archives instance and it implements move, the backend moves the file itself (backblaze copies
+ * server-side, disk renames, the storage server relocates it node-side) and the bytes never travel
+ * through us. Everything else is the safe fallback: copy, then CONFIRM the destination actually
+ * reports the file (getInfo), and only then delete the source - the one order in which no failure
+ * can lose the file, only at worst leave it in both places. Throws when the source doesn't have the
+ * file, or when the copy cannot be confirmed (the source is then left untouched).
+ */
+export async function moveArchiveFile(config: {
+    from: IArchives;
+    to: IArchives;
+    path: string;
+    /** The path at the destination - defaults to path (moving between two archives). Required in practice when from and to are the same archives, where the same path would be a no-op. */
+    toPath?: string;
+    noFallbacks?: boolean;
+}): Promise<void> {
+    let { from, to, path } = config;
+    let toPath = config.toPath || path;
+    if (from === to) {
+        if (path === toPath) return;
+        if (from.move) {
+            await from.move({ fromPath: path, toPath });
+            return;
+        }
+    }
+    let copied = await copyArchiveFile({ from, to, path, toPath, noFallbacks: config.noFallbacks });
+    if (!copied) {
+        throw new Error(`Cannot move ${JSON.stringify(path)}: ${from.getDebugName()} does not have it`);
+    }
+    let confirmed = await to.getInfo(toPath, { noFallbacks: config.noFallbacks });
+    if (!confirmed) {
+        throw new Error(`Not deleting ${JSON.stringify(path)} from ${from.getDebugName()} after copying it: ${to.getDebugName()} does not report ${JSON.stringify(toPath)} (the copy claimed to succeed, so something is wrong - the file is left at the source)`);
+    }
+    await from.del(path);
+}

package/storage/backblaze.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, DelConfig, FindConfig, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
+import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig, SourceConfig } from "./IArchives";
 export declare class ArchivesBackblaze implements IArchives {
     private config;
     constructor(config: {
@@ -16,6 +16,8 @@ export declare class ArchivesBackblaze implements IArchives {
     enableLogging(): void;
     private log;
     getDebugName(): string;
+    /** Policy flags changed in the routing config. The bucket-level settings getBucketAPI applied (CORS, cache time) are NOT re-applied - those are one-time bucket setup, and re-running them on every config edit would rewrite the bucket's settings from whichever process noticed first. */
+    updateSourceConfig(sourceConfig: SourceConfig): void;
     private getBucketAPI;
     private currentReset;
     private last503Reset;
@@ -28,13 +30,11 @@ export declare class ArchivesBackblaze implements IArchives {
     } | undefined>;
     getConfig(): Promise<ArchivesConfig>;
     hasWriteAccess(): Promise<boolean>;
+    /** Whether an existing file already beats this write, so it must not be sent at all. Shared by set and setLargeFile - the size a value happens to have must never change which ordering rules apply to it. */
+    private writeIsSuperseded;
     set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(fileName: string, config?: DelConfig): Promise<void>;
-    setLargeFile(config: {
-        path: string;
-        lastModified?: number;
-        getNextData(): Promise<Buffer | undefined>;
-    }): Promise<void>;
+    setLargeFile(config: SetLargeFileConfig): Promise<void>;
     getInfo(fileName: string, config?: GetInfoConfig): Promise<{
         writeTime: number;
         size: number;
@@ -47,12 +47,7 @@ export declare class ArchivesBackblaze implements IArchives {
         size: number;
     }[]>;
     assertPathValid(path: string): Promise<void>;
-    move(config: {
-        path: string;
-        target: IArchives;
-        targetPath: string;
-        copyInstead?: boolean;
-    }): Promise<void>;
+    move(config: MoveFileConfig): Promise<void>;
     copy(config: {
         path: string;
         target: IArchives;

package/storage/backblaze.ts

@@ -9,7 +9,7 @@ import debugbreak from "debugbreak";
 import dns from "dns";
 import { getSecret } from "../misc/getSecret";
 import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
-import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, DelConfig, FindConfig, GetConfig, GetInfoConfig, SetConfig, assertValidLastModified, bufferChunkStream, IMMUTABLE_CACHE_TIME } from "./IArchives";
+import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig, SourceConfig, assertValidLastModified, bufferChunkStream, IMMUTABLE_CACHE_TIME } from "./IArchives";
 import { filterChanges } from "./remoteStorage/remoteConfig";
 
 type BackblazeCreds = {
@@ -441,6 +441,14 @@ export class ArchivesBackblaze implements IArchives {
         return "backblaze " + this.config.bucketName;
     }
 
+    /** Policy flags changed in the routing config. The bucket-level settings getBucketAPI applied (CORS, cache time) are NOT re-applied - those are one-time bucket setup, and re-running them on every config edit would rewrite the bucket's settings from whichever process noticed first. */
+    public updateSourceConfig(sourceConfig: SourceConfig): void {
+        if (sourceConfig.type !== "backblaze") return;
+        this.config.public = sourceConfig.public;
+        this.config.immutable = sourceConfig.immutable;
+        this.config.allowedOrigins = sourceConfig.allowedOrigins;
+    }
+
     private getBucketAPI = lazy(async () => {
         let api = await getAPI();
 
@@ -710,6 +718,19 @@ export class ArchivesBackblaze implements IArchives {
             return false;
         }
     }
+    /** Whether an existing file already beats this write, so it must not be sent at all. Shared by set and setLargeFile - the size a value happens to have must never change which ordering rules apply to it. */
+    private async writeIsSuperseded(fileName: string, config?: SetConfig): Promise<boolean> {
+        if (!config?.lastModified) return false;
+        assertValidLastModified(config.lastModified);
+        // This comparison deliberately IGNORES noChecks: backblaze has no server of ours to enforce only-take-the-latest (unlike hosted targets, where the receiving store re-checks), so this comparison IS the ordering guard. Skipping it lets a stale push land over a newer value or tombstone - and because b2 stamps its own upload times, the stale data then becomes the newest-timestamped copy in the whole system, resurrecting globally through everyone's scans. includeTombstones: a deletion on b2 is a real size-0 file and must win this comparison too.
+        let existing = await this.getInfo(fileName, { includeTombstones: true });
+        if (!existing) return false;
+        // An older write never overwrites a newer one (see IArchives.set). B2 stamps its own upload time, so the exact lastModified is not preserved on the stored file.
+        if (config.lastModified < existing.writeTime) return true;
+        // Immutability wins: a synchronization push never overwrites an existing path on an immutable bucket (see SetConfig.forceSetImmutable)
+        if (config.forceSetImmutable && this.config.immutable) return true;
+        return false;
+    }
     public async set(fileName: string, data: Buffer, config?: SetConfig): Promise<string> {
         if (!data.length) {
             throw new Error(`set was called with an empty buffer for ${JSON.stringify(fileName)} on ${this.getDebugName()}: an empty file IS a deletion in this system and would read back as missing - call del instead`);
@@ -717,18 +738,10 @@ export class ArchivesBackblaze implements IArchives {
         if (config?.forceSetImmutable && !config.lastModified) {
             throw new Error(`forceSetImmutable requires lastModified (synchronization writes are ordered by their write time), writing ${fileName} to ${this.getDebugName()}`);
         }
-        if (config?.lastModified) {
-            assertValidLastModified(config.lastModified);
-            // This comparison deliberately IGNORES noChecks: backblaze has no server of ours to enforce only-take-the-latest (unlike hosted targets, where the receiving store re-checks), so this comparison IS the ordering guard. Skipping it lets a stale push land over a newer value or tombstone - and because b2 stamps its own upload times, the stale data then becomes the newest-timestamped copy in the whole system, resurrecting globally through everyone's scans. includeTombstones: a deletion on b2 is a real size-0 file and must win this comparison too.
-            let existing = await this.getInfo(fileName, { includeTombstones: true });
-            // An older write never overwrites a newer one (see IArchives.set). B2 stamps its own upload time, so the exact lastModified is not preserved on the stored file.
-            if (existing && config.lastModified < existing.writeTime) return fileName;
-            // Immutability wins: a synchronization push never overwrites an existing path on an immutable bucket (see SetConfig.forceSetImmutable)
-            if (existing && config.forceSetImmutable && this.config.immutable) return fileName;
-        }
+        if (await this.writeIsSuperseded(fileName, config)) return fileName;
         // Big buffers stream through the large-file API: a failed part retries alone instead of restarting the whole upload. The threshold MUST stay at two full chunks - below that setLargeFile falls back to plain set, which would recurse right back here.
         if (data.length >= LARGE_FILE_MIN_CHUNK_SIZE * 2) {
-            await this.setLargeFile({ path: fileName, lastModified: config?.lastModified, getNextData: bufferChunkStream(data) });
+            await this.setLargeFile({ path: fileName, ...config, ...bufferChunkStream(data) });
             return fileName;
         }
         this.log(`backblaze upload (${formatNumber(data.length)}B) ${fileName}`);
@@ -775,8 +788,10 @@ export class ArchivesBackblaze implements IArchives {
         // NOTE: Deletion SEEMS to work. This DOES break if we delete a file which keeps being recreated, ex, the heartbeat. let existsChecks = 10; while (existsChecks > 0) { let exists = await this.getInfo(fileName); if (!exists) break; await delay(1000); existsChecks--; } if (existsChecks === 0) { let exists = await this.getInfo(fileName); devDebugbreak(); console.warn(`File ${fileName} was deleted, but was still found afterwards`); exists = await this.getInfo(fileName); }
     }
 
-    // lastModified is accepted but cannot be honored - b2 stamps its own uploadTimestamp, which is what our getInfo/findInfo report as the write time
-    public async setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
+    // lastModified is accepted but cannot be honored - b2 stamps its own uploadTimestamp, which is what our getInfo/findInfo report as the write time. fallbacks means nothing here: a single bucket has nowhere to fall back to.
+    public async setLargeFile(config: SetLargeFileConfig): Promise<void> {
+        // Checked before a single byte moves: an upload that is already superseded must not be started at all (a cancelled large upload still costs the transfer)
+        if (await this.writeIsSuperseded(config.path, config)) return;
 
         let onError: (() => Promise<void>)[] = [];
         let time = Date.now();
@@ -805,18 +820,18 @@ export class ArchivesBackblaze implements IArchives {
             }
             // Backblaze disallows overly small files
             if (data.length < LARGE_FILE_MIN_CHUNK_SIZE) {
-                await this.set(fileName, data);
+                await this.set(fileName, data, config);
                 return;
             }
             // Backblaze disallows less than 2 chunks
             let secondData = await getNextData();
             if (!secondData?.length) {
-                await this.set(fileName, data);
+                await this.set(fileName, data, config);
                 return;
             }
             // ALSO, if there are two chunks, but one is too small, combine it. This helps allow us never send small chunks.
             if (secondData.length < LARGE_FILE_MIN_CHUNK_SIZE) {
-                await this.set(fileName, Buffer.concat([data, secondData]));
+                await this.set(fileName, Buffer.concat([data, secondData]), config);
                 return;
             }
             this.log(`Uploading large file ${config.path}`);
@@ -989,54 +1004,44 @@ export class ArchivesBackblaze implements IArchives {
         }
     }
 
-    public async move(config: {
+    public async move(config: MoveFileConfig): Promise<void> {
+        let { fromPath, toPath } = config;
+        if (fromPath === toPath) return;
+        // A b2 copy is a NEW file version, so its uploadTimestamp is stamped at copy time - the fresh write time move requires (see IArchives.move)
+        await this.copy({ path: fromPath, target: this, targetPath: toPath });
+        // The source is only deleted once the destination CONFIRMS the file - a failure here leaves it in both places, never in neither
+        let exists = await this.getInfo(toPath);
+        if (!exists) {
+            throw new Error(`Not deleting ${JSON.stringify(fromPath)} after the move on ${this.getDebugName()}: ${JSON.stringify(toPath)} was not found after the copy claimed to succeed (the file is left at the source)`);
+        }
+        await this.del(fromPath);
+    }
+
+    public async copy(config: {
         path: string;
         target: IArchives;
         targetPath: string;
-        copyInstead?: boolean;
-    }) {
+    }): Promise<void> {
         let { path, target, targetPath } = config;
-        // A self move should NOOP (and definitely not copy, and then delete itself!)
-        if (target === this && path === targetPath) {
-            this.log(`Backblaze move path to itself. Skipping move, as there is no work to do. ${path}`);
-            return;
-        }
+        if (target === this && path === targetPath) return;
         if (target instanceof ArchivesBackblaze) {
             let targetBucketId = target.bucketId;
-            if (targetBucketId === this.bucketId && path === targetPath) return;
-            await this.apiRetryLogic(`move ${path} -> ${targetPath}`, async (api) => {
+            await this.apiRetryLogic(`copy ${path} -> ${targetPath}`, async (api) => {
                 // Ugh... listing the file name sucks, but... I guess it's still better than downloading and re-uploading the entire file.
                 let info = await api.listFileNames({ bucketId: this.bucketId, prefix: path, maxFileCount: 10 });
                 let file = info.files.find(x => x.fileName === path);
-                if (!file) throw new Error(`File not found to move: ${path}`);
+                if (!file) throw new Error(`File not found to copy: ${path}`);
                 await api.copyFile({
                     sourceFileId: file.fileId,
                     fileName: targetPath,
                     destinationBucketId: targetBucketId,
                 });
             });
-        } else {
-            let data = await this.get(path);
-            if (!data) throw new Error(`File not found to move: ${path}`);
-            await target.set(targetPath, data);
-        }
-
-        if (!config.copyInstead) {
-            let exists = await this.getInfo(targetPath);
-            if (!exists) {
-                console.error(`File not found after move. Leaving BOTH files. ${targetPath} was not found. Being moved from ${path}`);
-            } else {
-                await this.del(path);
-            }
+            return;
         }
-    }
-
-    public async copy(config: {
-        path: string;
-        target: IArchives;
-        targetPath: string;
-    }): Promise<void> {
-        return this.move({ ...config, copyInstead: true });
+        let data = await this.get(path);
+        if (!data) throw new Error(`File not found to copy: ${path}`);
+        await target.set(targetPath, data);
     }
 
     public async getURL(path: string) {