sliftutils 1.7.48 → 1.7.50

package/storage/IArchives.ts

@@ -58,7 +58,7 @@ export type HostedConfig = CommonConfig & {
     // Fast mode: the server acknowledges writes once they are in memory, flushing to disk after writeDelay (default 5 minutes) and coalescing writes to the same file. A server crash loses writes that haven't flushed yet.
     fast?: boolean;
     writeDelay?: number;
-    // The bucket is served straight from the server's disk, with no index — so no fast writes and no getChangesAfter/getSyncStatus.
+    // The bucket is served straight from the server's disk, with no index — so no fast writes, no getSyncStatus, and getChangesAfter2 falls back to a full listing.
     rawDisk?: boolean;
     // Writes to paths that already exist are disallowed (deletes still work).
     immutable?: boolean;
@@ -86,6 +86,20 @@ export const FULL_VALID_WINDOW: [number, number] = [0, Number.MAX_SAFE_INTEGER];
 
 
 
+export type ChangesAfterConfig = {
+    time: number;
+    /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
+    routes?: [number, number][];
+};
+
+export type SetConfig = {
+    lastModified?: number;
+    /** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
+    forceSetImmutable?: boolean;
+    /** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
+    noChecks?: boolean;
+};
+
 // createTime is a misnomer kept for compatibility — it is really the LAST-WRITE time, same as getInfo's writeTime. Neither Backblaze nor our remote storage tracks a distinct creation date: each write stamps a fresh timestamp on the current version, so both fields are just "when the bytes served by get() were most recently written".
 export type ArchiveFileInfo = { path: string; createTime: number; size: number };
 
@@ -102,7 +116,7 @@ export type SyncActivity = {
 };
 
 export type ArchivesConfig = {
-    // Whether getChangesAfter is implemented (fast change polling, instead of full rescans)
+    // Whether getChangesAfter2 is natively index-backed (fast change polling; every backend still serves it, but the others emulate it with a full listing)
     supportsChangesAfter?: boolean;
     // The bucket's full routing config (ROUTING_FILE). Absent for sources that don't have one (a bare disk source, or a bucket that doesn't exist yet).
     remoteConfig?: RemoteConfig;
@@ -119,6 +133,8 @@ export type ArchivesConfig = {
 // A synchronization source of a BlobStore (which synchronizes an index + local cache from them)
 export type ArchivesSource = {
     source: IArchives;
+    /** The persistent identity of the endpoint: its routing URL (hosted/backblaze), or the disk folder path for the base disk source. The store persists this (via its append-only sources list) as IndexEntry.sourcesListIndex, so it must mean the same endpoint forever. */
+    url: string;
     // From the source's CommonConfig. Values with write times outside the window are ignored when scanning, and once its end is past (see windowAcceptsWrites) the source stops receiving writes entirely - still scanned (it holds the authoritative data for its window), it just stops growing.
     validWindow: [number, number];
     // From the source's CommonConfig (intersected with the owning store's own route): only keys routing into [start, end) are accepted from this source's scans and sent to it in writes/reconciliation. The routing file is exempt - config flows everywhere. Absent = all keys.
@@ -145,6 +161,55 @@ export function windowAcceptsWrites(validWindow: [number, number] | undefined):
     return validWindow[1] > Date.now();
 }
 
+const LARGE_COPY_THRESHOLD = 64 * 1024 * 1024;
+const LARGE_COPY_CHUNK = 32 * 1024 * 1024;
+
+/** Copies one file between two archives. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. size/writeTime usually come from the caller's metadata scan; when either is omitted, getInfo fills them in. Returns the copied file's info, or undefined when the source doesn't have the file. */
+export async function copyArchiveFile(config: {
+    from: IArchives;
+    to: IArchives;
+    path: string;
+    size?: number;
+    writeTime?: number;
+    forceSetImmutable?: boolean;
+    noChecks?: boolean;
+}): Promise<{ writeTime: number; size: number } | undefined> {
+    let { from, to, path } = config;
+    let size = config.size;
+    let writeTime = config.writeTime;
+    if (size === undefined || writeTime === undefined) {
+        let info = await from.getInfo(path);
+        if (!info) return undefined;
+        size = info.size;
+        writeTime = info.writeTime;
+    }
+    if (size <= LARGE_COPY_THRESHOLD) {
+        let result = await from.get2(path);
+        if (!result) return undefined;
+        await to.set(path, result.data, { lastModified: result.writeTime, forceSetImmutable: config.forceSetImmutable, noChecks: config.noChecks });
+        return { writeTime: result.writeTime, size: result.data.length };
+    }
+    // Consts so the closure keeps the narrowed types
+    const totalSize = size;
+    const finalWriteTime = writeTime;
+    let offset = 0;
+    await to.setLargeFile({
+        path,
+        lastModified: finalWriteTime,
+        getNextData: async () => {
+            if (offset >= totalSize) return undefined;
+            let end = Math.min(offset + LARGE_COPY_CHUNK, totalSize);
+            let data = await from.get(path, { range: { start: offset, end } });
+            if (!data || !data.length) {
+                throw new Error(`Ranged read of ${JSON.stringify(path)} from ${from.getDebugName()} returned ${data && data.length || "nothing"} at ${offset}-${end} (expected ${end - offset} bytes of a ${totalSize} byte file - it changed or vanished mid-copy)`);
+            }
+            offset += data.length;
+            return data;
+        },
+    });
+    return { writeTime: finalWriteTime, size: totalSize };
+}
+
 export type ArchivesSyncSourceStatus = {
     debugName: string;
     validWindow: [number, number];
@@ -177,10 +242,10 @@ export interface IArchives {
      * VARIABLE_SHARD, where the shard value is materialized into the key (picked by shard latency,
      * see ArchivesChain) and the caller needs the returned key to ever read the value back.
      */
-    set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string>;
+    set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(fileName: string): Promise<void>;
-    /** Streams a file too large to hold in memory. getNextData returns undefined when done. */
-    setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void>;
+    /** Streams a file too large to hold in memory. getNextData returns undefined when done. lastModified stamps the finished file like set's (synchronized copies need it to keep write ordering); backends that stamp their own times (backblaze) accept and ignore it. */
+    setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined> }): Promise<void>;
     /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. */
     getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined>;
     find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]>;
@@ -190,10 +255,13 @@ export interface IArchives {
     /** The bucket's configuration, which tells whether the optional functions are supported. */
     getConfig(): Promise<ArchivesConfig>;
     /**
-     * All files changed after the given time. Only exists when getConfig().supportsChangesAfter;
-     * backed by an index, so it is fast (unlike a full findInfo scan). Deletions are not reported.
+     * All files changed after config.time, optionally restricted to keys routing into one of
+     * config.routes (used by scanning, so partially-overlapping shards only receive their slice).
+     * When getConfig().supportsChangesAfter, this is backed by an index (fast, and deletions ARE
+     * reported, as size-0 tombstone entries). Every other backend emulates it: a full findInfo
+     * listing filtered in memory - correct, but no cheaper than the listing itself.
      */
-    getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     /** Synchronization introspection, for backends that synchronize from sources (see BlobStore). */
     getSyncStatus?(): Promise<ArchivesSyncStatus>;
 }

package/storage/backblaze.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchivesConfig } from "./IArchives";
+import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, SetConfig } from "./IArchives";
 export declare class ArchivesBackblaze implements IArchives {
     private config;
     constructor(config: {
@@ -39,12 +39,11 @@ export declare class ArchivesBackblaze implements IArchives {
     } | undefined>;
     getConfig(): Promise<ArchivesConfig>;
     hasWriteAccess(): Promise<boolean>;
-    set(fileName: string, data: Buffer, config?: {
-        lastModified?: number;
-    }): Promise<string>;
+    set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(fileName: string): Promise<void>;
     setLargeFile(config: {
         path: string;
+        lastModified?: number;
         getNextData(): Promise<Buffer | undefined>;
     }): Promise<void>;
     getInfo(fileName: string): Promise<{
@@ -55,6 +54,7 @@ export declare class ArchivesBackblaze implements IArchives {
         shallow?: boolean;
         type: "files" | "folders";
     }): Promise<string[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     findInfo(prefix: string, config?: {
         shallow?: boolean;
         type: "files" | "folders";

package/storage/backblaze.ts

@@ -8,8 +8,9 @@ import { blue, green, magenta } from "socket-function/src/formatting/logColors";
 import debugbreak from "debugbreak";
 import dns from "dns";
 import { getSecret } from "../misc/getSecret";
-import { httpsRequest } from "socket-function/src/https";
-import { IArchives, ArchivesConfig, assertValidLastModified, IMMUTABLE_CACHE_TIME } from "./IArchives";
+import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
+import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, SetConfig, assertValidLastModified, IMMUTABLE_CACHE_TIME } from "./IArchives";
+import { filterChanges } from "./remoteStorage/remoteConfig";
 
 type BackblazeCreds = {
     applicationKeyId: string;
@@ -24,6 +25,17 @@ let backblazeCreds = lazy(async (): Promise<BackblazeCreds> => {
         applicationKey: key,
     };
 });
+// A B2 download/HEAD response's headers carry the file's metadata. content-range's total wins over content-length for ranged responses (which only report the slice's length).
+function parseFileMetadataHeaders(response: HttpsResponseInfo): { size: number; uploadTimestamp: number } {
+    let size = Number(response.headers["content-length"] || 0);
+    let contentRange = response.headers["content-range"];
+    let total = contentRange && Number(contentRange.split("/")[1]);
+    if (total && Number.isFinite(total)) {
+        size = total;
+    }
+    return { size, uploadTimestamp: Number(response.headers["x-bz-upload-timestamp"] || 0) };
+}
+
 const getAPI = lazy(async () => {
     let creds = await backblazeCreds();
 
@@ -148,6 +160,8 @@ const getAPI = lazy(async () => {
         bucketName: string;
         fileName: string;
         range?: { start: number; end: number; };
+        // Captures the response headers, which carry the file's metadata (x-bz-upload-timestamp, content-length / content-range) - so callers that need it don't pay a second API call
+        outResponse?: HttpsResponseInfo;
     }) {
         let fileName = encodePath(config.fileName);
 
@@ -160,10 +174,31 @@ const getAPI = lazy(async () => {
                 "Content-Type": "application/json",
                 Range: config.range ? `bytes=${config.range.start}-${config.range.end - 1}` : undefined,
             }).filter(x => x[1] !== undefined)),
+            outResponse: config.outResponse,
         });
         return result;
     }
 
+    /** A file's metadata by name, without the body: HEAD on the download URL, which is a class B (download-priced) transaction - b2_list_file_names is class C at 10x the price, and b2_get_file_info needs a fileId we don't have. Returns undefined for missing (or hidden) files. */
+    async function headFileByName(config: { bucketName: string; fileName: string }): Promise<{ size: number; uploadTimestamp: number } | undefined> {
+        let fileName = encodePath(config.fileName);
+        let response: HttpsResponseInfo = { headers: {} };
+        try {
+            await httpsRequest(auth.apiUrl + "/file/" + config.bucketName + "/" + fileName, undefined, "HEAD", undefined, {
+                headers: { Authorization: auth.authorizationToken },
+                outResponse: response,
+            });
+        } catch (e) {
+            if (response.statusCode === 404) return undefined;
+            // HEAD responses have no body, so the expired_auth_token marker apiRetryLogic matches on never appears - re-add it so an expired token still refreshes and retries
+            if (response.statusCode === 401) {
+                throw new Error(`"expired_auth_token" HEAD ${config.bucketName}/${config.fileName} got a 401: ${(e as Error).stack ?? e}`);
+            }
+            throw e;
+        }
+        return parseFileMetadataHeaders(response);
+    }
+
     // Oh... apparently, we can't reuse these? Huh...
     const getUploadURL = (async (bucketId: string) => {
         // setTimeout(() => getUploadURL.clear(bucketId), timeInHour * 1);
@@ -353,6 +388,7 @@ const getAPI = lazy(async () => {
         updateBucket,
         listBuckets,
         downloadFileByName,
+        headFileByName,
         uploadFile,
         hideFile,
         getFileInfo,
@@ -587,6 +623,10 @@ export class ArchivesBackblaze implements IArchives {
     }
 
     public async get(fileName: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
+        let result = await this.get2(fileName, config);
+        return result && result.data || undefined;
+    }
+    public async get2(fileName: string, config?: { range?: { start: number; end: number; } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         let downloading = true;
         try {
             let time = Date.now();
@@ -596,33 +636,39 @@ export class ArchivesBackblaze implements IArchives {
                 setTimeout(downloadPoll, 5000);
             };
             setTimeout(downloadPoll, 5000);
-            let result = await this.apiRetryLogic(`get ${fileName}`, async (api) => {
+            let result = await this.apiRetryLogic(`get ${fileName}`, async (api): Promise<{ data: Buffer; writeTime: number; size: number }> => {
                 let range = config?.range;
                 if (range) {
+                    // The clamp needs the file's size (and gives us its metadata for free)
                     let fileInfo = await this.getInfo(fileName);
                     if (!fileInfo) throw new Error(`File ${fileName} not found`);
                     let rangeStart = range.start;
                     let rangeEnd = Math.min(range.end, fileInfo.size);
                     // NOTE: I think if we request nothing, it confuses Backblaze and ends up giving us the entire file.
-                    if (rangeEnd <= rangeStart) return Buffer.alloc(0);
-                    let result = await api.downloadFileByName({
+                    if (rangeEnd <= rangeStart) return { data: Buffer.alloc(0), writeTime: fileInfo.writeTime, size: fileInfo.size };
+                    let data = await api.downloadFileByName({
                         bucketName: this.bucketName,
                         fileName,
                         range: { start: rangeStart, end: rangeEnd },
                     });
-                    if (result.length !== rangeEnd - rangeStart) {
-                        console.error(`Backblaze range download returned the wrong number of bytes. Tried to get ${rangeStart}-${rangeEnd}, but received ${rangeStart}-${rangeStart + result.length}. For file: ${fileName}`);
+                    if (data.length !== rangeEnd - rangeStart) {
+                        console.error(`Backblaze range download returned the wrong number of bytes. Tried to get ${rangeStart}-${rangeEnd}, but received ${rangeStart}-${rangeStart + data.length}. For file: ${fileName}`);
                     }
-                    return result;
+                    return { data, writeTime: fileInfo.writeTime, size: fileInfo.size };
                 }
-                return await api.downloadFileByName({
+                // The download's own response headers carry the metadata, so a full read is a single API call
+                let response: HttpsResponseInfo = { headers: {} };
+                let data = await api.downloadFileByName({
                     bucketName: this.bucketName,
                     fileName,
+                    outResponse: response,
                 });
+                let meta = parseFileMetadataHeaders(response);
+                return { data, writeTime: meta.uploadTimestamp, size: data.length };
             });
             let timeStr = formatTime(Date.now() - time);
-            let rateStr = formatNumber(result.length / (Date.now() - time) * 1000) + "B/s";
-            this.log(`backblaze download (${formatNumber(result.length)}B${config?.range && `, ${formatNumber(config.range.start)} - ${formatNumber(config.range.end)}` || ""}) in ${timeStr} (${rateStr}, ${fileName})`);
+            let rateStr = formatNumber(result.data.length / (Date.now() - time) * 1000) + "B/s";
+            this.log(`backblaze download (${formatNumber(result.data.length)}B${config?.range && `, ${formatNumber(config.range.start)} - ${formatNumber(config.range.end)}` || ""}) in ${timeStr} (${rateStr}, ${fileName})`);
             return result;
         } catch (e) {
             this.log(`backblaze file does not exist ${fileName}`);
@@ -631,12 +677,6 @@ export class ArchivesBackblaze implements IArchives {
             downloading = false;
         }
     }
-    public async get2(fileName: string, config?: { range?: { start: number; end: number; } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
-        // B2 downloads don't return the upload time, so this takes a second API call
-        let [data, info] = await Promise.all([this.get(fileName, config), this.getInfo(fileName)]);
-        if (!data || !info) return undefined;
-        return { data, writeTime: info.writeTime, size: info.size };
-    }
     public async getConfig(): Promise<ArchivesConfig> {
         return {};
     }
@@ -656,28 +696,37 @@ export class ArchivesBackblaze implements IArchives {
             return false;
         }
     }
-    public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
+    public async set(fileName: string, data: Buffer, config?: SetConfig): Promise<string> {
+        if (config?.forceSetImmutable && !config.lastModified) {
+            throw new Error(`forceSetImmutable requires lastModified (synchronization writes are ordered by their write time), writing ${fileName} to ${this.getDebugName()}`);
+        }
         if (config?.lastModified) {
             assertValidLastModified(config.lastModified);
-            let existing = await this.getInfo(fileName);
-            // An older write never overwrites a newer one (see IArchives.set). B2 stamps its own upload time, so the exact lastModified is not preserved on the stored file.
-            if (existing && config.lastModified < existing.writeTime) return fileName;
+            if (!config.noChecks) {
+                let existing = await this.getInfo(fileName);
+                // An older write never overwrites a newer one (see IArchives.set). B2 stamps its own upload time, so the exact lastModified is not preserved on the stored file.
+                if (existing && config.lastModified < existing.writeTime) return fileName;
+                // Immutability wins: a synchronization push never overwrites an existing path on an immutable bucket (see SetConfig.forceSetImmutable)
+                if (existing && config.forceSetImmutable && this.config.immutable) return fileName;
+            }
         }
         this.log(`backblaze upload (${formatNumber(data.length)}B) ${fileName}`);
         let f = fileName;
         await this.apiRetryLogic(`uploadFile ${fileName}`, async (api) => {
             await api.uploadFile({ bucketId: this.bucketId, fileName, data: data, });
         });
-        let existsChecks = 30;
-        while (existsChecks > 0) {
-            let exists = await this.getInfo(fileName);
-            if (exists) break;
-            await delay(1000);
-            existsChecks--;
-        }
-        if (existsChecks === 0) {
-            let exists = await this.getInfo(fileName);
-            console.warn(`File ${fileName}/${f} was uploaded, but could not be found afterwards. Hopefully it was just deleted, very quickly? If backblaze is taking too long for files to propagate, then we might run into issues with the database atomicity.`);
+        if (!config?.noChecks) {
+            let existsChecks = 30;
+            while (existsChecks > 0) {
+                let exists = await this.getInfo(fileName);
+                if (exists) break;
+                await delay(1000);
+                existsChecks--;
+            }
+            if (existsChecks === 0) {
+                let exists = await this.getInfo(fileName);
+                console.warn(`File ${fileName}/${f} was uploaded, but could not be found afterwards. Hopefully it was just deleted, very quickly? If backblaze is taking too long for files to propagate, then we might run into issues with the database atomicity.`);
+            }
         }
         return fileName;
     }
@@ -694,7 +743,8 @@ export class ArchivesBackblaze implements IArchives {
         // NOTE: Deletion SEEMS to work. This DOES break if we delete a file which keeps being recreated, ex, the heartbeat. let existsChecks = 10; while (existsChecks > 0) { let exists = await this.getInfo(fileName); if (!exists) break; await delay(1000); existsChecks--; } if (existsChecks === 0) { let exists = await this.getInfo(fileName); devDebugbreak(); console.warn(`File ${fileName} was deleted, but was still found afterwards`); exists = await this.getInfo(fileName); }
     }
 
-    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
+    // lastModified is accepted but cannot be honored - b2 stamps its own uploadTimestamp, which is what our getInfo/findInfo report as the write time
+    public async setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
 
         let onError: (() => Promise<void>)[] = [];
         let time = Date.now();
@@ -829,9 +879,7 @@ export class ArchivesBackblaze implements IArchives {
     public async getInfo(fileName: string): Promise<{ writeTime: number; size: number; } | undefined> {
         return await this.apiRetryLogic(`getInfo ${fileName}`, async (api) => {
             try {
-                // NOTE: Apparently, there's no other way to do this, as the file name does not equal the file ID, and get file info requires the file ID.
-                let info = await api.listFileNames({ bucketId: this.bucketId, prefix: fileName, maxFileCount: 10 });
-                let file = info.files.find(x => x.fileName === fileName && x.action === "upload");
+                let file = await api.headFileByName({ bucketName: this.bucketName, fileName });
                 if (!file) {
                     this.log(`Backblaze file not exists ${fileName}`);
                     return undefined;
@@ -839,7 +887,7 @@ export class ArchivesBackblaze implements IArchives {
                 this.log(`Backblaze file exists ${fileName}`);
                 return {
                     writeTime: file.uploadTimestamp,
-                    size: file.contentLength,
+                    size: file.size,
                 };
             } catch (e: any) {
                 if (e.stack.includes(`file_not_found`)) {
@@ -856,6 +904,11 @@ export class ArchivesBackblaze implements IArchives {
         let result = await this.findInfo(prefix, config);
         return result.map(x => x.path);
     }
+    public async getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]> {
+        // No native change feed (b2 supports neither time nor route filtering) - a full listing filtered in memory
+        return filterChanges(await this.findInfo(""), config);
+    }
+
     public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
         return await this.apiRetryLogic(`findInfo ${prefix}`, async (api) => {
             if (!config?.shallow && config?.type === "folders") {