sliftutils 1.7.48 → 1.7.50

package/storage/remoteStorage/ArchivesRemote.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, SetConfig } from "../IArchives";
 export type ArchivesRemoteConfig = {
     url: string;
     waitForAccess?: boolean;
@@ -55,9 +55,7 @@ export declare class ArchivesRemote implements IArchives {
         writeTime: number;
         size: number;
     } | undefined>;
-    set(fileName: string, data: Buffer, config?: {
-        lastModified?: number;
-    }): Promise<string>;
+    set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
     del(fileName: string): Promise<void>;
     getInfo(fileName: string): Promise<{
         writeTime: number;
@@ -71,11 +69,12 @@ export declare class ArchivesRemote implements IArchives {
         shallow?: boolean;
         type: "files" | "folders";
     }): Promise<string[]>;
-    getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     getConfig(): Promise<ArchivesConfig>;
     getSyncStatus(): Promise<ArchivesSyncStatus>;
     setLargeFile(config: {
         path: string;
+        lastModified?: number;
         getNextData(): Promise<Buffer | undefined>;
     }): Promise<void>;
     getURL(path: string): Promise<string>;

package/storage/remoteStorage/ArchivesRemote.ts

@@ -2,7 +2,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { timeInMinute } from "socket-function/src/misc";
 import { delay } from "socket-function/src/batching";
 import { getIdentityCA, loadIdentityCA, sign } from "../../misc/https/certs";
-import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, SetConfig } from "../IArchives";
 import { parseHostedUrl, getBucketBaseUrl, buildFileUrl } from "./remoteConfig";
 import {
     RemoteStorageController, STORAGE_AUTH_PURPOSE,
@@ -145,8 +145,8 @@ export class ArchivesRemote implements IArchives {
         let result = await this.call(() => this.controller.get2(this.account, this.bucketName, fileName, config?.range));
         return result && { data: Buffer.from(result.data), writeTime: result.writeTime, size: result.size } || undefined;
     }
-    public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
-        await this.call(() => this.controller.set(this.account, this.bucketName, fileName, data, config?.lastModified));
+    public async set(fileName: string, data: Buffer, config?: SetConfig): Promise<string> {
+        await this.call(() => this.controller.set(this.account, this.bucketName, fileName, data, config?.lastModified, config?.forceSetImmutable));
         return fileName;
     }
     public async del(fileName: string): Promise<void> {
@@ -161,8 +161,8 @@ export class ArchivesRemote implements IArchives {
     public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
         return (await this.findInfo(prefix, config)).map(x => x.path);
     }
-    public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
-        return await this.call(() => this.controller.getChangesAfter(this.account, this.bucketName, time));
+    public async getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]> {
+        return await this.call(() => this.controller.getChangesAfter2(this.account, this.bucketName, config));
     }
     public async getConfig(): Promise<ArchivesConfig> {
         return await this.call(() => this.controller.getArchivesConfig(this.account, this.bucketName));
@@ -171,10 +171,10 @@ export class ArchivesRemote implements IArchives {
         return await this.call(() => this.controller.getSyncStatus(this.account, this.bucketName));
     }
 
-    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
+    public async setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
         // Ensure we're authenticated with access BEFORE consuming any data (the stream cannot be rewound, so we can't use the retry loop around the actual upload)
         await this.call(() => this.controller.getInfo(this.account, this.bucketName, config.path));
-        let uploadId = await this.controller.startLargeFile(this.account, this.bucketName, config.path);
+        let uploadId = await this.controller.startLargeFile(this.account, this.bucketName, config.path, config.lastModified);
         try {
             while (true) {
                 let data = await config.getNextData();

package/storage/remoteStorage/ArchivesUrl.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig } from "../IArchives";
 export declare class ArchivesUrl implements IArchives {
     private base;
     constructor(base: string);
@@ -42,6 +42,7 @@ export declare class ArchivesUrl implements IArchives {
         shallow?: boolean;
         type: "files" | "folders";
     }): Promise<ArchiveFileInfo[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     getURL(path: string): Promise<string>;
     getConfig(): Promise<ArchivesConfig>;
     hasWriteAccess(): Promise<boolean>;

package/storage/remoteStorage/ArchivesUrl.ts

@@ -1,5 +1,5 @@
 import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
-import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig } from "../IArchives";
 import { buildFileUrl } from "./remoteConfig";
 
 // Read-only IArchives over a public bucket's plain-URL form (our storage server's
@@ -53,8 +53,15 @@ export class ArchivesUrl implements IArchives {
         return { data, writeTime, size };
     }
     public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
-        let result = await this.get2(fileName);
-        return result && { writeTime: result.writeTime, size: result.size } || undefined;
+        // A 1-byte ranged read: Content-Range carries the full size and Last-Modified the write time, so metadata never downloads the file
+        try {
+            let result = await this.get2(fileName, { range: { start: 0, end: 1 } });
+            return result && { writeTime: result.writeTime, size: result.size } || undefined;
+        } catch {
+            // Some servers reject ranged reads of empty files (416 instead of an empty 206); a full read is the fallback, and for an empty file it is free anyway
+            let result = await this.get2(fileName);
+            return result && { writeTime: result.writeTime, size: result.size } || undefined;
+        }
     }
 
     public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
@@ -72,6 +79,9 @@ export class ArchivesUrl implements IArchives {
     public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
         throw this.readOnlyError("findInfo (listing)");
     }
+    public async getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]> {
+        throw this.readOnlyError("getChangesAfter2 (listing)");
+    }
 
     public async getURL(path: string): Promise<string> {
         return buildFileUrl(this.base, path);

package/storage/remoteStorage/blobStore.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, SyncActivity } from "../IArchives";
+import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, SyncActivity } from "../IArchives";
 export declare const DEFAULT_FAST_WRITE_DELAY: number;
 export declare const WINDOW_END_FLUSH_MARGIN: number;
 export type WriteConfig = {
@@ -35,7 +35,7 @@ export type IBucketStore = {
         shallow?: boolean;
         type?: "files" | "folders";
     }): Promise<ArchiveFileInfo[]>;
-    getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     getSyncStatus?(): Promise<ArchivesSyncStatus>;
     getSyncProgress?(): {
         index: {
@@ -61,11 +61,12 @@ export type IBucketStore = {
     }>;
     startLargeUpload(): Promise<string>;
     appendLargeUpload(id: string, data: Buffer): Promise<void>;
-    finishLargeUpload(id: string, key: string): Promise<void>;
+    finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void>;
     cancelLargeUpload(id: string): Promise<void>;
 };
 export type BlobSourceSpec = {
     identity: string;
+    url: string;
     validWindow: [number, number];
     route?: [number, number];
     noFullSync?: boolean;
@@ -79,6 +80,7 @@ export declare class BlobStore implements IBucketStore {
         onIndexChanged?: ((key: string) => void) | undefined;
         readerDiskLimit?: number | undefined;
         onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined;
+        resolveSourceUrl?: ((url: string) => IArchives) | undefined;
     } | undefined);
     private stopped;
     private index;
@@ -92,7 +94,14 @@ export declare class BlobStore implements IBucketStore {
     private overlay;
     private sourceStates;
     private syncStarted;
+    private sourcesList;
+    private slotSourcesListIndexes;
+    private slotRegistrations;
     private isLive;
+    private registerSlot;
+    private sourcesListIndexOfSlot;
+    private slotForSourcesListIndex;
+    private getEntryHolder;
     init: {
         (): Promise<void>;
         reset(): void;
@@ -141,7 +150,7 @@ export declare class BlobStore implements IBucketStore {
     private runSourceSync;
     private scanSource;
     private reconcileSource;
-    private applyScanned;
+    private updateScanIndex;
     private pollChanges;
     private copySourceFiles;
     private waitForRequiredScans;
@@ -176,12 +185,12 @@ export declare class BlobStore implements IBucketStore {
         shallow?: boolean;
         type?: "files" | "folders";
     }): Promise<ArchiveFileInfo[]>;
-    getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
+    getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
     getSyncStatus(): Promise<ArchivesSyncStatus>;
     private getDiskSource;
     startLargeUpload(): Promise<string>;
     appendLargeUpload(id: string, data: Buffer): Promise<void>;
-    finishLargeUpload(id: string, key: string): Promise<void>;
+    finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void>;
     cancelLargeUpload(id: string): Promise<void>;
     private flushOverlay;
     private evicting;