sliftutils 1.7.10 → 1.7.12

package/storage/remoteStorage/ArchivesRemote.ts

@@ -73,6 +73,14 @@ export class ArchivesRemote implements IArchives {
         return `remoteStorage/${this.parsed.address}:${this.parsed.port}/${this.account}/${this.bucketName}`;
     }
 
+    public isConnected(): boolean {
+        return SocketFunction.isNodeConnected(this.nodeId);
+    }
+
+    public async ping(): Promise<void> {
+        await this.controller.ping();
+    }
+
     private async authenticate(): Promise<void> {
         await authenticateStorage({ address: this.parsed.address, port: this.parsed.port, nodeId: this.nodeId });
     }
@@ -103,17 +111,24 @@ export class ArchivesRemote implements IArchives {
         };
     }
 
-    private async onAccessDenied(): Promise<void> {
+    public async hasWriteAccess(): Promise<boolean> {
+        let state = await this.callAuthed(() => this.controller.getAccessState(this.account));
+        return !!state.hasAccess;
+    }
+
+    // Registers our access request server-side (so an admin has a requestId to grant) and logs the
+    // grant instructions, at most once a minute
+    private async registerAccessRequest(): Promise<void> {
         let requested = await this.callAuthed(() => this.controller.requestAccess(this.account));
         if (Date.now() - this.lastDeniedLog > timeInMinute) {
             this.lastDeniedLog = Date.now();
-            console.log(`No access to storage account ${JSON.stringify(this.account)} on ${this.parsed.address}:${this.parsed.port} (our machine ${requested.machineId}, ip ${requested.ip}). Waiting for access to be granted. See https://${this.parsed.address}:${this.parsed.port}/${this.account} - or grant it with: ${requested.grantAccessCommand}`);
+            console.log(`No access to storage account ${JSON.stringify(this.account)} on ${this.parsed.address}:${this.parsed.port} (our machine ${requested.machineId}, ip ${requested.ip}). See https://${this.parsed.address}:${this.parsed.port}/${this.account} - or grant it with: ${requested.grantAccessCommand}`);
         }
-        await delay(ACCESS_RETRY_DELAY);
     }
 
     // Runs a call, authenticating (and re-authenticating after reconnects) and waiting for account
-    // access as needed (unless waitForAccess is false, in which case denied calls throw).
+    // access as needed. With waitForAccess false, denied calls throw immediately instead - but the
+    // access request is still registered (in the background), so the denial is grantable.
     private async call<T>(fnc: () => Promise<T>): Promise<T> {
         while (true) {
             try {
@@ -124,8 +139,13 @@ export class ArchivesRemote implements IArchives {
                     await this.authenticate();
                     continue;
                 }
-                if (message.includes(STORAGE_ACCESS_DENIED) && this.config.waitForAccess !== false) {
-                    await this.onAccessDenied();
+                if (message.includes(STORAGE_ACCESS_DENIED)) {
+                    if (this.config.waitForAccess === false) {
+                        void this.registerAccessRequest().catch(() => { });
+                        throw e;
+                    }
+                    await this.registerAccessRequest();
+                    await delay(ACCESS_RETRY_DELAY);
                     continue;
                 }
                 throw e;
@@ -137,9 +157,9 @@ export class ArchivesRemote implements IArchives {
         let result = await this.get2(fileName, config);
         return result && result.data || undefined;
     }
-    public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
+    public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         let result = await this.call(() => this.controller.get2(this.account, this.bucketName, fileName, config?.range));
-        return result && { data: Buffer.from(result.data), writeTime: result.writeTime } || undefined;
+        return result && { data: Buffer.from(result.data), writeTime: result.writeTime, size: result.size } || undefined;
     }
     public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
         await this.call(() => this.controller.set(this.account, this.bucketName, fileName, data, config?.lastModified));

package/storage/remoteStorage/ArchivesUrl.d.ts

@@ -20,6 +20,7 @@ export declare class ArchivesUrl implements IArchives {
     }): Promise<{
         data: Buffer;
         writeTime: number;
+        size: number;
     } | undefined>;
     getInfo(fileName: string): Promise<{
         writeTime: number;
@@ -43,4 +44,5 @@ export declare class ArchivesUrl implements IArchives {
     }): Promise<ArchiveFileInfo[]>;
     getURL(path: string): Promise<string>;
     getConfig(): Promise<ArchivesConfig>;
+    hasWriteAccess(): Promise<boolean>;
 }

package/storage/remoteStorage/ArchivesUrl.ts

@@ -23,7 +23,7 @@ export class ArchivesUrl implements IArchives {
         let result = await this.get2(fileName, config);
         return result && result.data || undefined;
     }
-    public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
+    public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         let url = buildFileUrl(this.base, fileName);
         let headers: Record<string, string> = {};
         let range = config?.range;
@@ -36,17 +36,25 @@ export class ArchivesUrl implements IArchives {
             throw new Error(`Read of ${url} failed: ${response.status} ${response.statusText}`);
         }
         let data = Buffer.from(await response.arrayBuffer());
-        // Servers that don't support ranges (ours doesn't) return the full file with a 200
+        // A real 206 only returns the requested slice, so the full size lives in Content-Range's total.
+        let size = data.length;
+        let contentRange = response.headers.get("content-range");
+        let total = contentRange && Number(contentRange.split("/")[1]);
+        if (total && Number.isFinite(total)) {
+            size = total;
+        }
+        // Servers that don't support ranges return the full file with a 200 (ours serves real
+        // 206s, but backblaze friendly URLs and proxies may not)
         if (range && response.status === 200) {
             data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
         }
         let lastModified = response.headers.get("last-modified");
         let writeTime = lastModified && new Date(lastModified).getTime() || 0;
-        return { data, writeTime };
+        return { data, writeTime, size };
     }
     public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
         let result = await this.get2(fileName);
-        return result && { writeTime: result.writeTime, size: result.data.length } || undefined;
+        return result && { writeTime: result.writeTime, size: result.size } || undefined;
     }
 
     public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
@@ -71,4 +79,8 @@ export class ArchivesUrl implements IArchives {
     public async getConfig(): Promise<ArchivesConfig> {
         return {};
     }
+
+    public async hasWriteAccess(): Promise<boolean> {
+        return false;
+    }
 }

package/storage/remoteStorage/blobStore.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus } from "../IArchives";
+import { ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, SyncActivity } from "../IArchives";
 export declare const DEFAULT_FAST_WRITE_DELAY: number;
 export type WriteConfig = {
     fast?: boolean;
@@ -22,6 +22,7 @@ export type IBucketStore = {
     }): Promise<{
         data: Buffer;
         writeTime: number;
+        size: number;
     } | undefined>;
     set(fileName: string, data: Buffer, config?: WriteConfig): Promise<void>;
     del(fileName: string, config?: WriteConfig): Promise<void>;
@@ -35,6 +36,28 @@ export type IBucketStore = {
     }): Promise<ArchiveFileInfo[]>;
     getChangesAfter?(time: number): Promise<ArchiveFileInfo[]>;
     getSyncStatus?(): Promise<ArchivesSyncStatus>;
+    getSyncProgress?(): {
+        index: {
+            fileCount: number;
+            byteCount: number;
+        };
+        sources: {
+            debugName: string;
+            fileCount: number;
+            byteCount: number;
+        }[];
+        readerDiskLimit?: number;
+        syncing: SyncActivity[];
+    };
+    computeIndexTotals?(): Promise<{
+        fileCount: number;
+        byteCount: number;
+        sources: {
+            debugName: string;
+            fileCount: number;
+            byteCount: number;
+        }[];
+    }>;
     startLargeUpload(): Promise<string>;
     appendLargeUpload(id: string, data: Buffer): Promise<void>;
     finishLargeUpload(id: string, key: string): Promise<void>;
@@ -46,10 +69,16 @@ export declare class BlobStore implements IBucketStore {
     private config?;
     constructor(folder: string, sources: ArchivesSource[], config?: {
         onIndexChanged?: ((key: string) => void) | undefined;
+        readerDiskLimit?: number | undefined;
     } | undefined);
     private stopped;
     private index;
     private mem;
+    private indexFileCount;
+    private indexByteCount;
+    private sourceFileCounts;
+    private sourceByteCounts;
+    private syncActivities;
     private dirty;
     private overlay;
     private sourceStates;
@@ -60,11 +89,38 @@ export declare class BlobStore implements IBucketStore {
     };
     dispose(): Promise<void>;
     private loadIndex;
+    private countEntry;
     private setIndexEntry;
     private deleteIndexEntry;
+    /** The cheap always-current totals plus any in-progress background synchronization. */
+    getSyncProgress(): {
+        index: {
+            fileCount: number;
+            byteCount: number;
+        };
+        sources: {
+            debugName: string;
+            fileCount: number;
+            byteCount: number;
+        }[];
+        readerDiskLimit?: number;
+        syncing: SyncActivity[];
+    };
+    /** Walks the whole index for exact totals - more expensive than getSyncProgress, but immune to
+     *  any drift in the maintained counters (and loads the index first, so it's never cold zeros). */
+    computeIndexTotals(): Promise<{
+        fileCount: number;
+        byteCount: number;
+        sources: {
+            debugName: string;
+            fileCount: number;
+            byteCount: number;
+        }[];
+    }>;
     private flushIndex;
     private runSourceSync;
     private scanSource;
+    private reconcileSource;
     private applyScanned;
     private pollChanges;
     private copySourceFiles;
@@ -85,12 +141,13 @@ export declare class BlobStore implements IBucketStore {
     }): Promise<{
         data: Buffer;
         writeTime: number;
+        size: number;
     } | undefined>;
     private cacheRead;
     set(key: string, data: Buffer, config?: WriteConfig): Promise<void>;
-    private writeToSources;
     del(key: string, config?: WriteConfig): Promise<void>;
-    private deleteFromSources;
+    private getWritableSources;
+    private writeToSources;
     getInfo(key: string): Promise<{
         writeTime: number;
         size: number;
@@ -107,4 +164,7 @@ export declare class BlobStore implements IBucketStore {
     finishLargeUpload(id: string, key: string): Promise<void>;
     cancelLargeUpload(id: string): Promise<void>;
     private flushOverlay;
+    private evicting;
+    private enforceDiskLimit;
+    private cleanupTombstones;
 }