sliftutils 1.7.67 → 1.7.69

package/storage/remoteStorage/storageController.d.ts

@@ -2,6 +2,8 @@
 /// <reference types="node" />
 import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig } from "../IArchives";
 import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
+import { AccessTotals, AccessSummaryState } from "./accessStats";
+import type { SummaryEntry } from "../../treeSummary";
 export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
 export declare const STORAGE_NOT_AUTHENTICATED = "REMOTE_STORAGE_NOT_AUTHENTICATED_cf2f7b1e";
@@ -71,7 +73,7 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         size: number;
     } | undefined>;
     set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean, internal?: boolean) => Promise<void>;
-    del: (account: string, bucketName: string, path: string) => Promise<void>;
+    del: (account: string, bucketName: string, path: string, lastModified?: number, internal?: boolean) => Promise<void>;
     getInfo: (account: string, bucketName: string, path: string, includeTombstones?: boolean) => Promise<{
         writeTime: number;
         size: number;
@@ -88,6 +90,12 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
     clearWriteStats: (account: string) => Promise<{
         clearedBuckets: number;
     }>;
+    getAccessStats: (account: string) => Promise<AccessTotals>;
+    getAccessSummaries: (account: string, config: {
+        operation: string;
+        maxCount: number;
+        weightBySize?: boolean;
+    }) => Promise<SummaryEntry<AccessSummaryState>[]>;
     getIndexInfo: (account: string, bucketName: string) => Promise<{
         fileCount: number;
         byteCount: number;

package/storage/remoteStorage/storageController.ts

@@ -15,6 +15,8 @@ import {
     getActiveBucket, activateBucket, ActiveBucketInfo, getActiveBucketKeys,
 } from "./storageServerState";
 import { StorageClientController } from "./storageClientController";
+import { trackAccess, getAccessTotals, readAccessSummaries, clearAccountAccessStats, AccessTotals, AccessSummaryState } from "./accessStats";
+import type { SummaryEntry } from "../../treeSummary";
 
 export const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
@@ -292,37 +294,60 @@ class RemoteStorageControllerBase {
     async get2(account: string, bucketName: string, path: string, range?: { start: number; end: number }, internal?: boolean): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         assertValidPath(path);
         let bucket = await getBucket(account, bucketName);
-        if (!bucket) return undefined;
-        if (internal && bucket.store.getInternal2) {
-            return await bucket.store.getInternal2(path, { range });
+        let result: { data: Buffer; writeTime: number; size: number } | undefined;
+        if (bucket) {
+            if (internal && bucket.store.getInternal2) {
+                result = await bucket.store.getInternal2(path, { range });
+            } else {
+                result = await bucket.store.get2(path, { range });
+            }
         }
-        return await bucket.store.get2(path, { range });
+        trackAccess({ account, operation: "get", path: `${bucketName}/${path}`, size: result && result.data.length || 0 });
+        return result;
     }
     async set(account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean, internal?: boolean): Promise<void> {
         assertValidName(bucketName, "bucket name");
         assertValidPath(path);
+        if (!data.length) {
+            throw new Error(`set was called with an empty buffer for ${JSON.stringify(path)} in bucket ${account}/${bucketName}: an empty file IS a deletion in this system and would read back as missing - call del instead`);
+        }
+        trackAccess({ account, operation: "set", path: `${bucketName}/${path}`, size: data.length });
         await writeBucketFile(account, bucketName, path, Buffer.from(data), { lastModified, forceSetImmutable, internal });
     }
-    async del(account: string, bucketName: string, path: string): Promise<void> {
+    async del(account: string, bucketName: string, path: string, lastModified?: number, internal?: boolean): Promise<void> {
         assertValidName(bucketName, "bucket name");
         assertValidPath(path);
-        await deleteBucketFile(account, bucketName, path);
+        trackAccess({ account, operation: "del", path: `${bucketName}/${path}` });
+        await deleteBucketFile(account, bucketName, path, { lastModified, internal });
     }
     async getInfo(account: string, bucketName: string, path: string, includeTombstones?: boolean): Promise<{ writeTime: number; size: number } | undefined> {
         assertValidPath(path);
+        trackAccess({ account, operation: "getInfo", path: `${bucketName}/${path}` });
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return undefined;
         return await bucket.store.getInfo(path, { includeTombstones });
     }
     async findInfo(account: string, bucketName: string, prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
         let bucket = await getBucket(account, bucketName);
-        if (!bucket) return [];
-        return await bucket.store.findInfo(prefix, config);
+        let results = bucket && await bucket.store.findInfo(prefix, config) || [];
+        // The paths are only known once the results are in; an empty result still counts as one access at the prefix.
+        if (results.length) {
+            for (let info of results) {
+                trackAccess({ account, operation: "findInfo", path: `${bucketName}/${info.path}` });
+            }
+        } else {
+            trackAccess({ account, operation: "findInfo", path: `${bucketName}/${prefix}` });
+        }
+        return results;
     }
     async getChangesAfter2(account: string, bucketName: string, config: ChangesAfterConfig): Promise<ArchiveFileInfo[]> {
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return [];
-        return await bucket.store.getChangesAfter2(config);
+        let results = await bucket.store.getChangesAfter2(config);
+        for (let info of results) {
+            trackAccess({ account, operation: "getChangesAfter", path: `${bucketName}/${info.path}` });
+        }
+        return results;
     }
     async getArchivesConfig(account: string, bucketName: string): Promise<ArchivesConfig> {
         let bucket = await getBucket(account, bucketName);
@@ -348,10 +373,21 @@ class RemoteStorageControllerBase {
         assertValidName(bucketName, "bucket name");
         return await activateBucket(account, bucketName);
     }
-    /** Zeroes the write statistics listBuckets reports, for every bucket in the account. */
+    /** Zeroes the write statistics listBuckets reports and the in-memory access statistics, for every bucket in the account. */
     async clearWriteStats(account: string): Promise<{ clearedBuckets: number }> {
         assertValidName(account, "account");
-        return { clearedBuckets: await clearAccountWriteStats(account) };
+        clearAccountAccessStats(account);
+        return { clearedBuckets: clearAccountWriteStats(account) };
+    }
+    /** In-memory totals per operation type since startup (or the last clearWriteStats). */
+    async getAccessStats(account: string): Promise<AccessTotals> {
+        assertValidName(account, "account");
+        return getAccessTotals(account);
+    }
+    /** A path breakdown of one operation's accesses (operation names come from getAccessStats). maxCount is passed straight to TreeSummary.getSummaries. weightBySize is ignored for count-only operations, which return their count breakdown. */
+    async getAccessSummaries(account: string, config: { operation: string; maxCount: number; weightBySize?: boolean }): Promise<SummaryEntry<AccessSummaryState>[]> {
+        assertValidName(account, "account");
+        return readAccessSummaries({ account, ...config });
     }
     async getIndexInfo(account: string, bucketName: string): Promise<{ fileCount: number; byteCount: number; sources: { debugName: string; fileCount: number; byteCount: number }[] } | undefined> {
         let bucket = await getBucket(account, bucketName);
@@ -383,6 +419,7 @@ class RemoteStorageControllerBase {
         assertWritesAllowed();
         let info = largeUploadInfo.get(uploadId);
         if (!info) throw new Error(`Unknown large upload ${uploadId}`);
+        trackAccess({ account: info.account, operation: "uploadPart", path: `${info.bucketName}/${info.path}`, size: data.length });
         let bucket = await getBucket(info.account, info.bucketName);
         if (!bucket) throw new Error(`Bucket ${info.account}/${info.bucketName} no longer exists`);
         await bucket.store.appendLargeUpload(uploadId, Buffer.from(data));
@@ -437,6 +474,7 @@ class RemoteStorageControllerBase {
         }
         let info = await bucket.store.getInfo(filePath);
         if (!info || !info.size) {
+            trackAccess({ account, operation: "httpGet", path: `${bucketName}/${filePath}`, size: 0 });
             return setHTTPResultHeaders(Buffer.from(""), { status: "404" });
         }
         let ext = filePath.split(".").pop() || "";
@@ -477,6 +515,7 @@ class RemoteStorageControllerBase {
             }
         }
         let result = await bucket.store.get2(filePath, { range });
+        trackAccess({ account, operation: "httpGet", path: `${bucketName}/${filePath}`, size: result && result.data.length || 0 });
         if (!result) {
             return setHTTPResultHeaders(Buffer.from(""), { status: "404" });
         }
@@ -532,6 +571,8 @@ export const RemoteStorageController = SocketFunction.register(
         getActiveBucket: { hooks: [accountAccess] },
         activateBucket: { hooks: [accountAccess] },
         clearWriteStats: { hooks: [accountAccess] },
+        getAccessStats: { hooks: [accountAccess] },
+        getAccessSummaries: { hooks: [accountAccess] },
         getSyncStatus: { hooks: [accountAccess] },
         startLargeFile: { hooks: [accountAccess] },
         uploadPart: { hooks: [uploadAccess] },

package/storage/remoteStorage/storageServerState.d.ts

@@ -28,8 +28,8 @@ export type BucketWriteStats = {
     flushedWrites: number;
     flushedBytes: number;
 };
-/** Zeroes the write statistics of every bucket in the account, including counts not yet flushed. */
-export declare function clearAccountWriteStats(account: string): Promise<number>;
+/** Zeroes the write statistics of every bucket in the account. */
+export declare function clearAccountWriteStats(account: string): number;
 export declare function setTrustedMachines(config: {
     account: string;
     machineIds: string[];
@@ -111,5 +111,8 @@ export declare function getActiveBucket(account: string, bucketName: string): Pr
 /** Loads a bucket that exists on this server's disk into memory, which starts its synchronization and window timers, and returns its live state. Nothing is written and no other server is contacted - unlike building an ArchivesChain for it, which would probe every source and could write the routing config. Already-loaded buckets just return their state. */
 export declare function activateBucket(account: string, bucketName: string): Promise<ActiveBucketInfo | string>;
 export declare function listAccountBuckets(account: string): Promise<ServerBucketInfo[]>;
-export declare function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void>;
+export declare function deleteBucketFile(account: string, bucketName: string, filePath: string, config?: {
+    lastModified?: number;
+    internal?: boolean;
+}): Promise<void>;
 export declare function getLocalArchives(account: string, bucketName: string): IArchives;

package/storage/remoteStorage/storageServerState.ts

@@ -10,7 +10,7 @@ import { ArchivesDisk } from "../ArchivesDisk";
 import { BlobStore, IBucketStore, DEFAULT_FAST_WRITE_DELAY, WINDOW_END_FLUSH_MARGIN } from "./blobStore";
 import {
     RemoteConfig, HostedConfig, BackblazeConfig, IArchives, ArchivesSource, ArchiveFileInfo, ArchivesConfig,
-    ArchivesSyncStatus, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig,
+    ArchivesSyncStatus, ChangesAfterConfig, DelConfig, GetConfig, GetInfoConfig, SetConfig,
     STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
 } from "../IArchives";
 import { ROUTING_FILE, parseRoutingData, serializeRemoteConfig, parseHostedUrl, replaceHostedUrlPort, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection, normalizeSource } from "./remoteConfig";
@@ -98,78 +98,36 @@ export type BucketWriteStats = {
 function emptyWriteStats(): BucketWriteStats {
     return { originalWrites: 0, originalBytes: 0, flushedWrites: 0, flushedBytes: 0 };
 }
-function addWriteStats(a: BucketWriteStats, b: BucketWriteStats): BucketWriteStats {
-    return {
-        originalWrites: a.originalWrites + b.originalWrites,
-        originalBytes: a.originalBytes + b.originalBytes,
-        flushedWrites: a.flushedWrites + b.flushedWrites,
-        flushedBytes: a.flushedBytes + b.flushedBytes,
-    };
-}
-function getWriteStatsStorage(): Promise<IStorage<BucketWriteStats>> {
-    return getSystemStorage<BucketWriteStats>("writeStats");
-}
 
-// Counted in memory as deltas since the last flush, so counting a write never touches the disk. The flush reads the stored totals and adds the delta, which also makes it correct across restarts.
-const writeStatDeltas = new Map<string, BucketWriteStats>();
-
-async function flushWriteStats(): Promise<void> {
-    if (!writeStatDeltas.size) return;
-    let pending = [...writeStatDeltas];
-    writeStatDeltas.clear();
-    let storage = await getWriteStatsStorage();
-    for (let [key, delta] of pending) {
-        try {
-            await storage.set(key, addWriteStats(await storage.get(key) || emptyWriteStats(), delta));
-        } catch (e) {
-            // Put the delta back, so a failed flush loses nothing
-            writeStatDeltas.set(key, addWriteStats(writeStatDeltas.get(key) || emptyWriteStats(), delta));
-            console.error(`Flushing write stats for bucket ${key} failed: ${(e as Error).stack ?? e}`);
-        }
-    }
-}
-
-const startWriteStatsFlushing = lazy(() => {
-    runInfinitePoll(WRITE_STATS_FLUSH_INTERVAL, flushWriteStats);
-});
+// In memory only: totals since this process started (or the last clearWriteStats). Persisting them to disk was more machinery than the numbers were worth.
+const writeStats = new Map<string, BucketWriteStats>();
 
 function countBucketWrite(key: string, kind: "original" | "flushed", bytes: number): void {
-    let delta = writeStatDeltas.get(key);
-    if (!delta) {
-        delta = emptyWriteStats();
-        writeStatDeltas.set(key, delta);
+    let stats = writeStats.get(key);
+    if (!stats) {
+        stats = emptyWriteStats();
+        writeStats.set(key, stats);
     }
     if (kind === "original") {
-        delta.originalWrites++;
-        delta.originalBytes += bytes;
+        stats.originalWrites++;
+        stats.originalBytes += bytes;
     } else {
-        delta.flushedWrites++;
-        delta.flushedBytes += bytes;
+        stats.flushedWrites++;
+        stats.flushedBytes += bytes;
     }
-    startWriteStatsFlushing();
 }
 
-async function getBucketWriteStats(key: string): Promise<BucketWriteStats> {
-    let storage = await getWriteStatsStorage();
-    let stored = await storage.get(key) || emptyWriteStats();
-    let delta = writeStatDeltas.get(key);
-    return delta && addWriteStats(stored, delta) || stored;
+function getBucketWriteStats(key: string): BucketWriteStats {
+    return writeStats.get(key) || emptyWriteStats();
 }
 
-/** Zeroes the write statistics of every bucket in the account, including counts not yet flushed. */
-export async function clearAccountWriteStats(account: string): Promise<number> {
-    let storage = await getWriteStatsStorage();
+/** Zeroes the write statistics of every bucket in the account. */
+export function clearAccountWriteStats(account: string): number {
     let prefix = `${account}/`;
     let cleared = 0;
-    for (let key of await storage.getKeys()) {
-        if (!key.startsWith(prefix)) continue;
-        writeStatDeltas.delete(key);
-        await storage.remove(key);
-        cleared++;
-    }
-    for (let key of [...writeStatDeltas.keys()]) {
+    for (let key of [...writeStats.keys()]) {
         if (!key.startsWith(prefix)) continue;
-        writeStatDeltas.delete(key);
+        writeStats.delete(key);
         cleared++;
     }
     console.log(`Cleared the write statistics of ${cleared} buckets in account ${account}`);
@@ -211,7 +169,6 @@ const buckets = new Map<string, Promise<LoadedBucket | undefined>>();
 
 const MAX_REBUILD_TIMER_DELAY = 2 ** 31 - 1;
 const REBUILD_BOUNDARY_BUFFER = 1000;
-const WRITE_STATS_FLUSH_INTERVAL = 5 * 60 * 1000;
 
 function getBucketFolder(account: string, bucketName: string): string {
     return path.join(getStorageServerConfig().folder, "buckets2", account, bucketName);
@@ -1070,15 +1027,11 @@ export async function listAccountBuckets(account: string): Promise<ServerBucketI
             let key = `${account}/${bucketName}`;
             let folder = getBucketFolder(account, bucketName);
             let base: ServerBucketInfo = { bucketName, active: false, folder };
-            let [disk, writeStats] = await Promise.all([
-                timed("statfs", () => getDiskInfo(folder)).catch((e: Error) => {
-                    base.diskError = String(e.stack ?? e).slice(0, 500);
-                    return undefined;
-                }),
-                timed("writeStats", () => getBucketWriteStats(key)),
-            ]);
-            base.disk = disk;
-            base.writeStats = writeStats;
+            base.disk = await timed("statfs", () => getDiskInfo(folder)).catch((e: Error) => {
+                base.diskError = String(e.stack ?? e).slice(0, 500);
+                return undefined;
+            });
+            base.writeStats = getBucketWriteStats(key);
             const loadedPromise = buckets.get(key);
             if (loadedPromise) {
                 try {
@@ -1108,13 +1061,33 @@ export async function listAccountBuckets(account: string): Promise<ServerBucketI
     }
 }
 
-export async function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void> {
+export async function deleteBucketFile(account: string, bucketName: string, filePath: string, config?: { lastModified?: number; internal?: boolean }): Promise<void> {
+    assertWritesAllowed();
     if (filePath === ROUTING_FILE) {
         throw new Error(`The routing config ${JSON.stringify(ROUTING_FILE)} cannot be deleted (overwrite it to change the bucket's configuration)`);
     }
     let loaded = await getLoadedBucket(account, bucketName);
     if (!loaded) return;
-    await loaded.store.del(filePath, getWriteConfig(loaded, Date.now(), getRoute(filePath)));
+    let writeTime = config?.lastModified || Date.now();
+    let route = getRoute(filePath);
+    if (config?.internal) {
+        if (!config.lastModified) {
+            throw new Error(`Internal deletions must carry lastModified (they are synchronization pushes, ordered by their write time), deleting ${JSON.stringify(filePath)} in bucket ${account}/${bucketName}`);
+        }
+        // Same acceptance rule as internal writes (see SetConfig.internal): the stamp must land inside SOME window+route this server is configured for
+        let covered = loaded.selfEntries.some(x => writeTime >= x.validWindow[0] && writeTime < x.validWindow[1] && routeContains(x.route, route));
+        if (!covered) {
+            throw new Error(`Internal deletion of ${JSON.stringify(filePath)} in bucket ${account}/${bucketName} rejected: writeTime ${writeTime} (${new Date(writeTime).toISOString()}) at route ${route} is outside every window/route this server is configured for: ${JSON.stringify(loaded.selfEntries.map(x => ({ validWindow: x.validWindow, route: x.route || FULL_ROUTE })))}`);
+        }
+        if (loaded.store.setInternal) {
+            // setInternal treats an empty buffer as exactly a deletion: disk removal plus a tombstone index entry, no fan-out
+            await loaded.store.setInternal(filePath, Buffer.alloc(0), { lastModified: writeTime });
+            return;
+        }
+        await loaded.store.del(filePath);
+        return;
+    }
+    await loaded.store.del(filePath, { ...getWriteConfig(loaded, writeTime, route), lastModified: writeTime });
 }
 
 const LARGE_FILE_PART_SIZE = 8 * 1024 * 1024;
@@ -1145,8 +1118,8 @@ class ArchivesLocalBucket implements IArchives {
         await writeBucketFile(this.account, this.bucketName, fileName, data, config);
         return fileName;
     }
-    public async del(fileName: string): Promise<void> {
-        await deleteBucketFile(this.account, this.bucketName, fileName);
+    public async del(fileName: string, config?: DelConfig): Promise<void> {
+        await deleteBucketFile(this.account, this.bucketName, fileName, config);
     }
     public async setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
         assertWritesAllowed();

package/test.ts

@@ -57,7 +57,7 @@ function summarize(files: FileInfo[], weightBySize: boolean) {
             if (weightBySize) return summary.totalSize;
             return summary.count;
         },
-        expectedOutputCount: SUMMARY_COUNT,
+        expectedOutputCount: 100,
     });
     let addStart = performance.now();
     for (let file of files) {

package/treeSummary.d.ts

@@ -0,0 +1,31 @@
+export type SummaryEntry<S> = {
+    path: string;
+    kind: "self" | "subtree" | "group" | "truncated";
+    summary: S;
+    weight: number;
+};
+export declare class TreeSummary<T, S> {
+    private config;
+    private root;
+    private nodeCount;
+    private maxTrackedNodes;
+    constructor(config: {
+        getPath: (value: T) => string;
+        createSummary: () => S;
+        addToSummary: (value: T, summary: S) => void;
+        mergeSummaries: (target: S, source: S) => void;
+        getWeight: (summary: S) => number;
+        expectedOutputCount?: number;
+    });
+    add(value: T): void;
+    private splitNode;
+    getTrackedNodeCount(): number;
+    getSummaries(maxCount: number): SummaryEntry<S>[];
+    private isRefinable;
+    private refine;
+    private splitParts;
+    private entryForParts;
+    private subtreeEntry;
+    private buildOutputSummary;
+    private prune;
+}