sliftutils 1.7.67 → 1.7.68

package/storage/remoteStorage/storageController.d.ts

@@ -2,6 +2,8 @@
 /// <reference types="node" />
 import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig } from "../IArchives";
 import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
+import { AccessTotals, AccessSummaryState } from "./accessStats";
+import type { SummaryEntry } from "../../treeSummary";
 export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
 export declare const STORAGE_NOT_AUTHENTICATED = "REMOTE_STORAGE_NOT_AUTHENTICATED_cf2f7b1e";
@@ -88,6 +90,12 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
     clearWriteStats: (account: string) => Promise<{
         clearedBuckets: number;
     }>;
+    getAccessStats: (account: string) => Promise<AccessTotals>;
+    getAccessSummaries: (account: string, config: {
+        operation: string;
+        maxCount: number;
+        weightBySize?: boolean;
+    }) => Promise<SummaryEntry<AccessSummaryState>[]>;
     getIndexInfo: (account: string, bucketName: string) => Promise<{
         fileCount: number;
         byteCount: number;

package/storage/remoteStorage/storageController.ts

@@ -15,6 +15,8 @@ import {
     getActiveBucket, activateBucket, ActiveBucketInfo, getActiveBucketKeys,
 } from "./storageServerState";
 import { StorageClientController } from "./storageClientController";
+import { trackAccess, getAccessTotals, readAccessSummaries, clearAccountAccessStats, AccessTotals, AccessSummaryState } from "./accessStats";
+import type { SummaryEntry } from "../../treeSummary";
 
 export const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
@@ -292,37 +294,57 @@ class RemoteStorageControllerBase {
     async get2(account: string, bucketName: string, path: string, range?: { start: number; end: number }, internal?: boolean): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
         assertValidPath(path);
         let bucket = await getBucket(account, bucketName);
-        if (!bucket) return undefined;
-        if (internal && bucket.store.getInternal2) {
-            return await bucket.store.getInternal2(path, { range });
+        let result: { data: Buffer; writeTime: number; size: number } | undefined;
+        if (bucket) {
+            if (internal && bucket.store.getInternal2) {
+                result = await bucket.store.getInternal2(path, { range });
+            } else {
+                result = await bucket.store.get2(path, { range });
+            }
         }
-        return await bucket.store.get2(path, { range });
+        trackAccess({ account, operation: "get", path: `${bucketName}/${path}`, size: result && result.data.length || 0 });
+        return result;
     }
     async set(account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean, internal?: boolean): Promise<void> {
         assertValidName(bucketName, "bucket name");
         assertValidPath(path);
+        trackAccess({ account, operation: "set", path: `${bucketName}/${path}`, size: data.length });
         await writeBucketFile(account, bucketName, path, Buffer.from(data), { lastModified, forceSetImmutable, internal });
     }
     async del(account: string, bucketName: string, path: string): Promise<void> {
         assertValidName(bucketName, "bucket name");
         assertValidPath(path);
+        trackAccess({ account, operation: "del", path: `${bucketName}/${path}` });
         await deleteBucketFile(account, bucketName, path);
     }
     async getInfo(account: string, bucketName: string, path: string, includeTombstones?: boolean): Promise<{ writeTime: number; size: number } | undefined> {
         assertValidPath(path);
+        trackAccess({ account, operation: "getInfo", path: `${bucketName}/${path}` });
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return undefined;
         return await bucket.store.getInfo(path, { includeTombstones });
     }
     async findInfo(account: string, bucketName: string, prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
         let bucket = await getBucket(account, bucketName);
-        if (!bucket) return [];
-        return await bucket.store.findInfo(prefix, config);
+        let results = bucket && await bucket.store.findInfo(prefix, config) || [];
+        // The paths are only known once the results are in; an empty result still counts as one access at the prefix.
+        if (results.length) {
+            for (let info of results) {
+                trackAccess({ account, operation: "findInfo", path: `${bucketName}/${info.path}` });
+            }
+        } else {
+            trackAccess({ account, operation: "findInfo", path: `${bucketName}/${prefix}` });
+        }
+        return results;
     }
     async getChangesAfter2(account: string, bucketName: string, config: ChangesAfterConfig): Promise<ArchiveFileInfo[]> {
         let bucket = await getBucket(account, bucketName);
         if (!bucket) return [];
-        return await bucket.store.getChangesAfter2(config);
+        let results = await bucket.store.getChangesAfter2(config);
+        for (let info of results) {
+            trackAccess({ account, operation: "getChangesAfter", path: `${bucketName}/${info.path}` });
+        }
+        return results;
     }
     async getArchivesConfig(account: string, bucketName: string): Promise<ArchivesConfig> {
         let bucket = await getBucket(account, bucketName);
@@ -348,10 +370,21 @@ class RemoteStorageControllerBase {
         assertValidName(bucketName, "bucket name");
         return await activateBucket(account, bucketName);
     }
-    /** Zeroes the write statistics listBuckets reports, for every bucket in the account. */
+    /** Zeroes the write statistics listBuckets reports and the in-memory access statistics, for every bucket in the account. */
     async clearWriteStats(account: string): Promise<{ clearedBuckets: number }> {
         assertValidName(account, "account");
-        return { clearedBuckets: await clearAccountWriteStats(account) };
+        clearAccountAccessStats(account);
+        return { clearedBuckets: clearAccountWriteStats(account) };
+    }
+    /** In-memory totals per operation type since startup (or the last clearWriteStats). */
+    async getAccessStats(account: string): Promise<AccessTotals> {
+        assertValidName(account, "account");
+        return getAccessTotals(account);
+    }
+    /** A path breakdown of one operation's accesses (operation names come from getAccessStats). maxCount is passed straight to TreeSummary.getSummaries. weightBySize is ignored for count-only operations, which return their count breakdown. */
+    async getAccessSummaries(account: string, config: { operation: string; maxCount: number; weightBySize?: boolean }): Promise<SummaryEntry<AccessSummaryState>[]> {
+        assertValidName(account, "account");
+        return readAccessSummaries({ account, ...config });
     }
     async getIndexInfo(account: string, bucketName: string): Promise<{ fileCount: number; byteCount: number; sources: { debugName: string; fileCount: number; byteCount: number }[] } | undefined> {
         let bucket = await getBucket(account, bucketName);
@@ -383,6 +416,7 @@ class RemoteStorageControllerBase {
         assertWritesAllowed();
         let info = largeUploadInfo.get(uploadId);
         if (!info) throw new Error(`Unknown large upload ${uploadId}`);
+        trackAccess({ account: info.account, operation: "uploadPart", path: `${info.bucketName}/${info.path}`, size: data.length });
         let bucket = await getBucket(info.account, info.bucketName);
         if (!bucket) throw new Error(`Bucket ${info.account}/${info.bucketName} no longer exists`);
         await bucket.store.appendLargeUpload(uploadId, Buffer.from(data));
@@ -437,6 +471,7 @@ class RemoteStorageControllerBase {
         }
         let info = await bucket.store.getInfo(filePath);
         if (!info || !info.size) {
+            trackAccess({ account, operation: "httpGet", path: `${bucketName}/${filePath}`, size: 0 });
             return setHTTPResultHeaders(Buffer.from(""), { status: "404" });
         }
         let ext = filePath.split(".").pop() || "";
@@ -477,6 +512,7 @@ class RemoteStorageControllerBase {
             }
         }
         let result = await bucket.store.get2(filePath, { range });
+        trackAccess({ account, operation: "httpGet", path: `${bucketName}/${filePath}`, size: result && result.data.length || 0 });
         if (!result) {
             return setHTTPResultHeaders(Buffer.from(""), { status: "404" });
         }
@@ -532,6 +568,8 @@ export const RemoteStorageController = SocketFunction.register(
         getActiveBucket: { hooks: [accountAccess] },
         activateBucket: { hooks: [accountAccess] },
         clearWriteStats: { hooks: [accountAccess] },
+        getAccessStats: { hooks: [accountAccess] },
+        getAccessSummaries: { hooks: [accountAccess] },
         getSyncStatus: { hooks: [accountAccess] },
         startLargeFile: { hooks: [accountAccess] },
         uploadPart: { hooks: [uploadAccess] },

package/storage/remoteStorage/storageServerState.d.ts

@@ -28,8 +28,8 @@ export type BucketWriteStats = {
     flushedWrites: number;
     flushedBytes: number;
 };
-/** Zeroes the write statistics of every bucket in the account, including counts not yet flushed. */
-export declare function clearAccountWriteStats(account: string): Promise<number>;
+/** Zeroes the write statistics of every bucket in the account. */
+export declare function clearAccountWriteStats(account: string): number;
 export declare function setTrustedMachines(config: {
     account: string;
     machineIds: string[];

package/storage/remoteStorage/storageServerState.ts

@@ -98,78 +98,36 @@ export type BucketWriteStats = {
 function emptyWriteStats(): BucketWriteStats {
     return { originalWrites: 0, originalBytes: 0, flushedWrites: 0, flushedBytes: 0 };
 }
-function addWriteStats(a: BucketWriteStats, b: BucketWriteStats): BucketWriteStats {
-    return {
-        originalWrites: a.originalWrites + b.originalWrites,
-        originalBytes: a.originalBytes + b.originalBytes,
-        flushedWrites: a.flushedWrites + b.flushedWrites,
-        flushedBytes: a.flushedBytes + b.flushedBytes,
-    };
-}
-function getWriteStatsStorage(): Promise<IStorage<BucketWriteStats>> {
-    return getSystemStorage<BucketWriteStats>("writeStats");
-}
 
-// Counted in memory as deltas since the last flush, so counting a write never touches the disk. The flush reads the stored totals and adds the delta, which also makes it correct across restarts.
-const writeStatDeltas = new Map<string, BucketWriteStats>();
-
-async function flushWriteStats(): Promise<void> {
-    if (!writeStatDeltas.size) return;
-    let pending = [...writeStatDeltas];
-    writeStatDeltas.clear();
-    let storage = await getWriteStatsStorage();
-    for (let [key, delta] of pending) {
-        try {
-            await storage.set(key, addWriteStats(await storage.get(key) || emptyWriteStats(), delta));
-        } catch (e) {
-            // Put the delta back, so a failed flush loses nothing
-            writeStatDeltas.set(key, addWriteStats(writeStatDeltas.get(key) || emptyWriteStats(), delta));
-            console.error(`Flushing write stats for bucket ${key} failed: ${(e as Error).stack ?? e}`);
-        }
-    }
-}
-
-const startWriteStatsFlushing = lazy(() => {
-    runInfinitePoll(WRITE_STATS_FLUSH_INTERVAL, flushWriteStats);
-});
+// In memory only: totals since this process started (or the last clearWriteStats). Persisting them to disk was more machinery than the numbers were worth.
+const writeStats = new Map<string, BucketWriteStats>();
 
 function countBucketWrite(key: string, kind: "original" | "flushed", bytes: number): void {
-    let delta = writeStatDeltas.get(key);
-    if (!delta) {
-        delta = emptyWriteStats();
-        writeStatDeltas.set(key, delta);
+    let stats = writeStats.get(key);
+    if (!stats) {
+        stats = emptyWriteStats();
+        writeStats.set(key, stats);
     }
     if (kind === "original") {
-        delta.originalWrites++;
-        delta.originalBytes += bytes;
+        stats.originalWrites++;
+        stats.originalBytes += bytes;
     } else {
-        delta.flushedWrites++;
-        delta.flushedBytes += bytes;
+        stats.flushedWrites++;
+        stats.flushedBytes += bytes;
     }
-    startWriteStatsFlushing();
 }
 
-async function getBucketWriteStats(key: string): Promise<BucketWriteStats> {
-    let storage = await getWriteStatsStorage();
-    let stored = await storage.get(key) || emptyWriteStats();
-    let delta = writeStatDeltas.get(key);
-    return delta && addWriteStats(stored, delta) || stored;
+function getBucketWriteStats(key: string): BucketWriteStats {
+    return writeStats.get(key) || emptyWriteStats();
 }
 
-/** Zeroes the write statistics of every bucket in the account, including counts not yet flushed. */
-export async function clearAccountWriteStats(account: string): Promise<number> {
-    let storage = await getWriteStatsStorage();
+/** Zeroes the write statistics of every bucket in the account. */
+export function clearAccountWriteStats(account: string): number {
     let prefix = `${account}/`;
     let cleared = 0;
-    for (let key of await storage.getKeys()) {
-        if (!key.startsWith(prefix)) continue;
-        writeStatDeltas.delete(key);
-        await storage.remove(key);
-        cleared++;
-    }
-    for (let key of [...writeStatDeltas.keys()]) {
+    for (let key of [...writeStats.keys()]) {
         if (!key.startsWith(prefix)) continue;
-        writeStatDeltas.delete(key);
+        writeStats.delete(key);
         cleared++;
     }
     console.log(`Cleared the write statistics of ${cleared} buckets in account ${account}`);
@@ -211,7 +169,6 @@ const buckets = new Map<string, Promise<LoadedBucket | undefined>>();
 
 const MAX_REBUILD_TIMER_DELAY = 2 ** 31 - 1;
 const REBUILD_BOUNDARY_BUFFER = 1000;
-const WRITE_STATS_FLUSH_INTERVAL = 5 * 60 * 1000;
 
 function getBucketFolder(account: string, bucketName: string): string {
     return path.join(getStorageServerConfig().folder, "buckets2", account, bucketName);
@@ -1070,15 +1027,11 @@ export async function listAccountBuckets(account: string): Promise<ServerBucketI
             let key = `${account}/${bucketName}`;
             let folder = getBucketFolder(account, bucketName);
             let base: ServerBucketInfo = { bucketName, active: false, folder };
-            let [disk, writeStats] = await Promise.all([
-                timed("statfs", () => getDiskInfo(folder)).catch((e: Error) => {
-                    base.diskError = String(e.stack ?? e).slice(0, 500);
-                    return undefined;
-                }),
-                timed("writeStats", () => getBucketWriteStats(key)),
-            ]);
-            base.disk = disk;
-            base.writeStats = writeStats;
+            base.disk = await timed("statfs", () => getDiskInfo(folder)).catch((e: Error) => {
+                base.diskError = String(e.stack ?? e).slice(0, 500);
+                return undefined;
+            });
+            base.writeStats = getBucketWriteStats(key);
             const loadedPromise = buckets.get(key);
             if (loadedPromise) {
                 try {

package/test.ts

@@ -57,7 +57,7 @@ function summarize(files: FileInfo[], weightBySize: boolean) {
             if (weightBySize) return summary.totalSize;
             return summary.count;
         },
-        expectedOutputCount: SUMMARY_COUNT,
+        expectedOutputCount: 100,
     });
     let addStart = performance.now();
     for (let file of files) {

package/treeSummary.d.ts

@@ -0,0 +1,31 @@
+export type SummaryEntry<S> = {
+    path: string;
+    kind: "self" | "subtree" | "group" | "truncated";
+    summary: S;
+    weight: number;
+};
+export declare class TreeSummary<T, S> {
+    private config;
+    private root;
+    private nodeCount;
+    private maxTrackedNodes;
+    constructor(config: {
+        getPath: (value: T) => string;
+        createSummary: () => S;
+        addToSummary: (value: T, summary: S) => void;
+        mergeSummaries: (target: S, source: S) => void;
+        getWeight: (summary: S) => number;
+        expectedOutputCount?: number;
+    });
+    add(value: T): void;
+    private splitNode;
+    getTrackedNodeCount(): number;
+    getSummaries(maxCount: number): SummaryEntry<S>[];
+    private isRefinable;
+    private refine;
+    private splitParts;
+    private entryForParts;
+    private subtreeEntry;
+    private buildOutputSummary;
+    private prune;
+}