querysub 0.588.0 → 0.590.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.588.0",
+  "version": "0.590.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -71,8 +71,8 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.59",
-    "socket-function": "^1.2.26",
+    "sliftutils": "^1.7.65",
+    "socket-function": "^1.2.27",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-a-archives/archiveCache.ts

@@ -48,13 +48,15 @@ const getCacheDir = lazy(() => {
 const TEMP_SUFFIX = ".tmp";
 const TEMP_THRESHOLD = timeInHour * 3;
 
-function getTempFilePath() {
+export function getTempFilePath() {
     return os.tmpdir() + "/" + nextId() + TEMP_SUFFIX;
 }
 
-const LARGE_FILE_CHUNK = 1024 * 1024 * 32;
+export const LARGE_FILE_CHUNK = 1024 * 1024 * 32;
 const CACHE_SUFFIX = ".cache";
-export function getArchiveCachePath(archives: Archives, key: string): string {
+/** The cache only needs the debug name (for cache-path hashing), so any archives flavor (old Archives or IArchives) works. */
+export type CacheKeySource = { getDebugName(): string };
+export function getArchiveCachePath(archives: CacheKeySource, key: string): string {
     // Remove all "/backblaze/" values from the path, as they make equivalent paths
     //  hash differently, which we don't want.
     let fullFileName = (archives.getDebugName() + "/" + key).replaceAll("//", "/").replaceAll("backblaze/", "/");
@@ -134,7 +136,7 @@ const getDiskMetricsBase = async () => {
     }
 
     // We require our input to be a temp file, so our update can be atomic
-    async function addCacheFile(archives: Archives, key: string, sourceTempFile: string): Promise<void> {
+    async function addCacheFile(archives: CacheKeySource, key: string, sourceTempFile: string): Promise<void> {
         let path = getArchiveCachePath(archives, key);
         let hasFile = fileSizes.some(x => x.path === path);
         if (hasFile) {
@@ -187,7 +189,7 @@ const getDiskMetricsBase = async () => {
         }
     }
     async function getCacheFile(
-        archives: Archives,
+        archives: CacheKeySource,
         key: string,
         config?: { range?: { start: number; end: number } }
     ): Promise<Buffer | undefined> {
@@ -215,7 +217,7 @@ const getDiskMetricsBase = async () => {
         }
         return buffer;
     }
-    async function delCacheFile(archives: Archives, key: string): Promise<void> {
+    async function delCacheFile(archives: CacheKeySource, key: string): Promise<void> {
         let path = getArchiveCachePath(archives, key);
         if (removeFile(path)) {
             try {
@@ -235,7 +237,7 @@ const getDiskMetricsBase = async () => {
 };
 let curSeqNum = 1;
 const getDiskMetricSeqNum = cache((seqNum: number) => getDiskMetricsBase());
-async function getDiskMetrics() {
+export async function getDiskMetrics() {
     return getDiskMetricSeqNum(curSeqNum);
 }
 

package/src/-a-archives/archiveCache2.ts

@@ -0,0 +1,153 @@
+import fs from "fs";
+import { GetConfig, IArchives, SetConfig } from "sliftutils/storage/IArchives";
+import { getArchiveCachePath, getDiskMetrics, getTempFilePath, LARGE_FILE_CHUNK } from "./archiveCache";
+
+let cacheArchives2Symbol = Symbol("cacheArchives2");
+
+/** IMPORTANT! The cache assumes file contents are immutable: files are only created and deleted, never mutated in place.
+ *      - Shares the disk cache pool (directory, size limits, eviction) with the old wrapArchivesWithCache.
+ */
+export function wrapArchivesWithCache2(archives: IArchives, rootConfig?: {
+    /** Files are never deleted either, so we can serve cache hits (and even getInfo) without checking the source. */
+    immutable?: boolean;
+}): IArchives & {
+    debugGetPath(key: string): string;
+} {
+    if (cacheArchives2Symbol in archives) {
+        return archives as any;
+    }
+
+    async function cacheBuffer(fileName: string, data: Buffer) {
+        let metrics = await getDiskMetrics();
+        const tempPath = getTempFilePath();
+        await fs.promises.writeFile(tempPath, data);
+        await metrics.addCacheFile(archives, fileName, tempPath);
+    }
+
+    async function setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined>; }) {
+        const tempPath = getTempFilePath();
+        let handle: fs.promises.FileHandle | undefined;
+        try {
+            handle = await fs.promises.open(tempPath, "w");
+            let pos = 0;
+            while (true) {
+                let data = await config.getNextData();
+                if (!data?.length) break;
+                await handle.write(data, 0, data.length, pos);
+                pos += data.length;
+            }
+        } finally {
+            if (handle) {
+                try {
+                    await handle.close();
+                } catch { }
+            }
+        }
+
+        let metrics = await getDiskMetrics();
+        await metrics.addCacheFile(archives, config.path, tempPath);
+        let cachePath = getArchiveCachePath(archives, config.path);
+
+        let pos = 0;
+        let cacheHandle: fs.promises.FileHandle = await fs.promises.open(cachePath, "r");
+        let data = Buffer.alloc(LARGE_FILE_CHUNK);
+        async function getNextData(): Promise<Buffer | undefined> {
+            try {
+                let read = await cacheHandle.read(data, 0, LARGE_FILE_CHUNK, pos);
+                if (read.bytesRead === 0) return undefined;
+                let curData = data;
+                if (read.bytesRead < LARGE_FILE_CHUNK) {
+                    curData = curData.slice(0, read.bytesRead);
+                }
+                pos += read.bytesRead;
+                return curData;
+            } catch {
+                return undefined;
+            }
+        }
+        try {
+            await archives.setLargeFile({
+                ...config,
+                getNextData,
+            });
+        } finally {
+            await cacheHandle.close();
+        }
+    }
+
+    return {
+        [cacheArchives2Symbol]: true,
+        debugGetPath: (key: string) => getArchiveCachePath(archives, key),
+
+        getDebugName: () => archives.getDebugName(),
+        hasWriteAccess: () => archives.hasWriteAccess(),
+        getConfig: () => archives.getConfig(),
+        getURL: (path: string) => archives.getURL(path),
+        find: (prefix, config) => archives.find(prefix, config),
+        findInfo: (prefix, config) => archives.findInfo(prefix, config),
+        getChangesAfter2: (config) => archives.getChangesAfter2(config),
+        getSyncStatus: archives.getSyncStatus && (() => archives.getSyncStatus!()),
+
+        get: async (fileName: string, config?: GetConfig) => {
+            let metrics = await getDiskMetrics();
+            // Get info to at least see if it is removed or not. This makes sure we don't get in
+            //  a bad state, at least for immutable files (for mutable files we don't have an easy
+            //  way to check)
+            if (!rootConfig?.immutable) {
+                let info = await archives.getInfo(fileName);
+                if (!info) {
+                    // If it is gone remotely, remove it from the cache, to save space.
+                    await metrics.delCacheFile(archives, fileName);
+                    return undefined;
+                }
+            }
+            let buffer = await metrics.getCacheFile(archives, fileName, config);
+            if (buffer) return buffer;
+            let result = await archives.get(fileName);
+            if (result) {
+                await cacheBuffer(fileName, result);
+                let range = config?.range;
+                if (range) {
+                    result = result.slice(range.start, range.end);
+                }
+            }
+            return result;
+        },
+        get2: async (fileName: string, config?: GetConfig) => {
+            let result = await archives.get2(fileName, config);
+            // Range reads are partial, so they can't populate the cache.
+            if (result && !config?.range) {
+                await cacheBuffer(fileName, result.data);
+            }
+            return result;
+        },
+        getInfo: async (fileName: string) => {
+            // When the archives are immutable, the local cache file is a byte-for-byte copy, so we can
+            //  answer getInfo from a local stat and skip the (slow, network) getInfo on the underlying archives.
+            if (rootConfig?.immutable) {
+                try {
+                    let stat = await fs.promises.stat(getArchiveCachePath(archives, fileName));
+                    return { writeTime: stat.mtimeMs, size: stat.size };
+                } catch {
+                    // Not in our cache yet — fall through to the source.
+                }
+            }
+            return archives.getInfo(fileName);
+        },
+        set: async (fileName: string, data: Buffer, config?: SetConfig) => {
+            let writtenKey = await archives.set(fileName, data, config);
+            // VARIABLE_SHARD keys materialize into a different key on write, in which case
+            //  fileName isn't the readable key, so caching under it would be wrong.
+            if (writtenKey === fileName) {
+                await cacheBuffer(fileName, data);
+            }
+            return writtenKey;
+        },
+        setLargeFile,
+        del: async (fileName: string) => {
+            let metrics = await getDiskMetrics();
+            await metrics.delCacheFile(archives, fileName);
+            await archives.del(fileName);
+        },
+    };
+}

package/src/-a-archives/archives.ts

@@ -8,6 +8,7 @@ import debugbreak from "debugbreak";
 import { isClient } from "../config2";
 import { wrapArchivesWithCache } from "./archiveCache";
 import { Args } from "socket-function/src/types";
+import { IArchives } from "sliftutils/storage/IArchives";
 
 export interface Archives {
     getDebugName(): string;
@@ -104,8 +105,8 @@ export function createArchivesOverride<T extends Partial<Archives>>(
     };
 }
 
-export function nestArchives(path: string, archives: Archives): Archives {
-    if (!path) return archives;
+export function nestArchives(path: string, archives: Archives): Archives & IArchives {
+    if (!path) return archives as any;
     if (!path.endsWith("/")) {
         path = path + "/";
     }
@@ -115,6 +116,7 @@ export function nestArchives(path: string, archives: Archives): Archives {
         }
         return file;
     }
+    // NOTE: Just ignore the errors in here. We're going to be getting rid of this soon enough. 
     return {
         getDebugName: () => archives.getDebugName() + "/" + path,
         get: (fileName: string, config) => archives.get(path + stripFilePrefix(fileName), config),

package/src/-a-archives/archives2.ts

@@ -1,4 +1,4 @@
-import { IArchives, RemoteConfigBase } from "sliftutils/storage/IArchives";
+import { ChangesAfterConfig, GetConfig, IArchives, RemoteConfigBase, SetConfig } from "sliftutils/storage/IArchives";
 import { createArchives } from "sliftutils/storage/remoteStorage/createArchives";
 import { cache, cacheJSONArgsEqual } from "socket-function/src/caching";
 import { formatDateTime } from "socket-function/src/formatting/format";
@@ -39,8 +39,10 @@ function createSourceWindows(
 }
 
 function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
+    // REMEMBER TO INCREMENT VERSION WHEN YOU MAKE CHANGES!
+    // REMEMBER TO RERUN initArchives2 if you change this (It's probably something you should call in your entry point.
     const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
-    const HETZNER = `https://65-109-93-113.querysubtest.com:5233/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
+    const HETZNER = `https://65-109-93-113.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const BACKBLAZE = `https://f002.backblazeb2.com/file/${bucket}/storage/storagerouting.json`;
     let sources = createSourceWindows(overrides, [
         {
@@ -66,7 +68,7 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     ]);
 
     return createArchives({
-        version: 15,
+        version: 16,
         sources,
     });
 }
@@ -79,17 +81,18 @@ const archivesBuilderCache = cacheJSONArgsEqual((domain: string, overrides: Part
 });
 function nestBucket(folder: string, archives: ReturnType<typeof createArchives>) {
     if (!folder) return archives;
+    folder = folder.replaceAll("\\", "/");
     if (!folder.endsWith("/")) {
         folder = folder + "/";
     }
     return {
-        get: (path: string) => archives.get(folder + path),
-        set: (path: string, data: Buffer) => archives.set(folder + path, data),
+        get: (path: string, config?: GetConfig) => archives.get(folder + path, config),
+        set: (path: string, data: Buffer, config?: SetConfig) => archives.set(folder + path, data, config),
         del: (path: string) => archives.del(folder + path),
-        find: (prefix: string) => archives.find(folder + prefix),
-        findInfo: (prefix: string) => archives.findInfo(folder + prefix),
-        get2: (path: string) => archives.get2(folder + path),
-        setLargeFile: (config: { path: string; getNextData(): Promise<Buffer | undefined>; }) => archives.setLargeFile({ path: folder + config.path, getNextData: config.getNextData }),
+        find: async (prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }) => (await archives.find(folder + prefix, config)).map(x => x.slice(folder.length)),
+        findInfo: async (prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }) => (await archives.findInfo(folder + prefix, config)).map(x => ({ ...x, path: x.path.slice(folder.length) })),
+        get2: (path: string, config?: GetConfig) => archives.get2(folder + path, config),
+        setLargeFile: (config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined>; }) => archives.setLargeFile({ ...config, path: folder + config.path }),
         getInfo: (path: string) => archives.getInfo(folder + path),
         getDebugName: () => archives.getDebugName() + "/" + folder,
         hasWriteAccess: () => archives.hasWriteAccess(),
@@ -101,6 +104,19 @@ function nestBucket(folder: string, archives: ReturnType<typeof createArchives>)
             return (path: string) => getUrls(folder + path);
         },
         getShardKey: async (path: string) => (await archives.getShardKey(folder + path)).slice(folder.length),
+        // NOTE: It's recommended that you only store the keys and don't store the URLs. If you absolutely need a URL, it's best to call it lazily.
+        getGetFastURLs: async () => {
+            let getFastUrls = await archives.getGetFastURLs();
+            return (path: string) => getFastUrls(folder + path);
+        },
+        // getFast has less latency than get. Writes won't be available immediately after sets however. If it's a buffered archives the delay will be bufferDelay, ex 5 minutes, instead of a few seconds.
+        getFast: async (path: string, config?: GetConfig) => {
+            return await archives.getFast(folder + path, config);
+        },
+        getChangesAfter2: async (config: ChangesAfterConfig) => {
+            let changes = await archives.getChangesAfter2(config);
+            return changes.filter(x => x.path.startsWith(folder)).map(x => ({ ...x, path: x.path.slice(folder.length) }));
+        },
     };
 }
 

package/src/-a-archives/archivesJSONT.ts

@@ -1,6 +1,7 @@
 import { lazy } from "socket-function/src/caching";
 import { Archives } from "./archives";
 import { green } from "socket-function/src/formatting/logColors";
+import { IArchives } from "sliftutils/storage/IArchives";
 
 export type ArchiveT<T> = {
     get(key: string): Promise<T | undefined>;
@@ -12,7 +13,7 @@ export type ArchiveT<T> = {
     [Symbol.asyncIterator](): AsyncIterator<[string, T]>;
 };
 
-export function archiveJSONT<T>(archives: () => Archives): ArchiveT<T> {
+export function archiveJSONT<T>(archives: () => IArchives): ArchiveT<T> {
     archives = lazy(archives);
 
     let valuesCache = new Map<string, {
@@ -60,18 +61,28 @@ export function archiveJSONT<T>(archives: () => Archives): ArchiveT<T> {
 
         if (needsUpdate) {
             let maxRetries = 10;
+            let updated = false;
             for (let attempt = 0; attempt < maxRetries; attempt++) {
                 infos = await archives().findInfo("");
 
-                valuesCache.clear();
+                let newCache = new Map<string, {
+                    createTime: number;
+                    size: number;
+                    value: Buffer;
+                }>();
                 let allFound = true;
 
                 await Promise.all(infos.map(async info => {
+                    let cached = valuesCache.get(info.path);
+                    if (cached && cached.createTime === info.createTime && cached.size === info.size) {
+                        newCache.set(info.path, cached);
+                        return;
+                    }
                     let buffer = await archives().get(info.path);
                     if (!buffer) {
                         allFound = false;
                     } else {
-                        valuesCache.set(info.path, {
+                        newCache.set(info.path, {
                             createTime: info.createTime,
                             size: info.size,
                             value: buffer
@@ -94,10 +105,14 @@ export function archiveJSONT<T>(archives: () => Archives): ArchiveT<T> {
                         }
                     }
                 }
-                if (!anyChanged()) break;
+                if (!anyChanged()) {
+                    valuesCache = newCache;
+                    updated = true;
+                    break;
+                }
             }
 
-            if (needsUpdate && valuesCache.size === 0) {
+            if (!updated) {
                 throw new Error(`Failed to get consistent snapshot of values after ${maxRetries} attempts`);
             }
         }

package/src/-g-core-values/NodeCapabilities.ts

@@ -25,6 +25,7 @@ import { PromiseObj } from "../promise";
 import { getTrueTimeOffset } from "socket-function/time/trueTimeShim";
 import { getGitRefLive } from "../4-deploy/git";
 import { getScheduledShutdownTime } from "./scheduledShutdown";
+import type { MeasureChunk, MeasureSummary } from "../diagnostics/watchdog";
 setImmediate(() => {
     import("../diagnostics/MachineThreadInfo");
 });
@@ -164,6 +165,16 @@ class NodeCapabilitiesControllerBase {
         return process.memoryUsage();
     }
 
+    public async getMeasureSummary(): Promise<MeasureSummary | undefined> {
+        let { getLastMeasureSummary } = await import("../diagnostics/watchdog");
+        return getLastMeasureSummary();
+    }
+
+    public async getMeasureBreakdown(range: { startTime: number; endTime: number }): Promise<MeasureChunk | undefined> {
+        let { getMeasureBreakdown } = await import("../diagnostics/watchdog");
+        return getMeasureBreakdown(range);
+    }
+
     public async getInspectURL() {
         return await getDebuggerUrl();
     }
@@ -204,6 +215,8 @@ export const NodeCapabilitiesController = SocketFunction.register(
     () => ({
         getMetadata: {},
         getMemoryUsage: {},
+        getMeasureSummary: {},
+        getMeasureBreakdown: {},
         getInspectURL: { hooks: [requiresNetworkTrustHook] },
         exposeExternalDebugPortOnce: { hooks: [requiresNetworkTrustHook] },
     }),

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -4,7 +4,7 @@ import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { getOwnNodeId, getOwnNodeIdAssert } from "../../-f-node-discovery/NodeDiscovery";
 import { pathValueArchives } from "../pathValueArchives";
 import { ArchiveLocker, ArchiveTransaction } from "./ArchiveLocks";
-import { Archives } from "../../-a-archives/archives";
+import { copyArchiveFile, IArchives } from "sliftutils/storage/IArchives";
 import debugbreak from "debugbreak";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
@@ -33,13 +33,20 @@ const MAX_APPLY_TRIES = 10;
 const CONCURRENT_READ_COUNT = 32;
 const CONCURRENT_WRITE_COUNT = 16;
 
+/** Moves by copy-then-delete (IArchives has no move). copyArchiveFile stamps the target with the source's write time, so ordering-by-writeTime survives the move. A missing source means another process already moved it, which is fine. */
+async function moveArchiveFile(from: IArchives, to: IArchives, path: string): Promise<void> {
+    let copied = await copyArchiveFile({ from, to, path });
+    if (!copied) return;
+    await from.del(path);
+}
+
 export function createArchiveLocker2(config: {
-    archiveValues: Archives;
+    archiveValues: IArchives;
     /** IMPORTANT! This must be the same independent of archiveValues, otherwise we won't
      *      lock moves between directories.
      */
-    archiveLocks: Archives;
-    archiveRecycleBin: Archives;
+    archiveLocks: IArchives;
+    archiveRecycleBin: IArchives;
 }): ArchiveLocker {
     let { archiveValues, archiveLocks, archiveRecycleBin } = config;
     function getArchives(key: string) {
@@ -85,11 +92,7 @@ export function createArchiveLocker2(config: {
                 return;
             }
             try {
-                await archives.move({
-                    path: key,
-                    targetPath: key,
-                    target: archiveRecycleBin,
-                });
+                await moveArchiveFile(archives, archiveRecycleBin, key);
                 logNodeStats(`archives|Deleted TΔ`, formatNumber, 1);
             } catch {
                 // It was probably just moved by another process
@@ -126,11 +129,7 @@ export function createArchiveLocker2(config: {
                 if (correctFiles.has(file)) return;
                 let info = await archiveValues.getInfo(file);
                 console.log(`Moving file ${file}, ${moveCount}/${valuePaths.size}, ${formatNumber(info?.size || 0)} bytes`);
-                await archiveValues.move({
-                    path: file,
-                    targetPath: file,
-                    target: archiveRecycleBin,
-                });
+                await moveArchiveFile(archiveValues, archiveRecycleBin, file);
                 console.log(`Moved file ${file}, ${moveCount}/${valuePaths.size}`);
             }
             let moveFileParallel = runInParallel({ parallelCount: 8 }, moveFile);
@@ -151,11 +150,7 @@ export function createArchiveLocker2(config: {
             let isInRecycleBin = !!(await archiveRecycleBin.getInfo(file));
             if (isInRecycleBin) {
                 console.log(`Restoring ${file}, ${restoreCount++}/${files.length}`);
-                await archiveRecycleBin.move({
-                    path: file,
-                    targetPath: file,
-                    target: archiveValues,
-                });
+                await moveArchiveFile(archiveRecycleBin, archiveValues, file);
             } else {
                 console.log(`File ${file} is not in the recycle bin, skipping, ${restoreCount++}/${files.length}`);
             }
@@ -312,6 +307,36 @@ type StorageType = {
      *          all writes should now be readable.
      */
 };
+function createCheckpointTimer() {
+    let last = Date.now();
+    let spans = new Map<string, { time: number; count: number }>();
+    return {
+        start() {
+            last = Date.now();
+            spans.clear();
+        },
+        /** Attributes the time since the previous checkpoint (or start) to name, accumulating across repeat visits. */
+        checkpoint(name: string) {
+            let now = Date.now();
+            let span = spans.get(name);
+            if (!span) {
+                span = { time: 0, count: 0 };
+                spans.set(name, span);
+            }
+            span.time += now - last;
+            span.count++;
+            last = now;
+        },
+        logAndFlush(prefix: string) {
+            let total = Array.from(spans.values()).reduce((sum, span) => sum + span.time, 0);
+            let parts = Array.from(spans.entries()).map(([name, span]) => `${name}=${formatTime(span.time)}${span.count > 1 ? ` (x${span.count})` : ""}`);
+            console.log(blue(`${prefix} time breakdown, total ${formatTime(total)}: ${parts.join(" | ")}`));
+            spans.clear();
+            last = Date.now();
+        },
+    };
+}
+
 export type Transaction = {
     ops: ({
         type: "create";
@@ -355,6 +380,8 @@ class TransactionLocker {
         private isTransactionValidOverride?: (transaction: Transaction, dataFiles: FileInfo[], rawDataFiles: FileInfo[]) => boolean
     ) { }
 
+    private perf = createCheckpointTimer();
+
     // #region Base File Ops
     public getConfirmKey(key: string): string {
         let { dir, name } = parsePath(key);
@@ -444,6 +471,7 @@ class TransactionLocker {
         const tryToRead = async () => {
             let time = Date.now();
             let files = await this.storage.getKeys();
+            this.perf.checkpoint("getKeys");
             if (this.lastFilesRead && this.lastFilesReadTime) {
                 let prevFiles = new Set(this.lastFilesRead.map(a => a.file));
                 let suspiciousThreshold = this.lastFilesReadTime - ARCHIVE_PROPAGATION_TIME * 10;
@@ -491,6 +519,7 @@ class TransactionLocker {
                     source: tFile,
                 });
             }
+            this.perf.checkpoint("read transaction files");
 
             // Check all of our files to see if any have changed (or if there a new files).
             //  Because we don't reuse file names, or change them, it means if a file exists
@@ -501,6 +530,7 @@ class TransactionLocker {
             //      (it pretty much only depends on files existing).
             {
                 let filesVerify = await this.storage.getKeys();
+                this.perf.checkpoint("verify getKeys");
                 let filesVerifySet = new Set(filesVerify.map(a => a.file));
                 // If it changes while reading, read again. Otherwise, if there were no changes while reading,
                 //  we know the state was all the state we read, at one time (maybe not now, but at an instant
@@ -563,6 +593,7 @@ class TransactionLocker {
                 //rawFiles: files.map(a => a.file),
                 //confirmedFiles: Array.from(currentDataFiles.values()).map(a => a.file),
             });
+            this.perf.checkpoint("categorize files");
 
             return {
                 rawDataFiles: files,
@@ -672,8 +703,10 @@ class TransactionLocker {
      *      - Might run a transaction
      */
     public async getFiles(): Promise<FileInfo[]> {
+        this.perf.start();
         let obj = await this.getFilesBase();
         logNodeStateStats(`ArchiveLock Data File`, formatNumber, obj.dataFiles.length);
+        this.perf.logAndFlush(`getFiles (${this.storage.debugKey})`);
         return obj.dataFiles;
     }
     private transactionAppliedCount = new Map<number, number>();
@@ -731,6 +764,7 @@ class TransactionLocker {
             let waitTime = threshold - Date.now();
             console.info(`Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
             await new Promise(resolve => setTimeout(resolve, waitTime));
+            this.perf.checkpoint("settle wait");
             return this.getFilesBase();
         }
 
@@ -752,6 +786,7 @@ class TransactionLocker {
                     await this.storage.deleteKey(t.source.file);
                 }
             }
+            this.perf.checkpoint("delete old transactions");
 
             // Delete any create files that are WAY too old, and not confirmed
             {
@@ -780,6 +815,7 @@ class TransactionLocker {
                         console.warn(`Almost deleted ${unconfirmedOldFiles.length} very old unconfirmed files. This is bad, did we miss their confirmations that first time? If we missed them twice in a row, we might literally delete the database, and need to enter recovery mode to fix it...`, { files: unconfirmedOldFiles });
                     }
                 }
+                this.perf.checkpoint("cleanup old unconfirmed files");
             }
 
 
@@ -805,6 +841,7 @@ class TransactionLocker {
                         }
                     }
                 }
+                this.perf.checkpoint("cleanup old confirms");
             }
 
             return {
@@ -818,6 +855,7 @@ class TransactionLocker {
             console.log(`Applying transaction ${activeT.seqNum}, cur try ${applyCount} / ${MAX_APPLY_TRIES}`);
         }
         await this.applyTransaction(activeT);
+        this.perf.checkpoint("apply transaction");
         this.transactionAppliedCount.set(activeT.seqNum, applyCount + 1);
 
         // Run again, until we can be reasonable sure activeT isn't changing. We can be wrong,