querysub 0.43.0 → 0.45.0

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -32,6 +32,8 @@ import { DEPTH_TO_DATA, MODULE_INDEX, getCurrentCall, getCurrentCallObj } from "
 import { inlineNestedCalls } from "../3-path-functions/syncSchema";
 import { interceptCalls, runCall } from "../3-path-functions/PathFunctionHelpers";
 import { deepCloneCborx } from "../misc/cloneHelpers";
+import { addStatPeriodic, addTimeProfileDistribution, onTimeProfile } from "../5-diagnostics/nodeMetadata";
+import { formatPercent } from "socket-function/src/formatting/format";
 
 // TODO: Break this into two parts:
 //  1) Run and get accesses
@@ -158,7 +160,19 @@ interface WatcherOptions<Result> {
     trackTriggers?: boolean;
 }
 
-const startTime = Date.now();
+let harvestableReadyLoopCount = 0;
+let harvestableWaitingLoopCount = 0;
+let lastZombieCount = 0;
+// NOTE: Only the % waiting for active watchers. Which... is fine
+addStatPeriodic("Watcher % Waiting", () => {
+    let totalLoop = harvestableReadyLoopCount + harvestableWaitingLoopCount;
+    if (totalLoop === 0) return 0;
+    let frac = harvestableWaitingLoopCount / totalLoop;
+    harvestableWaitingLoopCount = 0;
+    harvestableReadyLoopCount = 0;
+    return frac;
+}, formatPercent);
+addStatPeriodic("Stalled Watchers", () => lastZombieCount);
 
 //  Used to prevent local rejections on remote values (as local functions aren't rerun)
 //  Also used for security on servers, so they can read from untrusted domains, but can't have values
@@ -966,7 +980,19 @@ export class PathValueProxyWatcher {
         watcher.dispose = dispose;
         this.allWatchers.add(watcher);
 
-        const baseFunction = measureWrap(options.watchFunction, watcher.debugName);
+        let baseFunction = options.watchFunction;
+        baseFunction = measureWrap(baseFunction, watcher.debugName);
+        {
+            let base = baseFunction;
+            baseFunction = (...args: unknown[]) => {
+                let time = Date.now();
+                try {
+                    return (base as any)(...args);
+                } finally {
+                    onTimeProfile("Watcher", time);
+                }
+            };
+        }
 
         watcher.setCurrentReadTime = (time, code) => {
             let prev = watcher.currentReadTime;
@@ -1430,13 +1456,24 @@ export class PathValueProxyWatcher {
                 this.runningWatcher = undefined;
             }
 
+            const nonLooping = options.runImmediately || options.allowUnsyncedReads;
+
             const readyToCommit = (
-                options.runImmediately || options.allowUnsyncedReads || (
+                nonLooping || (
                     watcher.lastUnsyncedAccesses.size === 0
                     && watcher.lastUnsyncedParentAccesses.size === 0
                     && !watcher.lastSpecialPromiseUnsynced
                 )
             );
+
+            if (!nonLooping) {
+                if (readyToCommit) {
+                    harvestableReadyLoopCount++;
+                } else {
+                    harvestableWaitingLoopCount++;
+                }
+            }
+
             // Commit writes BEFORE we watch. This prevents us from triggering ourself, in our first watch
             //  (on subsequent watches ClientWatcher will prevent self watches. It can't for the first
             //  watch, as it wasn't the triggerer).
@@ -1881,6 +1918,7 @@ registerPeriodic(function checkForZombieWatches() {
     }
 
     let zombies = Array.from(proxyWatcher.getAllWatchers()).filter(isZombieWatch);
+    lastZombieCount = zombies.length;
     if (zombies.length > 0) {
         console.warn(red(`Found ${zombies.length} zombie watchers`));
         let namesSet = new Set(zombies.map(x => x.debugName));

package/src/3-path-functions/PathFunctionRunner.ts

@@ -542,11 +542,14 @@ export class PathFunctionRunner {
                         }
 
                         let evalTimeStart = Date.now();
-                        let args = parseArgs(callPath);
-                        overrideCurrentCall({ spec: callPath, fnc: syncedSpec, }, () => {
-                            baseFunction(...args);
-                        });
-                        evalTime += Date.now() - evalTimeStart;
+                        try {
+                            let args = parseArgs(callPath);
+                            overrideCurrentCall({ spec: callPath, fnc: syncedSpec, }, () => {
+                                baseFunction(...args);
+                            });
+                        } finally {
+                            evalTime += Date.now() - evalTimeStart;
+                        }
 
                         // NOTE: The results are only temporarily stored, but serve as a way to let clients
                         //  know figure out the dependencies for the call, which makes function call prediction
@@ -573,6 +576,7 @@ export class PathFunctionRunner {
                             });
                             proxyWatcher.setEventPath(() => syncedModule.Results[callPath.CallId]);
                         });
+
                     },
                 }, {
                     onWritesCommitted(writes) {
@@ -631,7 +635,16 @@ export class PathFunctionRunner {
         if (PathFunctionRunner.DEBUG_CALLS) {
             console.log(`FINISHED${nooped ? " (skipped)" : ""} ${getDebugName(callPath, functionSpec, true)}, writes: ${finalWrites?.length}`);
         }
-        diskLog(`Finished FunctionRunner funciton`, { ...callPath, argsEncoded: "", functionSpec });
+
+        let wallTime = Date.now() - startTime;
+        let syncTime = wallTime - evalTime;
+
+
+        diskLog(`Finished FunctionRunner function`, {
+            ...callPath, argsEncoded: "", functionSpec,
+            wallTime, syncTime, evalTime,
+            loops: runCount,
+        });
         PathFunctionRunner.RUN_FINISH_COUNT++;
     }
 }

package/src/3-path-functions/pathFunctionLoader.ts

@@ -22,6 +22,7 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { getControllerNodeId, getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
 import { sha256 } from "js-sha256";
 import os from "os";
+import { formatTime } from "socket-function/src/formatting/format";
 
 // Get localPathRemappings using yargs, so it is easy to configure in multiple entry points
 let yargObj = isNodeTrue() && yargs(process.argv)
@@ -158,6 +159,8 @@ let moduleResolver = async (spec: FunctionSpec) => {
         await getFileLock(lockPath, async () => {
             if (fs.existsSync(repoPath + loadTimeIndicatorFileName)) return;
 
+            let time = Date.now();
+
             // Remove any previous attempt to sync it
             if (fs.existsSync(repoPath)) {
                 await fs.promises.rename(repoPath, repoPath + "_" + Date.now());
@@ -171,6 +174,9 @@ let moduleResolver = async (spec: FunctionSpec) => {
 
             // Mark it as loaded. If we don't reach this point we will move the folder and try again next time
             await fs.promises.writeFile(repoPath + loadTimeIndicatorFileName, Date.now() + "");
+
+            time = Date.now() - time;
+            console.log(blue(`Cloned and yarn installed repo`), { gitURL, time });
         });
     }
 
@@ -415,7 +421,7 @@ let everLoadedRepos = new Set<string>();
 export async function preloadFunctions(specs: FunctionSpec[]) {
     let nodeIds = await getControllerNodeIdList(functionPreloadController);
     await Promise.all(nodeIds.map(async nodeId => {
-        let controller = functionPreloadController.nodes[nodeId];
+        let controller = functionPreloadController.nodes[nodeId.nodeId];
         console.log(blue(`Preloading functions on ${nodeId}`));
         await errorToUndefined(controller.preloadFunctions(specs));
         console.log(blue(`Finished preloading functions on ${nodeId}`));

package/src/4-querysub/QuerysubController.ts

@@ -125,16 +125,6 @@ interface PermissionsObj {
 let permissionsPerNode = new Map<string, PermissionsObj>();
 registerResource("querysub|permissionsPerNode", permissionsPerNode);
 
-const lastCallTime = cache((nodeId: string): { value: Time } => {
-    SocketFunction.onNextDisconnect(nodeId, () => {
-        setTimeout(() => {
-            if (SocketFunction.isNodeConnected(nodeId)) return;
-            lastCallTime.clear(nodeId);
-        }, 1000 * 60);
-    });
-    return { value: epochTime };
-});
-
 export const isTrustedByDomain = cache(async function isTrustedByDomain(domain: string): Promise<boolean> {
     // NOTE: When getControllerNodeId accepts a domain we will pass them the domain we are using.
 

package/src/5-diagnostics/GenericFormat.tsx

@@ -1,4 +1,4 @@
-import { formatDateTime, formatNiceDateTime, formatNumber, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
+import { formatDateTime, formatNiceDateTime, formatNumber, formatPercent, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
 import { css } from "typesafecss";
 import { canHaveChildren } from "socket-function/src/types";
 import { qreact } from "../4-dom/qreact";
@@ -22,6 +22,7 @@ export type JSXFormatter<T = unknown, RowT extends RowType = RowType> = (
 type StringFormatters = (
     "guess"
     | "string" | "number"
+    | "percent"
     | "timeSpan" | "date"
     | "error" | "link"
     | "toSpaceCase"
@@ -38,6 +39,7 @@ const endGuessDateRange = +new Date(2040, 0, 1).getTime();
 let formatters: { [formatter in StringFormatters]: (value: unknown) => preact.ComponentChild } = {
     string: (value) => d(value, String(value || "")),
     number: (value) => d(value, formatNumber(Number(value))),
+    percent: (value) => d(value, formatPercent(Number(value))),
     timeSpan: (value) => d(value, formatTime(Number(value))),
     date: (value) => d(value, formatVeryNiceDateTime(Number(value))),
     error: (value) => d(value, <span class={errorMessage}>{String(value)}</span>),

package/src/5-diagnostics/TimeGrouper.tsx

@@ -93,6 +93,7 @@ export class TimeGrouper {
         });
     }
 
+    /** For sampled values, ex, % CPU usage */
     public getStateSummary(format: (value: number) => string): string {
         return this.getSummary().map(bucket => {
             if (bucket.duration === 0 || bucket.min === bucket.max) {
@@ -102,6 +103,7 @@ export class TimeGrouper {
             return `(${bucket.name}) ${format(bucket.min)} to ${format(bucket.max)}`;
         }).filter(isDefined).join("\n");
     }
+    /** Sums up the values. Ex, for time waiting. */
     public getEventSummary(format: (value: number) => string): string {
         return this.getSummary().map(bucket => {
             if (bucket.duration === 0) {

package/src/5-diagnostics/nodeMetadata.ts

@@ -5,13 +5,16 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { cache } from "socket-function/src/caching";
 import { TimeGrouper } from "./TimeGrouper";
-import { runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { throttleFunction, timeInMinute } from "socket-function/src/misc";
+import { runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { sort, throttleFunction, timeInMinute } from "socket-function/src/misc";
 import { formatNumber, formatPercent, formatTime } from "socket-function/src/formatting/format";
 import os from "os";
 import { isNode } from "typesafecss";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 
+
+const POLL_INTERVAL = timeInMinute * 5;
+
 // Undefined means we infer the column
 // Null means the column is removed
 export type ColumnType = undefined | null | {
@@ -39,14 +42,14 @@ export function registerNodeMetadata(getter: ExtraMetadata) {
     extraMetadatas.push(getter);
 }
 
+// IMPORTANT! We no longer diskLog here! diskLogging is not ideal for metrics like these. You almost only
+//      care about the current metrics OR past trends. A single value in the logs from the past is just
+//      NOT useful.
+
 let stateValues = cache((title: string, format: (value: number) => string): { (value: number): void } => {
     let timeGrouper = new TimeGrouper();
-    let logToDisk = throttleFunction(timeInMinute, (value: number) => {
-        diskLog(title, { value, formatted: format(value) });
-    });
     function onValue(value: number) {
         timeGrouper.onValueChanged(value);
-        void logToDisk(value);
     }
     registerNodeMetadata({
         columnName: title,
@@ -59,12 +62,8 @@ let stateValues = cache((title: string, format: (value: number) => string): { (v
 });
 let eventValues = cache((title: string, format: (value: number) => string): { (value: number): void } => {
     let timeGrouper = new TimeGrouper();
-    let logToDisk = throttleFunction(timeInMinute, (value: number) => {
-        diskLog(title, { value, formatted: format(value) });
-    });
     function onValue(value: number) {
         timeGrouper.onValueChanged(value);
-        void logToDisk(value);
     }
     registerNodeMetadata({
         columnName: title,
@@ -76,15 +75,142 @@ let eventValues = cache((title: string, format: (value: number) => string): { (v
     return onValue;
 });
 
+/** For values that should be summed to get the value. */
 export function logNodeStats(title: string, format: (value: number) => string): { (value: number): void } {
     return eventValues(title, format);
 }
+/** For state values (ex, % CPU usage), for which summing wouldn't make sense. */
 export function logNodeStateStats(title: string, format: (value: number) => string): { (value: number): void } {
     return stateValues(title, format);
 }
 
+let existingPeriodic = new Set<string>();
+/** For state values (ex, % CPU usage), for which summing wouldn't make sense. */
+export function addStatPeriodic(
+    title: string,
+    getValue: () => number,
+    format: (value: number) => string = formatNumber,
+) {
+    let hash = JSON.stringify({ title, getValue: getValue.toString(), format: format.toString() });
+    // Ignoring duplicates, otherwise we end up logging way too much
+    if (globalThis.isHotReloading?.() && existingPeriodic.has(hash)) return;
+
+    let timeGrouper = new TimeGrouper();
+    void runInfinitePollCallAtStart(POLL_INTERVAL, () => {
+        timeGrouper.onValueChanged(getValue());
+    });
+    registerNodeMetadata({
+        columnName: title,
+        column: {},
+        getValue() {
+            return timeGrouper.getStateSummary(format);
+        },
+    });
+}
+/** For values that should be summed to get the value. */
+export function addStatSumPeriodic(
+    title: string,
+    getValue: () => number,
+    format: (value: number) => string = formatNumber,
+) {
+    let hash = JSON.stringify({ title, getValue: getValue.toString(), format: format.toString() });
+    // Ignoring duplicates, otherwise we end up logging way too much
+    if (globalThis.isHotReloading?.() && existingPeriodic.has(hash)) return;
+
+    let timeGrouper = new TimeGrouper();
+    void runInfinitePollCallAtStart(POLL_INTERVAL, () => {
+        timeGrouper.onValueChanged(getValue());
+    });
+    registerNodeMetadata({
+        columnName: title,
+        column: {},
+        getValue() {
+            return timeGrouper.getEventSummary(format);
+        },
+    });
+}
+
+let pendingTimeProfiles = new Map<string, { start: number; end: number }[]>();
+export function onTimeProfile(title: string, startTime: number) {
+    let profiles = pendingTimeProfiles.get(title);
+    if (!profiles) {
+        profiles = [];
+        pendingTimeProfiles.set(title, profiles);
+        addTimeProfileDistribution(title, () => {
+            let values = pendingTimeProfiles.get(title);
+            pendingTimeProfiles.set(title, []);
+            return values || [];
+        });
+    }
+    profiles.push({ start: startTime, end: Date.now() });
+}
+
+/** Ex, for tracking the time distribution of API calls, or watchFunction evaluation, etc.
+ *      - Specifically for time, so we can handle overlap cases (such as waiting for 10 calls at once),
+ *          which requires specially handling otherwise the values will be meaningless (because
+ *          waiting from time 0 to 100ms for 10 calls is VERY different than waiting
+ *          from time 0 to 1000ms for 1 call).
+ * 
+ *  NOTE: Usually you should call onTimeProfile, which will wrap this call. But, sometimes
+ *      you need to harvest values from an external library, in which case, you will need
+ *      this function.
+ */
+export function addTimeProfileDistribution(
+    title: string,
+    /** It's expected that this will return all values since the last call to harvestValues
+     *      (basically, copy your value buffer and clear the old one).
+     */
+    harvestTimeRanges: () => { start: number; end: number; }[],
+) {
+    let hash = JSON.stringify({ title, getValues: harvestTimeRanges.toString() });
+    // Ignoring duplicates, otherwise we end up logging way too much
+    if (globalThis.isHotReloading?.() && existingPeriodic.has(hash)) return;
+
+    let fractionTimeWaitingGroup = new TimeGrouper();
+    let ratePerSecondGroup = new TimeGrouper();
+    let lastPollTime = Date.now();
+    runInfinitePoll(POLL_INTERVAL, () => {
+        let originalStartTime = lastPollTime;
+        let values = harvestTimeRanges();
+        values = values.slice();
+        sort(values, x => x.start);
+        let freeTime = 0;
+        for (let { start, end } of values) {
+            if (start > lastPollTime) {
+                freeTime += start - lastPollTime;
+            }
+            lastPollTime = end;
+        }
+        let now = Date.now();
+        freeTime += now - lastPollTime;
+        lastPollTime = now;
+
+        let duration = lastPollTime - originalStartTime;
+        let waitingTime = duration - freeTime;
+        let ratePerSecond = values.length / (duration / 1000);
+
+        fractionTimeWaitingGroup.onValueChanged(waitingTime / duration);
+        ratePerSecondGroup.onValueChanged(ratePerSecond);
+    });
+
+    registerNodeMetadata({
+        columnName: `% Time in ${title}`,
+        column: {},
+        getValue() {
+            return fractionTimeWaitingGroup.getStateSummary(formatPercent);
+        },
+    });
+    registerNodeMetadata({
+        columnName: `${title}/S`,
+        column: {},
+        getValue() {
+            return ratePerSecondGroup.getStateSummary(formatNumber);
+        },
+    });
+}
+
 if (isNode()) {
-    void runInfinitePollCallAtStart(timeInMinute, async () => {
+    void runInfinitePollCallAtStart(POLL_INTERVAL, async () => {
         // Get system memory usage
 
         const free = os.freemem();

package/src/diagnostics/NodeViewer.tsx

@@ -185,13 +185,22 @@ export class NodeViewer extends qreact.Component {
         }
 
         let builtinGroups = {
-            "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "Est % Usage", "Heap", "Buffers", "All Memory", "Blocking Lag", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
+            "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "% Profiled", "All Memory", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
+            "Performance": [
+                "All Memory",
+                "Heap", "Buffers",
+                "% Profiled",
+                "Blocking Lag",
+                "Top Profiled",
+            ],
         };
         // Column => group
-        let builtInGroupsLookup = new Map<string, string>();
+        let builtInGroupsLookup = new Map<string, string[]>();
         for (let [group, columns] of Object.entries(builtinGroups)) {
             for (let column of columns) {
-                builtInGroupsLookup.set(column, group);
+                let groups = builtInGroupsLookup.get(column) || [];
+                groups.push(group);
+                builtInGroupsLookup.set(column, groups);
             }
         }
         let groupedTables: {
@@ -206,12 +215,14 @@ export class NodeViewer extends qreact.Component {
             for (let table of tableList) {
                 let row = table.table?.row || {};
                 for (let column in row) {
-                    let groupTitle = builtInGroupsLookup.get(column) || "Other";
-                    if (column.includes("|")) {
-                        groupTitle = column.split("|")[0];
+                    let groupTitles = builtInGroupsLookup.get(column) || ["Other"];
+                    for (let groupTitle of groupTitles) {
+                        if (column.includes("|")) {
+                            groupTitle = column.split("|")[0];
+                        }
+                        groupedTables[groupTitle] = groupedTables[groupTitle] || {};
+                        groupedTables[groupTitle][column] = true;
                     }
-                    groupedTables[groupTitle] = groupedTables[groupTitle] || {};
-                    groupedTables[groupTitle][column] = true;
                 }
             }
         }

package/src/diagnostics/logs/DiskLoggerPage.tsx

@@ -4,7 +4,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { getSourceVSCodeLink, qreact } from "../../4-dom/qreact";
 import { assertIsManagementUser } from "../managementPages";
 import { LogFile, LogObj, getLogBuffer, getLogFiles, parseLogBuffer } from "./diskLogger";
-import { t } from "../../4-querysub/Querysub";
+import { Querysub, t } from "../../4-querysub/Querysub";
 import { URLParam } from "../../library-components/URLParam";
 import { getSyncedController } from "../../library-components/SyncedController";
 import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
@@ -16,7 +16,7 @@ import { DropdownSelector } from "../../library-components/DropdownSelector";
 import { Table, TableType } from "../../5-diagnostics/Table";
 import { LogType } from "../errorLogs/ErrorLogCore";
 import { canHaveChildren } from "socket-function/src/types";
-import { measureBlock } from "socket-function/src/profiling/measure";
+import { measureBlock, measureWrap } from "socket-function/src/profiling/measure";
 import { binarySearchBasic, list, sort, timeInDay, timeInHour } from "socket-function/src/misc";
 import { NodeViewerController, nodeViewerController } from "../NodeViewer";
 import { red } from "socket-function/src/formatting/logColors";
@@ -50,6 +50,10 @@ import { Zip } from "../../zip";
 //      - Probably by injecting ids? Although how would they be persistent? Hmm... Maybe line numbers
 //          is fine? I think we map the typescript (I hope we do), so they should be correct?
 
+// TODO: Use a binary format, but faster parsing.
+//  - If we could batch it, it would probably be very fast. We ARE just dealing with KVPs though,
+//      so... maybe it wouldn't be that much faster...
+
 module.hotreload = true;
 module.hotreload = true;
 
@@ -66,7 +70,7 @@ let inspectTimeURL = new URLParam("inspectTime", "");
 
 export class DiskLoggerPage extends qreact.Component {
     state = t.state({
-        x: t.number
+        downloadFilesHash: t.string,
     });
     render() {
         let selectedNodesIds = selectedNodeId.value.split("|").filter(x => x);
@@ -75,8 +79,8 @@ export class DiskLoggerPage extends qreact.Component {
 
         if (selectedNodesIds.length === 0) {
             // Group by machineId
-            let grouped = new Map<string, string[]>();
-            for (let nodeId of nodeIds) {
+            let grouped = new Map<string, { nodeId: string; entryPoint: string }[]>();
+            for (let { nodeId, entryPoint } of nodeIds) {
                 let obj = decodeNodeId(nodeId);
                 if (!obj) continue;
                 let { machineId } = obj;
@@ -84,7 +88,7 @@ export class DiskLoggerPage extends qreact.Component {
                 if (!list) {
                     grouped.set(machineId, list = []);
                 }
-                list.push(nodeId);
+                list.push({ nodeId, entryPoint });
             }
             return (
                 <div class={css.pad2(10).vbox(10)}>
@@ -95,12 +99,12 @@ export class DiskLoggerPage extends qreact.Component {
                             <div class={css.vbox(10).wrap.maxWidth("30vw").pad2(5).hsla(0, 0, 0, 0.1)}>
                                 <div class={css.fontSize(30)}>{getMachineId(machineId)}</div>
                                 <div class={css.hbox(20, 5).wrap}>
-                                    {list.map(nodeId =>
+                                    {list.map(({ nodeId, entryPoint }) =>
                                         <Anchor
                                             class={css.hbox(10)}
                                             values={[selectedNodeId.getOverride(nodeId)]}
                                         >
-                                            {nodeId}
+                                            {nodeId} ({entryPoint.replaceAll("\\", "/").split("/").pop()})
                                         </Anchor>
                                     )}
                                 </div>
@@ -136,17 +140,23 @@ export class DiskLoggerPage extends qreact.Component {
         let firstTime = Number.MAX_SAFE_INTEGER;
         let lastTime = Number.MIN_SAFE_INTEGER;
 
+        let filesHash = JSON.stringify(files.map(x => x.path));
+
         for (let file of files) {
             try {
                 if (file.startTime < firstTime) firstTime = file.startTime;
                 if (file.endTime > lastTime) lastTime = file.endTime;
+                if (filesHash !== this.state.downloadFilesHash) {
+                    loadingCount++;
+                    continue;
+                }
                 let buffer = controller.getRemoteLogBuffer(file.nodeId, file.path);
-                if (buffer) {
-                    buffer = Zip.gunzipSync(buffer);
-                    buffers.push(buffer);
-                } else {
+                if (!buffer) {
                     loadingCount++;
+                    continue;
                 }
+                buffer = unzipCached(buffer);
+                buffers.push(buffer);
             } catch (e) {
                 console.log("Error reading buffer", file, e);
             }
@@ -279,9 +289,8 @@ export class DiskLoggerPage extends qreact.Component {
                     picked={selectedNodesIds}
                     addPicked={x => selectedNodeId.value = [...selectedNodesIds, x].join("|")}
                     removePicked={x => selectedNodeId.value = selectedNodesIds.filter(y => y !== x).join("|")}
-                    options={nodeIds.map(x => ({ value: x, label: x }))}
+                    options={nodeIds.map(({ nodeId, entryPoint }) => ({ value: nodeId, label: `${nodeId} (${entryPoint.replaceAll("\\", "/").split("/").pop()})` }))}
                 />
-                {loadingCount > 0 && <h1>Loading {loadingCount} files</h1>}
                 <div className={css.fontSize(28).boldStyle}>
                     {selectedNodesIds.length} nodes, {files.length} files, filtered logs {formatNumber(filteredCount)} / {formatNumber(totalCount)}
                 </div>
@@ -296,6 +305,13 @@ export class DiskLoggerPage extends qreact.Component {
                         )}
                     </div>
                 </ShowMore>
+                {filesHash !== this.state.downloadFilesHash && <div>
+                    <button onClick={() => {
+                        this.state.downloadFilesHash = filesHash;
+                    }}>
+                        Download Files
+                    </button>
+                </div>}
                 {!inspectTime && <>
                     <TimeRangeSelector
                         start={startTimeURL} end={endTimeURL}
@@ -351,6 +367,7 @@ export class DiskLoggerPage extends qreact.Component {
                         number
                     />
                 )}
+                {loadingCount > 0 && <h1>Loading {loadingCount} files</h1>}
                 <Table {...table} initialLimit={30} />
                 {!!inspectTime && (
                     <InputLabel
@@ -365,6 +382,8 @@ export class DiskLoggerPage extends qreact.Component {
     }
 }
 
+const unzipCached = cacheLimited(100, Zip.gunzipSync);
+
 
 let remainingCacheSize = 1024 * 1024 * 1024 * 2;
 let logBufferCache = new Map<string, {
@@ -372,7 +391,8 @@ let logBufferCache = new Map<string, {
     logs: LogObj[];
     lastAccess: number;
 }>();
-function parseLogBufferCached(hash: string, buffers: Buffer[]): LogObj[] {
+
+const parseLogBufferCached = measureWrap(function parseLogBufferCached(hash: string, buffers: Buffer[]): LogObj[] {
     let cached = logBufferCache.get(hash);
     if (cached) {
         cached.lastAccess = Date.now();
@@ -392,7 +412,7 @@ function parseLogBufferCached(hash: string, buffers: Buffer[]): LogObj[] {
         lastAccess: Date.now()
     });
     return logs;
-}
+});
 
 
 

package/src/diagnostics/logs/diskLogger.ts

@@ -9,6 +9,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { isNode } from "typesafecss";
 import { logGitHashes } from "./logGitHashes";
 import { Zip } from "../../zip";
+import { formatNumber } from "socket-function/src/formatting/format";
 
 if (isNode()) {
     // Delayed setup, as we depend on diskLogger early, and we don't want to force high level
@@ -297,21 +298,31 @@ if (isNode()) {
         }
     });
     // Wait a random time, so we hopefully don't synchronize with any other services on this machine
-    void runInfinitePollCallAtStart(timeInHour + (1 + Math.random()), async function compressOldLogs() {
+    runInfinitePoll(timeInHour + (1 + Math.random()), async function compressOldLogs() {
         let logFiles = await fs.promises.readdir(folder);
         let compressTime = Date.now() - LOG_FILE_DURATION * 2;
+        let filesCompressed = 0;
         for (let file of logFiles) {
             if (!file.endsWith(".log")) continue;
+            if (filesCompressed === 0) {
+                console.log("Compressing old logs");
+            }
+            filesCompressed++;
             let path = folder + file;
             if (decodeLogFileName(path).endTime > compressTime) continue;
             let basePath = path.split(".").slice(0, -1).join(".");
             let buffer = await fs.promises.readFile(path);
+            let beforeSize = buffer.length;
             buffer = await Zip.gzip(buffer);
+            console.log(`Compressed ${file} from ${formatNumber(beforeSize)}B to ${formatNumber(buffer.length)}B`);
             let tempPath = basePath + Math.random() + ".temp";
             await fs.promises.writeFile(tempPath, buffer);
             await fs.promises.rename(tempPath, basePath + ".zip");
             await fs.promises.unlink(path);
         }
+        if (filesCompressed > 0) {
+            console.log(`Compressed ${filesCompressed} old log files`);
+        }
     });
 }