querysub 0.457.0 → 0.459.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.457.0",
+  "version": "0.459.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/PathValueController.ts

@@ -23,6 +23,9 @@ import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
 import { delay } from "socket-function/src/batching";
 export { pathValueCommitter };
 
+let pathValueSendCount = 0;
+export function getPathValueSendCount() { return pathValueSendCount; }
+
 // ONLY returns non-canGCValues that are valid
 export type AuditSnapshotEntry = {
     path: string;
@@ -47,6 +50,7 @@ export class PathValueControllerBase {
         const { Querysub } = await import("../4-querysub/Querysub");
         let { pathValues, nodeId } = config;
 
+        pathValueSendCount += pathValues.length;
         let serializedValues = await pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK });
         if (isDebugLogEnabled()) {
             for (let value of pathValues) {
@@ -70,6 +74,7 @@ export class PathValueControllerBase {
         nodeId: string;
         pathValues: PathValue[];
         initialTriggers?: { values: Set<string>; parentPaths: Set<string> },
+        reason: string;
     }) {
         let { nodeId, pathValues, initialTriggers } = config;
         if (isDebugLogEnabled()) {
@@ -88,6 +93,7 @@ export class PathValueControllerBase {
                     initialTrigger: initialTriggers?.values.has(pathValue.path),
                     triggerId: nextId(),
                     totalChanges: pathValues.length,
+                    reason: config.reason,
                 });
             }
         }
@@ -96,14 +102,16 @@ export class PathValueControllerBase {
         nodeId: string;
         pathValues: PathValue[];
         initialTriggers?: { values: Set<string>; parentPaths: Set<string> },
+        reason: string;
     }) {
         let changes = config.pathValues;
         let { nodeId, initialTriggers } = config;
+        pathValueSendCount += changes.length;
         let buffers = await pathValueSerializer.serialize(changes, {
             noLocks: true,
             compress: getCompressNetwork(),
         });
-        this.logSendValues({ nodeId, pathValues: changes, initialTriggers, });
+        this.logSendValues({ nodeId, pathValues: changes, initialTriggers, reason: config.reason });
         return await PathValueController.nodes[nodeId].sendData({
             valueBuffers: buffers,
             initialTriggers: initialTriggers,
@@ -201,6 +209,7 @@ export class PathValueControllerBase {
                 await timeoutToError(timeInSecond * 15, PathValueControllerBase.sendValues({
                     nodeId: otherAuthority,
                     pathValues: values,
+                    reason: "authorityShareValues",
                 }), () => new Error(`Timeout forwarding shared values (${values.length} values) to authority ${otherAuthority}`));
             } catch (error: any) {
                 console.error(error.message, { otherAuthority, count: values.length });

package/src/0-path-value-core/PathWatcher.ts

@@ -494,6 +494,7 @@ class PathWatcher {
                     nodeId: watcher,
                     pathValues: changes,
                     initialTriggers: initialTriggers,
+                    reason: "triggerLatestWatcher",
                 });
             })());
         }

package/src/3-path-functions/PathFunctionRunner.ts

@@ -26,6 +26,11 @@ import { getRoutingOverride, getRoutingOverridePart } from "../0-path-value-core
 import { PathRouter } from "../0-path-value-core/PathRouter";
 setImmediate(() => import("../4-querysub/Querysub"));
 
+let functionCallOuterCount = 0;
+let functionCallInnerCount = 0;
+export function getFunctionCallOuterCount() { return functionCallOuterCount; }
+export function getFunctionCallInnerCount() { return functionCallInnerCount; }
+
 export const functionSchema = rawSchema<{
     [domainName: string]: {
         PathFunctionRunner: {
@@ -520,6 +525,7 @@ export class PathFunctionRunner {
 
         stats.isInsideRunCall = true;
         stats.outerLoopCount++;
+        functionCallOuterCount++;
         try {
             await this.runCallBase(callSpec, functionSpec, stats);
         } finally {
@@ -647,6 +653,7 @@ export class PathFunctionRunner {
                         runCount++;
                         stats.lastInternalLoopCount = runCount;
                         stats.totalInternalLoopCount++;
+                        functionCallInnerCount++;
                         if (PathFunctionRunner.DEBUG_CALLS) {
                             console.log(`Evaluating (try count ${runCount}) ${debugNameColored}`);
                         }

package/src/4-querysub/QuerysubController.ts

@@ -488,6 +488,7 @@ export class QuerysubControllerBase {
                         values: new Set(undefinedValues.map(value => value.path)),
                         parentPaths: new Set(newParentPathsNotAllowed)
                     },
+                    reason: "QuerysubController.watch disallowed paths (permissions)",
                 }));
             }
 

package/src/5-diagnostics/gross-stats/grossStats.ts

@@ -0,0 +1,102 @@
+// Gross cluster statistics — per-minute counters of high-level activity, kept for a
+//  week per-node and polled by the management UI. See managementPages.tsx GrossStatsInfo / GrossStatsPage.
+//
+// Sources of stats live in their own modules and just maintain monotonic counters; this file
+//  imports those counters and samples them once a minute, recording deltas against the
+//  previous sample. Sources never depend on this file.
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { runInfinitePoll } from "socket-function/src/batching";
+import { timeInMinute } from "socket-function/src/misc";
+import { isNode } from "typesafecss";
+import { getFunctionCallInnerCount, getFunctionCallOuterCount } from "../../3-path-functions/PathFunctionRunner";
+import { getPathValueSendCount } from "../../0-path-value-core/PathValueController";
+import { getLogWriteCount } from "../../diagnostics/logs/diskLogger";
+
+export const GROSS_STATS_FIELDS = [
+    "functionCallsInner",
+    "functionCallsOuter",
+    "pathValuesSent",
+    "socketCalls",
+    "socketUploadBytes",
+    "socketDownloadBytes",
+    "socketCallTimeMs",
+    "logsWritten",
+] as const;
+export type GrossStatsField = typeof GROSS_STATS_FIELDS[number];
+
+export type GrossStatsBucket = {
+    // Epoch ms of the END of the minute this bucket covers
+    time: number;
+    deltas: Record<GrossStatsField, number>;
+};
+
+const BUCKETS_PER_WEEK = 60 * 24 * 7;
+
+function zeroCounts(): Record<GrossStatsField, number> {
+    let out = {} as Record<GrossStatsField, number>;
+    for (let field of GROSS_STATS_FIELDS) out[field] = 0;
+    return out;
+}
+
+// socketUploadBytes / socketDownloadBytes / socketCallTimeMs come from SocketFunction callbacks
+//  rather than a foreign module's counter, so we maintain monotonic running totals locally.
+let socketTotals = { uploadBytes: 0, downloadBytes: 0, callTimeMs: 0 };
+let socketHooksInstalled = false;
+function installSocketHooks() {
+    if (socketHooksInstalled) return;
+    socketHooksInstalled = true;
+    SocketFunction.trackMessageSizes.upload.push(size => {
+        socketTotals.uploadBytes += size;
+    });
+    SocketFunction.trackMessageSizes.download.push(size => {
+        socketTotals.downloadBytes += size;
+    });
+    SocketFunction.trackMessageSizes.callTimes.push(({ start, end }) => {
+        socketTotals.callTimeMs += Math.max(0, end - start);
+    });
+}
+
+function readCurrentTotals(): Record<GrossStatsField, number> {
+    return {
+        functionCallsInner: getFunctionCallInnerCount(),
+        functionCallsOuter: getFunctionCallOuterCount(),
+        pathValuesSent: getPathValueSendCount(),
+        socketCalls: SocketFunction.TOTAL_CALLS,
+        socketUploadBytes: socketTotals.uploadBytes,
+        socketDownloadBytes: socketTotals.downloadBytes,
+        socketCallTimeMs: socketTotals.callTimeMs,
+        logsWritten: getLogWriteCount(),
+    };
+}
+
+let prevSnapshot = zeroCounts();
+let buckets: GrossStatsBucket[] = [];
+
+function sampleMinute() {
+    let snapshot = readCurrentTotals();
+    let deltas = {} as Record<GrossStatsField, number>;
+    for (let field of GROSS_STATS_FIELDS) {
+        deltas[field] = snapshot[field] - prevSnapshot[field];
+    }
+    prevSnapshot = snapshot;
+    buckets.push({ time: Date.now(), deltas });
+    if (buckets.length > BUCKETS_PER_WEEK) {
+        buckets.splice(0, buckets.length - BUCKETS_PER_WEEK);
+    }
+}
+
+if (isNode()) {
+    installSocketHooks();
+    prevSnapshot = readCurrentTotals();
+    void runInfinitePoll(timeInMinute, sampleMinute);
+}
+
+export function getGrossStatsBuckets(sinceTime?: number): GrossStatsBucket[] {
+    if (sinceTime === undefined) return buckets;
+    let i = 0;
+    while (i < buckets.length && buckets[i].time <= sinceTime) {
+        i++;
+    }
+    return i === 0 ? buckets : buckets.slice(i);
+}

package/src/deployManager/machineApplyMainCode.ts

@@ -984,13 +984,7 @@ export async function machineApplyMain() {
     await Querysub.hostService("machine-apply");
     onServiceConfigChange(resyncServices);
 
-    runInfinitePoll(DEBUGGER_WEDGE_POLL_INTERVAL, async () => {
-        try {
-            await unwedgeStuckDebuggerScreens();
-        } catch (e: any) {
-            console.error(`Error in debugger-wedge watcher: ${e.stack ?? e}`);
-        }
-    });
+    runInfinitePoll(DEBUGGER_WEDGE_POLL_INTERVAL, unwedgeStuckDebuggerScreens);
 
     runInfinitePoll(timeInMinute * 3, async () => {
         //console.log(magenta(`Quick outdated check at ${new Date().toISOString()}`));

package/src/diagnostics/auditDiskValues.ts

@@ -184,7 +184,7 @@ export async function auditDiskValues(spec: AuthoritySpec) {
     if (valuesToSync.length > 0) {
         let valuesPerAuthority = PathRouter.getAllAuthoritiesForValues(valuesToSync);
         for (let [authorityNodeId, values] of valuesPerAuthority) {
-            await PathValueControllerBase.sendValues({ nodeId: authorityNodeId, pathValues: values });
+            await PathValueControllerBase.sendValues({ nodeId: authorityNodeId, pathValues: values, reason: "auditDiskValues" });
         }
     }
     let timeSync = Date.now() - timeStartSync;

package/src/diagnostics/grossStats/GrossStatsController.ts

@@ -0,0 +1,111 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { lazy } from "socket-function/src/caching";
+import { runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { assertIsManagementUser } from "../managementPages";
+import {
+    getGrossStatsBuckets,
+    GrossStatsBucket,
+    GrossStatsField,
+    GROSS_STATS_FIELDS,
+} from "../../5-diagnostics/gross-stats/grossStats";
+import { getOwnNodeId, isOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
+import { authorityLookup } from "../../0-path-value-core/AuthorityLookup";
+import { timeoutToUndefinedSilent } from "../../errors";
+
+const POLL_INTERVAL = timeInMinute;
+const PER_NODE_TIMEOUT = 30 * timeInSecond;
+const HEADER_WINDOW_MS = 60 * timeInMinute;
+const RETENTION_MS = 7 * 24 * timeInHour;
+
+let polledBuckets: Map<string, GrossStatsBucket[]> = new Map();
+
+async function listClusterNodeIds(): Promise<string[]> {
+    let topology = await authorityLookup.getTopology();
+    let ids = new Set<string>();
+    for (let entry of topology) ids.add(entry.nodeId);
+    ids.add(getOwnNodeId());
+    return Array.from(ids);
+}
+
+async function pollOnce() {
+    let nodeIds = await listClusterNodeIds();
+    let cutoff = Date.now() - RETENTION_MS;
+    await Promise.all(nodeIds.map(async nodeId => {
+        if (isOwnNodeId(nodeId)) {
+            polledBuckets.set(nodeId, getGrossStatsBuckets().filter(b => b.time >= cutoff));
+            return;
+        }
+        let prev = polledBuckets.get(nodeId);
+        let sinceTime = prev && prev.length > 0 ? prev[prev.length - 1].time : undefined;
+
+        let result = await timeoutToUndefinedSilent(
+            PER_NODE_TIMEOUT,
+            GrossStatsController.nodes[nodeId].getGrossStatsLocal({ sinceTime }),
+        );
+        let merged = prev ? (result ? prev.concat(result.buckets) : prev) : (result?.buckets ?? []);
+        merged = merged.filter(b => b.time >= cutoff);
+        polledBuckets.set(nodeId, merged);
+    }));
+}
+
+let ensurePolling = lazy(() => {
+    void runInfinitePollCallAtStart(POLL_INTERVAL, pollOnce);
+});
+
+class GrossStatsControllerBase {
+    public async getGrossStatsLocal(config?: { sinceTime?: number }): Promise<{
+        nodeId: string;
+        buckets: GrossStatsBucket[];
+    }> {
+        return {
+            nodeId: getOwnNodeId(),
+            buckets: getGrossStatsBuckets(config?.sinceTime),
+        };
+    }
+
+    /** Trampoline: returns the latest polled buckets for every node in the cluster.
+     *  Starts the background poller on first call. */
+    public async getGrossStats(): Promise<{
+        bucketsByNode: Map<string, GrossStatsBucket[]>;
+    }> {
+        ensurePolling();
+        return { bucketsByNode: new Map(polledBuckets) };
+    }
+
+    /** Trampoline: computes the cluster-wide per-second sum over the last hour from polled buckets.
+     *  Cheap — at ~100 nodes × ~60 buckets/hour = ~6000 numbers per field. */
+    public async getGrossStatsHeaderSummary(): Promise<{
+        clusterSum: Record<GrossStatsField, number>;
+    }> {
+        ensurePolling();
+        let windowStart = Date.now() - HEADER_WINDOW_MS;
+        let windowSec = HEADER_WINDOW_MS / 1000;
+
+        let clusterSum = {} as Record<GrossStatsField, number>;
+        for (let f of GROSS_STATS_FIELDS) clusterSum[f] = 0;
+        for (let buckets of polledBuckets.values()) {
+            for (let bucket of buckets) {
+                if (bucket.time < windowStart) continue;
+                for (let f of GROSS_STATS_FIELDS) {
+                    clusterSum[f] += bucket.deltas[f];
+                }
+            }
+        }
+        for (let f of GROSS_STATS_FIELDS) clusterSum[f] /= windowSec;
+        return { clusterSum };
+    }
+}
+
+export const GrossStatsController = SocketFunction.register(
+    "GrossStatsController-0193f8aa-7b91-7402-a4be-91a4b1f9c93d",
+    new GrossStatsControllerBase(),
+    () => ({
+        getGrossStatsLocal: { compress: true },
+        getGrossStats: { compress: true },
+        getGrossStatsHeaderSummary: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+);

package/src/diagnostics/grossStats/GrossStatsInfo.tsx

@@ -0,0 +1,95 @@
+import { qreact } from "../../4-dom/qreact";
+import { css } from "../../4-dom/css";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { timeInMinute } from "socket-function/src/misc";
+import { isCurrentUserSuperUser } from "../../user-implementation/userData";
+import { ATag } from "../../library-components/ATag";
+import { managementPageURL, showingManagementURL } from "../managementPages";
+import { getSyncedController } from "../../library-components/SyncedController";
+import { GrossStatsController } from "./GrossStatsController";
+import { GrossStatsField } from "../../5-diagnostics/gross-stats/grossStats";
+
+module.hotreload = true;
+
+const REFRESH_INTERVAL_MS = 5 * timeInMinute;
+
+const HEADER_FIELDS: GrossStatsField[] = [
+    "functionCallsInner",
+    "pathValuesSent",
+    "socketCalls",
+    "socketUploadBytes",
+    "socketDownloadBytes",
+    "socketCallTimeMs",
+    "logsWritten",
+];
+const FIELD_GLYPHS: Partial<Record<GrossStatsField, string>> = {
+    functionCallsInner: "⚡",
+    pathValuesSent: "📤",
+    socketCalls: "📞",
+    socketUploadBytes: "⬆",
+    socketDownloadBytes: "⬇",
+    socketCallTimeMs: "⏱",
+    logsWritten: "📝",
+};
+const FIELD_TITLES: Partial<Record<GrossStatsField, string>> = {
+    functionCallsInner: "Function calls per second, last hour",
+    pathValuesSent: "Path values sent per second, last hour",
+    socketCalls: "SocketFunction calls per second, last hour",
+    socketUploadBytes: "Upload bytes per second, last hour",
+    socketDownloadBytes: "Download bytes per second, last hour",
+    socketCallTimeMs: "SocketFunction call time ms per second, last hour",
+    logsWritten: "Log entries per second, last hour",
+};
+
+function formatField(field: GrossStatsField, value: number): string {
+    if (field === "socketUploadBytes" || field === "socketDownloadBytes") {
+        if (value >= 1024 * 1024) return `${(value / 1024 / 1024).toFixed(2)}MB`;
+        if (value >= 1024) return `${(value / 1024).toFixed(1)}KB`;
+        return `${value.toFixed(0)}B`;
+    }
+    if (field === "socketCallTimeMs") {
+        return `${value.toFixed(1)}ms`;
+    }
+    return formatNumber(value);
+}
+
+let grossStatsController = getSyncedController(GrossStatsController);
+
+export class GrossStatsInfo extends qreact.Component {
+    refreshTimer: ReturnType<typeof setInterval> | undefined;
+    componentDidMount() {
+        this.refreshTimer = setInterval(() => {
+            grossStatsController.refreshAll();
+        }, REFRESH_INTERVAL_MS);
+    }
+    componentWillUnmount() {
+        if (this.refreshTimer) clearInterval(this.refreshTimer);
+    }
+    render() {
+        if (!isCurrentUserSuperUser()) return undefined;
+
+        let result = grossStatsController(SocketFunction.browserNodeId()).getGrossStatsHeaderSummary();
+        if (!result) return undefined;
+
+        let parts: preact.ComponentChild[] = [];
+        for (let field of HEADER_FIELDS) {
+            parts.push(
+                <span title={FIELD_TITLES[field]}>
+                    {formatField(field, result.clusterSum[field])}{FIELD_GLYPHS[field]}
+                </span>
+            );
+        }
+
+        return <ATag
+            className={css.button.hbox(7).pad2(4)}
+            values={[
+                showingManagementURL.getOverride(true),
+                managementPageURL.getOverride("GrossStatsPage"),
+            ]}
+            title="Cluster activity per second over the past hour. Click for detail."
+        >
+            {parts}
+        </ATag>;
+    }
+}

package/src/diagnostics/grossStats/GrossStatsPage.tsx

@@ -0,0 +1,297 @@
+import preact from "preact";
+import { qreact } from "../../4-dom/qreact";
+import { css } from "../../4-dom/css";
+import { Querysub } from "../../4-querysub/QuerysubController";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { timeInHour, timeInMinute } from "socket-function/src/misc";
+import { cacheArgsEqual } from "socket-function/src/caching";
+import { t } from "../../2-proxy/schema2";
+import { getSyncedController } from "../../library-components/SyncedController";
+import { GrossStatsController } from "./GrossStatsController";
+import {
+    GROSS_STATS_FIELDS,
+    GrossStatsField,
+    GrossStatsBucket,
+} from "../../5-diagnostics/gross-stats/grossStats";
+export { GrossStatsController } from "./GrossStatsController";
+
+module.hotreload = true;
+
+const REFRESH_INTERVAL_MS = 5 * timeInMinute;
+const CHART_WIDTH = 1200;
+const CHART_HEIGHT = 300;
+
+const TIME_RANGES: { label: string; ms: number }[] = [
+    { label: "1h", ms: timeInHour },
+    { label: "6h", ms: 6 * timeInHour },
+    { label: "24h", ms: 24 * timeInHour },
+    { label: "7d", ms: 7 * 24 * timeInHour },
+];
+
+let pageSchema = Querysub.createLocalSchema("grossStatsPage", {
+    rangeMs: t.atomic<number>(timeInHour),
+    excludedNodes: t.atomic<Set<string>>(new Set()),
+    selectedField: t.atomic<GrossStatsField>("functionCallsInner"),
+});
+
+const NODE_COLORS = [
+    "hsl(0, 70%, 55%)",
+    "hsl(30, 75%, 50%)",
+    "hsl(55, 80%, 45%)",
+    "hsl(120, 50%, 45%)",
+    "hsl(170, 60%, 45%)",
+    "hsl(210, 60%, 55%)",
+    "hsl(250, 55%, 60%)",
+    "hsl(290, 55%, 55%)",
+    "hsl(330, 65%, 55%)",
+];
+function colorForNode(index: number): string {
+    return NODE_COLORS[index % NODE_COLORS.length];
+}
+
+function shortNodeId(nodeId: string): string {
+    let parts = nodeId.split(".");
+    if (parts.length >= 2) return `${parts[0].slice(0, 6)}…${parts[1].slice(0, 6)}`;
+    return nodeId.slice(0, 12);
+}
+
+let grossStatsController = getSyncedController(GrossStatsController);
+
+// Render a stacked bar chart to a PNG data URL. Cached by argument identity:
+//  selectedField/rangeMs/width/height are primitives; nodeIds and bucketsArrays are
+//  expected to be stable references (the bucketsArrays come from syncedController and
+//  only change when refreshed).
+const renderChartPNG = cacheArgsEqual((
+    selectedField: GrossStatsField,
+    rangeMs: number,
+    width: number,
+    height: number,
+    nodeIds: readonly string[],
+    bucketsArrays: readonly GrossStatsBucket[][],
+): { pngUrl: string; maxTotal: number } => {
+    let canvas = document.createElement("canvas");
+    canvas.width = width;
+    canvas.height = height;
+    let ctx = canvas.getContext("2d")!;
+    ctx.fillStyle = "hsl(0, 0%, 97%)";
+    ctx.fillRect(0, 0, width, height);
+
+    let now = Date.now();
+    let windowStart = now - rangeMs;
+
+    let perPixel: { perNode: number[]; total: number; }[] = [];
+    for (let x = 0; x < width; x++) {
+        perPixel.push({ perNode: new Array(nodeIds.length).fill(0), total: 0 });
+    }
+    let maxTotal = 0;
+    for (let i = 0; i < nodeIds.length; i++) {
+        let buckets = bucketsArrays[i];
+        if (!buckets) continue;
+        for (let bucket of buckets) {
+            if (bucket.time < windowStart) continue;
+            let xFloat = (bucket.time - windowStart) / rangeMs * width;
+            let x = Math.min(width - 1, Math.max(0, Math.floor(xFloat)));
+            let cell = perPixel[x];
+            let val = bucket.deltas[selectedField] || 0;
+            cell.perNode[i] += val;
+            cell.total += val;
+            if (cell.total > maxTotal) maxTotal = cell.total;
+        }
+    }
+    if (maxTotal <= 0) maxTotal = 1;
+
+    for (let x = 0; x < width; x++) {
+        let cell = perPixel[x];
+        if (cell.total === 0) continue;
+        let y = height;
+        for (let i = 0; i < nodeIds.length; i++) {
+            let val = cell.perNode[i];
+            if (!val) continue;
+            let segHeight = (val / maxTotal) * height;
+            ctx.fillStyle = colorForNode(i);
+            ctx.fillRect(x, y - segHeight, 1, segHeight);
+            y -= segHeight;
+        }
+    }
+
+    return { pngUrl: canvas.toDataURL(), maxTotal };
+}, 5);
+
+export class GrossStatsPage extends qreact.Component {
+    refreshTimer: ReturnType<typeof setInterval> | undefined;
+
+    componentDidMount() {
+        this.refreshTimer = setInterval(() => {
+            grossStatsController.refreshAll();
+        }, REFRESH_INTERVAL_MS);
+    }
+    componentWillUnmount() {
+        if (this.refreshTimer) clearInterval(this.refreshTimer);
+    }
+
+    private renderControls(nodeIds: string[]) {
+        let state = pageSchema();
+        return <div className={css.vbox(8).pad2(8)}>
+            <div className={css.hbox(6)}>
+                <span>Range:</span>
+                {TIME_RANGES.map(r =>
+                    <button
+                        className={css.pad2(8, 4) + (state.rangeMs === r.ms ? " " + css.hsl(210, 70, 60).hslcolor(0, 0, 100).bold : "")}
+                        onClick={() => Querysub.commit(() => { state.rangeMs = r.ms; })}
+                    >{r.label}</button>
+                )}
+            </div>
+            <div className={css.hbox(6).wrap}>
+                <span>Field:</span>
+                {GROSS_STATS_FIELDS.map(f =>
+                    <button
+                        className={css.pad2(8, 4) + (state.selectedField === f ? " " + css.hsl(140, 60, 50).hslcolor(0, 0, 100).bold : "")}
+                        onClick={() => Querysub.commit(() => { state.selectedField = f; })}
+                    >{f}</button>
+                )}
+            </div>
+            <div className={css.hbox(8).wrap}>
+                <span>Servers:</span>
+                <button
+                    className={css.pad2(8, 4)}
+                    onClick={() => Querysub.commit(() => { state.excludedNodes = new Set(); })}
+                >Select all</button>
+                <button
+                    className={css.pad2(8, 4)}
+                    onClick={() => Querysub.commit(() => { state.excludedNodes = new Set(nodeIds); })}
+                >Select none</button>
+                {nodeIds.map((nodeId, i) =>
+                    <label className={css.hbox(4)}>
+                        <input
+                            type="checkbox"
+                            checked={!state.excludedNodes.has(nodeId)}
+                            onChange={() => {
+                                Querysub.commit(() => {
+                                    let next = new Set(state.excludedNodes);
+                                    if (next.has(nodeId)) next.delete(nodeId);
+                                    else next.add(nodeId);
+                                    state.excludedNodes = next;
+                                });
+                            }}
+                        />
+                        <span className={css.hslcolor(0, 0, 100).pad2(2)} style={{ background: colorForNode(i) }}>
+                            {shortNodeId(nodeId)}
+                        </span>
+                    </label>
+                )}
+            </div>
+        </div>;
+    }
+
+    private renderTable(nodeIds: string[], bucketsArrays: GrossStatsBucket[][]) {
+        let state = pageSchema();
+        let now = Date.now();
+        let windowStart = now - state.rangeMs;
+
+        let perNodeTotals: Record<GrossStatsField, number>[] = [];
+        let totalsRow: Record<GrossStatsField, number> = {} as Record<GrossStatsField, number>;
+        for (let f of GROSS_STATS_FIELDS) totalsRow[f] = 0;
+
+        for (let i = 0; i < nodeIds.length; i++) {
+            let totals = {} as Record<GrossStatsField, number>;
+            for (let f of GROSS_STATS_FIELDS) totals[f] = 0;
+            for (let bucket of bucketsArrays[i] ?? []) {
+                if (bucket.time < windowStart) continue;
+                for (let f of GROSS_STATS_FIELDS) {
+                    totals[f] += bucket.deltas[f];
+                    totalsRow[f] += bucket.deltas[f];
+                }
+            }
+            perNodeTotals.push(totals);
+        }
+
+        let maxPerField = {} as Record<GrossStatsField, number>;
+        for (let f of GROSS_STATS_FIELDS) {
+            maxPerField[f] = 0;
+            for (let totals of perNodeTotals) {
+                if (totals[f] > maxPerField[f]) maxPerField[f] = totals[f];
+            }
+        }
+        let highlightStyle = css.hsl(60, 90, 75);
+
+        return <table className={css.fillWidth}>
+            <thead>
+                <tr>
+                    <th className={css.textAlign("left").pad2(4)}>Server</th>
+                    {GROSS_STATS_FIELDS.map(f => <th className={css.textAlign("right").pad2(4)}>{f}</th>)}
+                </tr>
+            </thead>
+            <tbody>
+                <tr className={css.boldStyle}>
+                    <td className={css.pad2(4)}>TOTAL</td>
+                    {GROSS_STATS_FIELDS.map(f => <td className={css.textAlign("right").pad2(4)}>{formatNumber(totalsRow[f])}</td>)}
+                </tr>
+                {nodeIds.map((nodeId, i) =>
+                    <tr>
+                        <td className={css.pad2(4)} style={{ borderLeft: `4px solid ${colorForNode(i)}` }}>
+                            {shortNodeId(nodeId)}
+                        </td>
+                        {GROSS_STATS_FIELDS.map(f => {
+                            let isMax = perNodeTotals[i][f] > 0 && perNodeTotals[i][f] === maxPerField[f];
+                            return <td className={css.textAlign("right").pad2(4) + (isMax ? " " + highlightStyle : "")}>
+                                {formatNumber(perNodeTotals[i][f])}
+                            </td>;
+                        })}
+                    </tr>
+                )}
+            </tbody>
+        </table>;
+    }
+
+    render() {
+        let state = pageSchema();
+        let result = grossStatsController(SocketFunction.browserNodeId()).getGrossStats();
+        let bucketsByNode = result?.bucketsByNode ?? new Map<string, GrossStatsBucket[]>();
+        let allNodeIds = Array.from(bucketsByNode.keys()).sort();
+
+        let selectedNodeIds = allNodeIds.filter(n => !state.excludedNodes.has(n));
+        let bucketsArrays: GrossStatsBucket[][] = selectedNodeIds.map(n => bucketsByNode.get(n) ?? []);
+        let anyLoading = !result;
+
+        let chart = renderChartPNG(
+            state.selectedField,
+            state.rangeMs,
+            CHART_WIDTH,
+            CHART_HEIGHT,
+            selectedNodeIds,
+            bucketsArrays,
+        );
+        let now = Date.now();
+        let windowStart = now - state.rangeMs;
+
+        return <div className={css.vbox(8).pad2(8).fillWidth}>
+            <h2>Cluster Stats</h2>
+            {this.renderControls(allNodeIds)}
+            <div className={css.vbox(4)}>
+                <div className={css.hbox(8).bold}>
+                    <span>peak: {formatNumber(chart.maxTotal)} {state.selectedField} / minute</span>
+                </div>
+                <img
+                    src={chart.pngUrl}
+                    width={CHART_WIDTH}
+                    height={CHART_HEIGHT}
+                    className={css.hsl(0, 0, 100)}
+                    style={anyLoading ? { opacity: 0.5 } : undefined}
+                />
+                <div className={css.hbox(0).fillWidth}>
+                    <span className={css.flexShrink0}>{formatLocalTime(windowStart)}</span>
+                    <span className={css.fillBoth}></span>
+                    <span className={css.flexShrink0}>{formatLocalTime(now)}</span>
+                </div>
+            </div>
+            {this.renderTable(selectedNodeIds, bucketsArrays)}
+        </div>;
+    }
+}
+
+function formatLocalTime(ms: number): string {
+    let d = new Date(ms);
+    let pad = (n: number) => String(n).padStart(2, "0");
+    return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
+}

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -300,7 +300,10 @@ export class BufferIndex {
         dataReader: Reader;
         params: SearchParams;
         keepIterating: () => boolean;
-        onResult: (match: Buffer) => void;
+        // Returns true iff the caller actually retained the value. We use that
+        // to drive the per-file matchCount cap below: see the note at the
+        // `matchesPattern(buffer)` call for why we can't blindly count emits.
+        onResult: (match: Buffer) => boolean;
         results: IndexedLogResults;
         allSearchUnits: Unit[][];
         matchesPattern: (buffer: Buffer) => boolean;
@@ -390,13 +393,38 @@ export class BufferIndex {
                 const bufferStep = iterateForward ? 1 : -1;
 
                 for (let bufferIndex = bufferStartIdx; iterateForward ? bufferIndex < bufferEndIdx : bufferIndex > bufferEndIdx; bufferIndex += bufferStep) {
-                    if (matchCount >= params.limit || !config.keepIterating()) break;
+                    // No `matchCount >= params.limit` cap inside the block.
+                    // Buffer order within a block is not guaranteed to follow
+                    // the search direction (blocks are time-ordered, buffers
+                    // inside them are not), so stopping mid-block on a match
+                    // count would drop earlier-time buffers we haven't reached
+                    // yet. The block-level cap above is the only safe stop;
+                    // here we only honor cross-file `keepIterating` (which
+                    // applies to the whole file at once, so it's safe at any
+                    // granularity).
+                    if (!config.keepIterating()) break;
                     await config.results.limitGroup?.wait();
 
                     let buffer = buffers[bufferIndex];
                     if (matchesPattern(buffer)) {
-                        config.onResult(buffer);
-                        matchCount++;
+                        // Only count matches the caller actually kept. `onResult`
+                        // routes through `FindProgressTracker.addResult`, which
+                        // can reject for reasons we don't see from here — most
+                        // notably time-range filtering (an entry whose time is
+                        // outside the search window matched the text pattern
+                        // but isn't a real hit). Counting rejected emits would
+                        // let a stretch of out-of-window matches at the start
+                        // of a file blow the per-file cap and short-circuit the
+                        // scan before we reach the in-window region.
+                        //
+                        // The cost is that we keep matching and calling onResult
+                        // through those out-of-window blocks (mild inefficiency).
+                        // We can't skip ahead — buffers are scanned linearly and
+                        // we don't know up front which entries the caller will
+                        // reject — so this is the best we can do here.
+                        if (config.onResult(buffer)) {
+                            matchCount++;
+                        }
                     }
                 }
             } catch (e: any) {
@@ -531,7 +559,10 @@ export class BufferIndex {
         params: SearchParams;
 
         keepIterating: () => boolean;
-        onResult: (match: Buffer) => void;
+        // See the note on `findLocal.onResult` — return value drives the
+        // per-file matchCount cap so out-of-window emits don't short-circuit
+        // the scan.
+        onResult: (match: Buffer) => boolean;
         results: IndexedLogResults;
     }): Promise<{
         blockSearchTime: number;

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -505,7 +505,10 @@ export class BufferUnitIndex {
         params: SearchParams;
         allSearchUnits: Unit[][];
         keepIterating: () => boolean;
-        onResult: (match: Buffer) => void;
+        // Returns true iff the caller actually retained the value. Drives the
+        // `matchCounts` cap below — see the comment at the `matchesPattern`
+        // call for why we can't blindly count emits.
+        onResult: (match: Buffer) => boolean;
         index: Buffer;
         reader: Reader;
         results: IndexedLogResults;
@@ -577,14 +580,38 @@ export class BufferUnitIndex {
                     const step = iterateForward ? 1 : -1;
 
                     for (let i = startIdx; iterateForward ? i < endIdx : i > endIdx; i += step) {
-                        if (stopIterating()) break;
+                        // No matchCount-based cap inside the block. Buffer
+                        // order within a block is not guaranteed to follow the
+                        // search direction (blocks are time-ordered, buffers
+                        // inside them are not), so a mid-block stop on
+                        // `relevantCount >= limit` would drop earlier-time
+                        // buffers we haven't reached yet. Block-level
+                        // `stopIterating` is the safe granularity; here we
+                        // only honor cross-file `keepIterating`, which applies
+                        // to the whole file at once.
+                        if (!keepIterating()) break;
                         await results.limitGroup?.wait();
 
                         const buffer = await this.getBufferFromBlock(blockReader, i);
                         if (matchesPattern(buffer)) {
-                            config.onResult(buffer);
-                            matchCount++;
-                            matchCounts[blockIndex]++;
+                            // Only count matches the caller actually kept.
+                            // `onResult` routes through
+                            // `FindProgressTracker.addResult`, which can reject
+                            // for reasons opaque to us — most notably
+                            // time-range filtering. Counting rejected emits
+                            // would let a stretch of out-of-window matches at
+                            // the start of a file blow the per-file cap and
+                            // short-circuit the scan before we reach the
+                            // in-window region. The cost is that we keep
+                            // matching and calling onResult through those
+                            // out-of-window blocks (mild inefficiency); we
+                            // can't skip ahead because buffers are scanned
+                            // linearly and we don't know up front which
+                            // entries the caller will reject.
+                            if (config.onResult(buffer)) {
+                                matchCount++;
+                                matchCounts[blockIndex]++;
+                            }
                         }
                     }
                 } catch (e: any) {

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -633,9 +633,15 @@ export class IndexedLogs<T> {
                         limit: config.params.limit,
                     },
                     keepIterating: () => !results.cancel && progressTracker.isSourceRelevant(path),
-                    onResult: (match: Buffer) => {
-                        // NOTE: We absolutely cannot do any limiting here, as that would just result in a lot of values that might be newer than the old values being discarded. Each loop itself has to do some kind of limiting as it knows how many it's found in its specific loop, and it knows the direction it's iterating and the direction of the values. 
-                        progressTracker.addResult(match, path);
+                    onResult: (match: Buffer): boolean => {
+                        // Return value drives the per-file matchCount cap inside
+                        // BufferIndex.find / BufferUnitIndex.find. addResult
+                        // returns false when it rejects (time outside the
+                        // search range, or past the kept top-K cutoff); the
+                        // scanner uses that to avoid counting rejected emits
+                        // against the cap. See the comment on those scanners'
+                        // `matchesPattern` calls for the full reasoning.
+                        return progressTracker.addResult(match, path);
                     },
                     results,
                 });

package/src/diagnostics/logs/diskLogger.ts

@@ -10,6 +10,9 @@ import { isPublic } from "../../config";
 import type { IndexedLogs } from "./IndexedLogs/IndexedLogs";
 // IMPORTANT! We can't have any real imports here, because we are depended on so early in startup!
 
+let logWriteCount = 0;
+export function getLogWriteCount() { return logWriteCount; }
+
 if (isNode()) {
     // Delayed setup, as we depend on diskLogger early, and we don't want to force high level
     //  modules to be required before their level
@@ -150,6 +153,7 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
             } else {
                 errorLogs.append(logObj);
             }
+            logWriteCount++;
         }
 
     } catch (e: any) {

package/src/diagnostics/managementPages.tsx

@@ -167,6 +167,12 @@ export async function registerManagementPages2(config: {
         controllerName: "SyncTestController",
         getModule: () => import("./SyncTestPage"),
     });
+    inputPages.push({
+        title: "Gross Stats",
+        componentName: "GrossStatsPage",
+        controllerName: "GrossStatsController",
+        getModule: () => import("./grossStats/GrossStatsPage"),
+    });
     inputPages.push(...config.pages);
 
     // NOTE: We don't store the UI in the database (here, or anywhere else, at least
@@ -313,6 +319,7 @@ export function renderIsManagementUser() {
 
 const ErrorWarning = createLazyComponent(() => import("./logs/errorNotifications2/ErrorWarning"))("ErrorWarning");
 const LaunchTrackingHeader = createLazyComponent(() => import("../deployManager/LaunchTrackingHeader"))("LaunchTrackingHeader");
+const GrossStatsInfo = createLazyComponent(() => import("./grossStats/GrossStatsInfo"))("GrossStatsInfo");
 
 class ManagementRoot extends qreact.Component {
     state = {
@@ -386,6 +393,7 @@ class ManagementRoot extends qreact.Component {
                             <PathDistributionInfo />
                             <ValuePathWarning />
                             <LaunchTrackingHeader />
+                            <GrossStatsInfo />
                         </div>}
                     </div>
                     {currentPage &&