querysub 0.456.0 → 0.458.0

package/src/diagnostics/grossStats/GrossStatsPage.tsx

@@ -0,0 +1,259 @@
+import preact from "preact";
+import { qreact } from "../../4-dom/qreact";
+import { css } from "../../4-dom/css";
+import { Querysub } from "../../4-querysub/QuerysubController";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { timeInHour, timeInMinute } from "socket-function/src/misc";
+import { cacheArgsEqual } from "socket-function/src/caching";
+import { t } from "../../2-proxy/schema2";
+import { getSyncedController } from "../../library-components/SyncedController";
+import { GrossStatsController } from "./GrossStatsController";
+import {
+    GROSS_STATS_FIELDS,
+    GrossStatsField,
+    GrossStatsBucket,
+} from "../../5-diagnostics/gross-stats/grossStats";
+export { GrossStatsController } from "./GrossStatsController";
+
+module.hotreload = true;
+
+const REFRESH_INTERVAL_MS = 5 * timeInMinute;
+const CHART_WIDTH = 1200;
+const CHART_HEIGHT = 300;
+
+const TIME_RANGES: { label: string; ms: number }[] = [
+    { label: "1h", ms: timeInHour },
+    { label: "6h", ms: 6 * timeInHour },
+    { label: "24h", ms: 24 * timeInHour },
+    { label: "7d", ms: 7 * 24 * timeInHour },
+];
+
+let pageSchema = Querysub.createLocalSchema("grossStatsPage", {
+    rangeMs: t.atomic<number>(timeInHour),
+    excludedNodes: t.atomic<Set<string>>(new Set()),
+    selectedField: t.atomic<GrossStatsField>("functionCallsInner"),
+});
+
+const NODE_COLORS = [
+    "hsl(0, 70%, 55%)",
+    "hsl(30, 75%, 50%)",
+    "hsl(55, 80%, 45%)",
+    "hsl(120, 50%, 45%)",
+    "hsl(170, 60%, 45%)",
+    "hsl(210, 60%, 55%)",
+    "hsl(250, 55%, 60%)",
+    "hsl(290, 55%, 55%)",
+    "hsl(330, 65%, 55%)",
+];
+function colorForNode(index: number): string {
+    return NODE_COLORS[index % NODE_COLORS.length];
+}
+
+function shortNodeId(nodeId: string): string {
+    let parts = nodeId.split(".");
+    if (parts.length >= 2) return `${parts[0].slice(0, 6)}…${parts[1].slice(0, 6)}`;
+    return nodeId.slice(0, 12);
+}
+
+let grossStatsController = getSyncedController(GrossStatsController);
+
+// Render a stacked bar chart to a PNG data URL. Cached by argument identity:
+//  selectedField/rangeMs/width/height are primitives; nodeIds and bucketsArrays are
+//  expected to be stable references (the bucketsArrays come from syncedController and
+//  only change when refreshed).
+const renderChartPNG = cacheArgsEqual((
+    selectedField: GrossStatsField,
+    rangeMs: number,
+    width: number,
+    height: number,
+    nodeIds: readonly string[],
+    bucketsArrays: readonly GrossStatsBucket[][],
+): string => {
+    let canvas = document.createElement("canvas");
+    canvas.width = width;
+    canvas.height = height;
+    let ctx = canvas.getContext("2d")!;
+    ctx.fillStyle = "hsl(0, 0%, 97%)";
+    ctx.fillRect(0, 0, width, height);
+
+    let now = Date.now();
+    let windowStart = now - rangeMs;
+
+    let perPixel: { perNode: number[]; total: number; }[] = [];
+    for (let x = 0; x < width; x++) {
+        perPixel.push({ perNode: new Array(nodeIds.length).fill(0), total: 0 });
+    }
+    let maxTotal = 0;
+    for (let i = 0; i < nodeIds.length; i++) {
+        let buckets = bucketsArrays[i];
+        if (!buckets) continue;
+        for (let bucket of buckets) {
+            if (bucket.time < windowStart) continue;
+            let xFloat = (bucket.time - windowStart) / rangeMs * width;
+            let x = Math.min(width - 1, Math.max(0, Math.floor(xFloat)));
+            let cell = perPixel[x];
+            let val = bucket.deltas[selectedField] || 0;
+            cell.perNode[i] += val;
+            cell.total += val;
+            if (cell.total > maxTotal) maxTotal = cell.total;
+        }
+    }
+    if (maxTotal <= 0) maxTotal = 1;
+
+    for (let x = 0; x < width; x++) {
+        let cell = perPixel[x];
+        if (cell.total === 0) continue;
+        let y = height;
+        for (let i = 0; i < nodeIds.length; i++) {
+            let val = cell.perNode[i];
+            if (!val) continue;
+            let segHeight = (val / maxTotal) * height;
+            ctx.fillStyle = colorForNode(i);
+            ctx.fillRect(x, y - segHeight, 1, segHeight);
+            y -= segHeight;
+        }
+    }
+
+    return canvas.toDataURL();
+}, 5);
+
+export class GrossStatsPage extends qreact.Component {
+    refreshTimer: ReturnType<typeof setInterval> | undefined;
+
+    componentDidMount() {
+        this.refreshTimer = setInterval(() => {
+            grossStatsController.refreshAll();
+        }, REFRESH_INTERVAL_MS);
+    }
+    componentWillUnmount() {
+        if (this.refreshTimer) clearInterval(this.refreshTimer);
+    }
+
+    private renderControls(nodeIds: string[]) {
+        let state = pageSchema();
+        return <div className={css.vbox(8).pad2(8)}>
+            <div className={css.hbox(6)}>
+                <span>Range:</span>
+                {TIME_RANGES.map(r =>
+                    <span
+                        className={css.button.pad2(4, 2) + (state.rangeMs === r.ms ? " " + css.hsl(210, 70, 60).hslcolor(0, 0, 100) : "")}
+                        onClick={() => Querysub.commit(() => { state.rangeMs = r.ms; })}
+                    >{r.label}</span>
+                )}
+            </div>
+            <div className={css.hbox(6).wrap}>
+                <span>Field:</span>
+                {GROSS_STATS_FIELDS.map(f =>
+                    <span
+                        className={css.button.pad2(4, 2) + (state.selectedField === f ? " " + css.hsl(140, 60, 50).hslcolor(0, 0, 100) : "")}
+                        onClick={() => Querysub.commit(() => { state.selectedField = f; })}
+                    >{f}</span>
+                )}
+            </div>
+            <div className={css.hbox(8).wrap}>
+                <span>Servers:</span>
+                {nodeIds.map((nodeId, i) =>
+                    <label className={css.hbox(4)}>
+                        <input
+                            type="checkbox"
+                            checked={!state.excludedNodes.has(nodeId)}
+                            onChange={() => {
+                                Querysub.commit(() => {
+                                    let next = new Set(state.excludedNodes);
+                                    if (next.has(nodeId)) next.delete(nodeId);
+                                    else next.add(nodeId);
+                                    state.excludedNodes = next;
+                                });
+                            }}
+                        />
+                        <span className={css.hslcolor(0, 0, 100).pad2(2)} style={{ background: colorForNode(i) }}>
+                            {shortNodeId(nodeId)}
+                        </span>
+                    </label>
+                )}
+            </div>
+        </div>;
+    }
+
+    private renderTable(nodeIds: string[], bucketsArrays: GrossStatsBucket[][]) {
+        let state = pageSchema();
+        let now = Date.now();
+        let windowStart = now - state.rangeMs;
+
+        let perNodeTotals: Record<GrossStatsField, number>[] = [];
+        let totalsRow: Record<GrossStatsField, number> = {} as Record<GrossStatsField, number>;
+        for (let f of GROSS_STATS_FIELDS) totalsRow[f] = 0;
+
+        for (let i = 0; i < nodeIds.length; i++) {
+            let totals = {} as Record<GrossStatsField, number>;
+            for (let f of GROSS_STATS_FIELDS) totals[f] = 0;
+            for (let bucket of bucketsArrays[i] ?? []) {
+                if (bucket.time < windowStart) continue;
+                for (let f of GROSS_STATS_FIELDS) {
+                    totals[f] += bucket.deltas[f];
+                    totalsRow[f] += bucket.deltas[f];
+                }
+            }
+            perNodeTotals.push(totals);
+        }
+
+        return <table className={css.fillWidth}>
+            <thead>
+                <tr>
+                    <th className={css.textAlign("left").pad2(4)}>Server</th>
+                    {GROSS_STATS_FIELDS.map(f => <th className={css.textAlign("right").pad2(4)}>{f}</th>)}
+                </tr>
+            </thead>
+            <tbody>
+                <tr className={css.boldStyle}>
+                    <td className={css.pad2(4)}>TOTAL</td>
+                    {GROSS_STATS_FIELDS.map(f => <td className={css.textAlign("right").pad2(4)}>{formatNumber(totalsRow[f])}</td>)}
+                </tr>
+                {nodeIds.map((nodeId, i) =>
+                    <tr>
+                        <td className={css.pad2(4)} style={{ borderLeft: `4px solid ${colorForNode(i)}` }}>
+                            {shortNodeId(nodeId)}
+                        </td>
+                        {GROSS_STATS_FIELDS.map(f =>
+                            <td className={css.textAlign("right").pad2(4)}>{formatNumber(perNodeTotals[i][f])}</td>
+                        )}
+                    </tr>
+                )}
+            </tbody>
+        </table>;
+    }
+
+    render() {
+        let state = pageSchema();
+        let result = grossStatsController(SocketFunction.browserNodeId()).getGrossStats();
+        let bucketsByNode = result?.bucketsByNode ?? new Map<string, GrossStatsBucket[]>();
+        let allNodeIds = Array.from(bucketsByNode.keys()).sort();
+
+        let selectedNodeIds = allNodeIds.filter(n => !state.excludedNodes.has(n));
+        let bucketsArrays: GrossStatsBucket[][] = selectedNodeIds.map(n => bucketsByNode.get(n) ?? []);
+        let anyLoading = !result;
+
+        let chartPNG = renderChartPNG(
+            state.selectedField,
+            state.rangeMs,
+            CHART_WIDTH,
+            CHART_HEIGHT,
+            selectedNodeIds,
+            bucketsArrays,
+        );
+
+        return <div className={css.vbox(8).pad2(8).fillWidth}>
+            <h2>Cluster Stats</h2>
+            {this.renderControls(allNodeIds)}
+            <img
+                src={chartPNG}
+                width={CHART_WIDTH}
+                height={CHART_HEIGHT}
+                className={css.hsl(0, 0, 100)}
+                style={anyLoading ? { opacity: 0.5 } : undefined}
+            />
+            {this.renderTable(selectedNodeIds, bucketsArrays)}
+        </div>;
+    }
+}

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -300,7 +300,10 @@ export class BufferIndex {
         dataReader: Reader;
         params: SearchParams;
         keepIterating: () => boolean;
-        onResult: (match: Buffer) => void;
+        // Returns true iff the caller actually retained the value. We use that
+        // to drive the per-file matchCount cap below: see the note at the
+        // `matchesPattern(buffer)` call for why we can't blindly count emits.
+        onResult: (match: Buffer) => boolean;
         results: IndexedLogResults;
         allSearchUnits: Unit[][];
         matchesPattern: (buffer: Buffer) => boolean;
@@ -395,8 +398,24 @@ export class BufferIndex {
 
                     let buffer = buffers[bufferIndex];
                     if (matchesPattern(buffer)) {
-                        config.onResult(buffer);
-                        matchCount++;
+                        // Only count matches the caller actually kept. `onResult`
+                        // routes through `FindProgressTracker.addResult`, which
+                        // can reject for reasons we don't see from here — most
+                        // notably time-range filtering (an entry whose time is
+                        // outside the search window matched the text pattern
+                        // but isn't a real hit). Counting rejected emits would
+                        // let a stretch of out-of-window matches at the start
+                        // of a file blow the per-file cap and short-circuit the
+                        // scan before we reach the in-window region.
+                        //
+                        // The cost is that we keep matching and calling onResult
+                        // through those out-of-window blocks (mild inefficiency).
+                        // We can't skip ahead — buffers are scanned linearly and
+                        // we don't know up front which entries the caller will
+                        // reject — so this is the best we can do here.
+                        if (config.onResult(buffer)) {
+                            matchCount++;
+                        }
                     }
                 }
             } catch (e: any) {
@@ -531,7 +550,10 @@ export class BufferIndex {
         params: SearchParams;
 
         keepIterating: () => boolean;
-        onResult: (match: Buffer) => void;
+        // See the note on `findLocal.onResult` — return value drives the
+        // per-file matchCount cap so out-of-window emits don't short-circuit
+        // the scan.
+        onResult: (match: Buffer) => boolean;
         results: IndexedLogResults;
     }): Promise<{
         blockSearchTime: number;

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -505,7 +505,10 @@ export class BufferUnitIndex {
         params: SearchParams;
         allSearchUnits: Unit[][];
         keepIterating: () => boolean;
-        onResult: (match: Buffer) => void;
+        // Returns true iff the caller actually retained the value. Drives the
+        // `matchCounts` cap below — see the comment at the `matchesPattern`
+        // call for why we can't blindly count emits.
+        onResult: (match: Buffer) => boolean;
         index: Buffer;
         reader: Reader;
         results: IndexedLogResults;
@@ -582,9 +585,24 @@ export class BufferUnitIndex {
 
                         const buffer = await this.getBufferFromBlock(blockReader, i);
                         if (matchesPattern(buffer)) {
-                            config.onResult(buffer);
-                            matchCount++;
-                            matchCounts[blockIndex]++;
+                            // Only count matches the caller actually kept.
+                            // `onResult` routes through
+                            // `FindProgressTracker.addResult`, which can reject
+                            // for reasons opaque to us — most notably
+                            // time-range filtering. Counting rejected emits
+                            // would let a stretch of out-of-window matches at
+                            // the start of a file blow the per-file cap and
+                            // short-circuit the scan before we reach the
+                            // in-window region. The cost is that we keep
+                            // matching and calling onResult through those
+                            // out-of-window blocks (mild inefficiency); we
+                            // can't skip ahead because buffers are scanned
+                            // linearly and we don't know up front which
+                            // entries the caller will reject.
+                            if (config.onResult(buffer)) {
+                                matchCount++;
+                                matchCounts[blockIndex]++;
+                            }
                         }
                     }
                 } catch (e: any) {

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -620,12 +620,28 @@ export class IndexedLogs<T> {
                     dataReader,
                     params: {
                         ...config.params,
-                        limit: config.params.limit - results.matchCount,
+                        // Per-file cap = global `limit`, NOT `limit - matchCount`.
+                        // Subtracting the running count would starve parallel
+                        // files: once one file fills matchCount, every other
+                        // in-flight file gets limit=0 and stops scanning, even
+                        // though `isSourceRelevant` may still want their earlier
+                        // matches to displace the kept top-K. Within a single
+                        // file blocks are time-ordered in the scan direction,
+                        // so capping at `limit` per file is safe — anything
+                        // past the first `limit` matches in this file couldn't
+                        // survive the top-K sort.
+                        limit: config.params.limit,
                     },
                     keepIterating: () => !results.cancel && progressTracker.isSourceRelevant(path),
-                    onResult: (match: Buffer) => {
-                        // NOTE: We absolutely cannot do any limiting here, as that would just result in a lot of values that might be newer than the old values being discarded. Each loop itself has to do some kind of limiting as it knows how many it's found in its specific loop, and it knows the direction it's iterating and the direction of the values. 
-                        progressTracker.addResult(match, path);
+                    onResult: (match: Buffer): boolean => {
+                        // Return value drives the per-file matchCount cap inside
+                        // BufferIndex.find / BufferUnitIndex.find. addResult
+                        // returns false when it rejects (time outside the
+                        // search range, or past the kept top-K cutoff); the
+                        // scanner uses that to avoid counting rejected emits
+                        // against the cap. See the comment on those scanners'
+                        // `matchesPattern` calls for the full reasoning.
+                        return progressTracker.addResult(match, path);
                     },
                     results,
                 });

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -38,7 +38,7 @@ import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
 import { formatSearchString } from "./LogViewerParams";
 
 let excludePendingResults = new URLParam("excludePendingResults", false);
-let limitURL = new URLParam("limit", 16);
+let limitURL = new URLParam("limit", 200);
 let enableLogsURL = new URLParam("enableLogs", true);
 let enableInfosURL = new URLParam("enableInfos", true);
 let enableWarningsURL = new URLParam("enableWarnings", true);
@@ -46,7 +46,6 @@ let enableErrorsURL = new URLParam("enableErrors", true);
 
 let savedPathsURL = new URLParam("savedPaths", "");
 let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
-let useRelativeTimeURL = new URLParam("useRelativeTime", true);
 
 // NOTE: Because this doesn't cache properly, it lags a lot, so we shouldn't try to use it. 
 let showLifecycleColumnURL = new URLParam("showlifecycle", false);
@@ -269,7 +268,8 @@ export class LogViewer3 extends qreact.Component {
         let updateResults = throttleFunction(100, () => {
             if (this.searchSequenceNumber !== currentSequenceNumber) return;
             Querysub.commitLocal(() => {
-                this.state.results = results;
+                // LIMIT, as the other values will be fragmented and so are confusing.
+                this.state.results = results.slice(0, limitURL.value);
             });
         });
 
@@ -495,17 +495,19 @@ export class LogViewer3 extends qreact.Component {
 
         for (let field of selectedFields) {
             let column: ColumnType<unknown, LogDatum> = {};
-            if (field === "time") {
-                column.formatter = (x: unknown) => useRelativeTimeURL.value ? formatDateJSX(Number(x)) : <span title={formatDateTimeDetailed(Number(x))}>{formatDateTime(Number(x))}</span>;
-            } else if (field === "__machineId") {
-                column.formatter = (x: unknown, context) => {
-                    if (!context?.row || !context.row.__machineId) return <ObjectDisplay value={x} />;
-                    return <MachineThreadInfo machineId={context.row.__machineId} threadId={context.row.__threadId || undefined} />;
-                };
-            }
-            if (!column.formatter) {
-                column.formatter = (x: unknown) => <ObjectDisplay value={x} />;
+            if (field === "time" || field === "timeId") {
+                column.formatter = (x: unknown) => <span title={formatDateTimeDetailed(Number(x))}>{formatDateTimeDetailed(Number(x))}</span>;
             }
+            // ObjectDisplay is too slow.
+            // else if (field === "__machineId") {
+            //     column.formatter = (x: unknown, context) => {
+            //         if (!context?.row || !context.row.__machineId) return <ObjectDisplay value={x} />;
+            //         return <MachineThreadInfo machineId={context.row.__machineId} threadId={context.row.__threadId || undefined} />;
+            //     };
+            // }
+            // if (!column.formatter) {
+            //     column.formatter = (x: unknown) => <ObjectDisplay value={x} />;
+            // }
             columns[field] = column;
         }
 
@@ -622,11 +624,6 @@ export class LogViewer3 extends qreact.Component {
                         selectedFieldsURL.value = newValues;
                     }}
                 />
-                <InputLabelURL
-                    label="Use Relative Time"
-                    checkbox
-                    url={useRelativeTimeURL}
-                />
                 <InputLabelURL
                     label="Show Lifecycles"
                     checkbox