querysub 0.459.0 → 0.461.0

package/src/diagnostics/grossStats/GrossStatsPage.tsx

@@ -3,9 +3,8 @@ import { qreact } from "../../4-dom/qreact";
 import { css } from "../../4-dom/css";
 import { Querysub } from "../../4-querysub/QuerysubController";
 import { SocketFunction } from "socket-function/SocketFunction";
-import { formatNumber } from "socket-function/src/formatting/format";
+import { formatDateTime, formatNumber } from "socket-function/src/formatting/format";
 import { timeInHour, timeInMinute } from "socket-function/src/misc";
-import { cacheArgsEqual } from "socket-function/src/caching";
 import { t } from "../../2-proxy/schema2";
 import { getSyncedController } from "../../library-components/SyncedController";
 import { GrossStatsController } from "./GrossStatsController";
@@ -14,12 +13,12 @@ import {
     GrossStatsField,
     GrossStatsBucket,
 } from "../../5-diagnostics/gross-stats/grossStats";
+import { ChartBar, ChartData } from "../charts/Chart";
 export { GrossStatsController } from "./GrossStatsController";
 
 module.hotreload = true;
 
 const REFRESH_INTERVAL_MS = 5 * timeInMinute;
-const CHART_WIDTH = 1200;
 const CHART_HEIGHT = 300;
 
 const TIME_RANGES: { label: string; ms: number }[] = [
@@ -58,65 +57,33 @@ function shortNodeId(nodeId: string): string {
 
 let grossStatsController = getSyncedController(GrossStatsController);
 
-// Render a stacked bar chart to a PNG data URL. Cached by argument identity:
-//  selectedField/rangeMs/width/height are primitives; nodeIds and bucketsArrays are
-//  expected to be stable references (the bucketsArrays come from syncedController and
-//  only change when refreshed).
-const renderChartPNG = cacheArgsEqual((
+type ChartRow = { time: number; value: number };
+
+function buildChartRows(
     selectedField: GrossStatsField,
     rangeMs: number,
-    width: number,
-    height: number,
-    nodeIds: readonly string[],
-    bucketsArrays: readonly GrossStatsBucket[][],
-): { pngUrl: string; maxTotal: number } => {
-    let canvas = document.createElement("canvas");
-    canvas.width = width;
-    canvas.height = height;
-    let ctx = canvas.getContext("2d")!;
-    ctx.fillStyle = "hsl(0, 0%, 97%)";
-    ctx.fillRect(0, 0, width, height);
-
+    bucketsArrays: GrossStatsBucket[][],
+): { rows: ChartRow[]; peak: number } {
     let now = Date.now();
     let windowStart = now - rangeMs;
-
-    let perPixel: { perNode: number[]; total: number; }[] = [];
-    for (let x = 0; x < width; x++) {
-        perPixel.push({ perNode: new Array(nodeIds.length).fill(0), total: 0 });
-    }
-    let maxTotal = 0;
-    for (let i = 0; i < nodeIds.length; i++) {
-        let buckets = bucketsArrays[i];
+    // Sum across selected nodes, keyed by bucket time (each bucket is one minute).
+    let sumByTime = new Map<number, number>();
+    for (let buckets of bucketsArrays) {
         if (!buckets) continue;
         for (let bucket of buckets) {
             if (bucket.time < windowStart) continue;
-            let xFloat = (bucket.time - windowStart) / rangeMs * width;
-            let x = Math.min(width - 1, Math.max(0, Math.floor(xFloat)));
-            let cell = perPixel[x];
-            let val = bucket.deltas[selectedField] || 0;
-            cell.perNode[i] += val;
-            cell.total += val;
-            if (cell.total > maxTotal) maxTotal = cell.total;
+            let v = bucket.deltas[selectedField] || 0;
+            sumByTime.set(bucket.time, (sumByTime.get(bucket.time) || 0) + v);
         }
     }
-    if (maxTotal <= 0) maxTotal = 1;
-
-    for (let x = 0; x < width; x++) {
-        let cell = perPixel[x];
-        if (cell.total === 0) continue;
-        let y = height;
-        for (let i = 0; i < nodeIds.length; i++) {
-            let val = cell.perNode[i];
-            if (!val) continue;
-            let segHeight = (val / maxTotal) * height;
-            ctx.fillStyle = colorForNode(i);
-            ctx.fillRect(x, y - segHeight, 1, segHeight);
-            y -= segHeight;
-        }
+    let times = Array.from(sumByTime.keys()).sort((a, b) => a - b);
+    let rows: ChartRow[] = times.map(t => ({ time: t, value: sumByTime.get(t)! }));
+    let peak = 0;
+    for (let r of rows) {
+        if (r.value > peak) peak = r.value;
     }
-
-    return { pngUrl: canvas.toDataURL(), maxTotal };
-}, 5);
+    return { rows, peak };
+}
 
 export class GrossStatsPage extends qreact.Component {
     refreshTimer: ReturnType<typeof setInterval> | undefined;
@@ -137,7 +104,7 @@ export class GrossStatsPage extends qreact.Component {
                 <span>Range:</span>
                 {TIME_RANGES.map(r =>
                     <button
-                        className={css.pad2(8, 4) + (state.rangeMs === r.ms ? " " + css.hsl(210, 70, 60).hslcolor(0, 0, 100).bold : "")}
+                        className={css.pad2(8, 4) + (state.rangeMs === r.ms ? " " + css.hsl(210, 70, 60).hslcolor(0, 0, 100).boldStyle : "")}
                         onClick={() => Querysub.commit(() => { state.rangeMs = r.ms; })}
                     >{r.label}</button>
                 )}
@@ -146,7 +113,7 @@ export class GrossStatsPage extends qreact.Component {
                 <span>Field:</span>
                 {GROSS_STATS_FIELDS.map(f =>
                     <button
-                        className={css.pad2(8, 4) + (state.selectedField === f ? " " + css.hsl(140, 60, 50).hslcolor(0, 0, 100).bold : "")}
+                        className={css.pad2(8, 4) + (state.selectedField === f ? " " + css.hsl(140, 60, 50).hslcolor(0, 0, 100).boldStyle : "")}
                         onClick={() => Querysub.commit(() => { state.selectedField = f; })}
                     >{f}</button>
                 )}
@@ -254,44 +221,42 @@ export class GrossStatsPage extends qreact.Component {
         let bucketsArrays: GrossStatsBucket[][] = selectedNodeIds.map(n => bucketsByNode.get(n) ?? []);
         let anyLoading = !result;
 
-        let chart = renderChartPNG(
-            state.selectedField,
-            state.rangeMs,
-            CHART_WIDTH,
-            CHART_HEIGHT,
-            selectedNodeIds,
-            bucketsArrays,
-        );
-        let now = Date.now();
-        let windowStart = now - state.rangeMs;
+        let { rows, peak } = buildChartRows(state.selectedField, state.rangeMs, bucketsArrays);
+
+        let chartData: ChartData<ChartRow> = {
+            rows,
+            columns: {
+                time: {
+                    title: "Time",
+                    isDateType: true,
+                    formatter: (v) => formatDateTime(Number(v)),
+                },
+                value: {
+                    title: state.selectedField,
+                    formatter: (v) => formatNumber(Number(v)),
+                },
+            },
+        };
 
         return <div className={css.vbox(8).pad2(8).fillWidth}>
             <h2>Cluster Stats</h2>
             {this.renderControls(allNodeIds)}
-            <div className={css.vbox(4)}>
-                <div className={css.hbox(8).bold}>
-                    <span>peak: {formatNumber(chart.maxTotal)} {state.selectedField} / minute</span>
+            <div className={css.vbox(4).fillWidth}>
+                <div className={css.hbox(8).boldStyle}>
+                    <span>peak: {formatNumber(peak)} {state.selectedField} / minute</span>
                 </div>
-                <img
-                    src={chart.pngUrl}
-                    width={CHART_WIDTH}
-                    height={CHART_HEIGHT}
-                    className={css.hsl(0, 0, 100)}
+                <div
+                    className={css.fillWidth.height(CHART_HEIGHT).hsl(0, 0, 100)}
                     style={anyLoading ? { opacity: 0.5 } : undefined}
-                />
-                <div className={css.hbox(0).fillWidth}>
-                    <span className={css.flexShrink0}>{formatLocalTime(windowStart)}</span>
-                    <span className={css.fillBoth}></span>
-                    <span className={css.flexShrink0}>{formatLocalTime(now)}</span>
+                >
+                    <ChartBar<ChartRow>
+                        data={chartData}
+                        xAxis="time"
+                        yAxis="value"
+                    />
                 </div>
             </div>
             {this.renderTable(selectedNodeIds, bucketsArrays)}
         </div>;
     }
 }
-
-function formatLocalTime(ms: number): string {
-    let d = new Date(ms);
-    let pad = (n: number) => String(n).padStart(2, "0");
-    return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
-}

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -300,9 +300,6 @@ export class BufferIndex {
         dataReader: Reader;
         params: SearchParams;
         keepIterating: () => boolean;
-        // Returns true iff the caller actually retained the value. We use that
-        // to drive the per-file matchCount cap below: see the note at the
-        // `matchesPattern(buffer)` call for why we can't blindly count emits.
         onResult: (match: Buffer) => boolean;
         results: IndexedLogResults;
         allSearchUnits: Unit[][];
@@ -331,7 +328,20 @@ export class BufferIndex {
             }, `BufferIndex|readLocalBlocks`);
         });
 
-        let matchCount = 0;
+        // NOTE: The per-file matchCount cap (commented out below in both the
+        // block loop and the inner buffer loop) is intentionally disabled.
+        // We tested (see test.ts) and confirmed that blocks within a file
+        // aren't time-ordered — the move-to-public pipeline can leave a
+        // late-index block holding earlier-time entries than earlier-index
+        // blocks, and buffers within a single block aren't time-ordered
+        // either. The old `matchCount >= params.limit` short-circuit assumed
+        // ordering and was silently dropping blocks whose entries would have
+        // survived the top-K trim (broad queries lost their earliest
+        // matches). Without the cap we scan every candidate block per file,
+        // but the index pre-filter bounds the work and it isn't measurably
+        // slower in practice. If blocks/buffers ever get written in
+        // guaranteed time order this code can be re-enabled.
+        // let matchCount = 0;
         let blockSearchTimeStart = Date.now();
 
         results.totalBlockCount += indexEntries.length;
@@ -344,7 +354,8 @@ export class BufferIndex {
         const step = iterateForward ? 1 : -1;
 
         for (let i = startIdx; iterateForward ? i < endIdx : i > endIdx; i += step) {
-            if (matchCount >= params.limit || !config.keepIterating()) break;
+            // if (matchCount >= params.limit || !config.keepIterating()) break;
+            if (!config.keepIterating()) break;
             await config.results.limitGroup?.wait();
             const blockIndex = i;
 
@@ -393,38 +404,16 @@ export class BufferIndex {
                 const bufferStep = iterateForward ? 1 : -1;
 
                 for (let bufferIndex = bufferStartIdx; iterateForward ? bufferIndex < bufferEndIdx : bufferIndex > bufferEndIdx; bufferIndex += bufferStep) {
-                    // No `matchCount >= params.limit` cap inside the block.
-                    // Buffer order within a block is not guaranteed to follow
-                    // the search direction (blocks are time-ordered, buffers
-                    // inside them are not), so stopping mid-block on a match
-                    // count would drop earlier-time buffers we haven't reached
-                    // yet. The block-level cap above is the only safe stop;
-                    // here we only honor cross-file `keepIterating` (which
-                    // applies to the whole file at once, so it's safe at any
-                    // granularity).
+                    // See the note above the outer loop for why the
+                    // matchCount-based stop is gone.
+                    // if (matchCount >= params.limit || !config.keepIterating()) break;
                     if (!config.keepIterating()) break;
                     await config.results.limitGroup?.wait();
 
                     let buffer = buffers[bufferIndex];
                     if (matchesPattern(buffer)) {
-                        // Only count matches the caller actually kept. `onResult`
-                        // routes through `FindProgressTracker.addResult`, which
-                        // can reject for reasons we don't see from here — most
-                        // notably time-range filtering (an entry whose time is
-                        // outside the search window matched the text pattern
-                        // but isn't a real hit). Counting rejected emits would
-                        // let a stretch of out-of-window matches at the start
-                        // of a file blow the per-file cap and short-circuit the
-                        // scan before we reach the in-window region.
-                        //
-                        // The cost is that we keep matching and calling onResult
-                        // through those out-of-window blocks (mild inefficiency).
-                        // We can't skip ahead — buffers are scanned linearly and
-                        // we don't know up front which entries the caller will
-                        // reject — so this is the best we can do here.
-                        if (config.onResult(buffer)) {
-                            matchCount++;
-                        }
+                        config.onResult(buffer);
+                        // matchCount++;
                     }
                 }
             } catch (e: any) {
@@ -559,9 +548,7 @@ export class BufferIndex {
         params: SearchParams;
 
         keepIterating: () => boolean;
-        // See the note on `findLocal.onResult` — return value drives the
-        // per-file matchCount cap so out-of-window emits don't short-circuit
-        // the scan.
+        // Return value is unused — see `findLocal.onResult`.
         onResult: (match: Buffer) => boolean;
         results: IndexedLogResults;
     }): Promise<{

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -505,9 +505,6 @@ export class BufferUnitIndex {
         params: SearchParams;
         allSearchUnits: Unit[][];
         keepIterating: () => boolean;
-        // Returns true iff the caller actually retained the value. Drives the
-        // `matchCounts` cap below — see the comment at the `matchesPattern`
-        // call for why we can't blindly count emits.
         onResult: (match: Buffer) => boolean;
         index: Buffer;
         reader: Reader;
@@ -534,26 +531,42 @@ export class BufferUnitIndex {
         // Read blocks and search for matches
         let blockSearchTimeStart = Date.now();
         await measureBlock(async () => {
-            let matchCount = 0;
-            let matchCounts = list(blockCount).fill(0);
+            // NOTE: The matchCount / matchCounts tracking and the
+            // `stopIterating` cap below are commented out, not deleted.
+            // We tested (see test.ts) and confirmed that blocks within a
+            // file aren't actually time-ordered — the move-to-public
+            // pipeline can leave a late-index block holding earlier-time
+            // entries than earlier-index blocks. The old `relevantCount
+            // >= params.limit` short-circuit assumed time-ordered blocks
+            // and was silently skipping blocks whose entries would have
+            // survived the top-K trim (broad queries lost their earliest
+            // matches). The same applied to the inner-buffer cap (buffers
+            // within a block aren't time-ordered either). Removing both
+            // caps means we scan every candidate block per file, but the
+            // index pre-filter bounds the work and it isn't measurably
+            // slower in practice. If blocks ever get written in
+            // guaranteed time order, this code can be re-enabled.
+            // let matchCount = 0;
+            // let matchCounts = list(blockCount).fill(0);
 
             const searchBlock = async (blockIndex: number) => {
                 if (!candidateBlocksSet.has(blockIndex)) return;
-                // Check if we should stop iterating based on match counts and direction
-                let stopIterating = () => {
-                    let relevantCount = 0;
-                    if (params.searchFromStart) {
-                        for (let i = 0; i <= blockIndex; i++) {
-                            relevantCount += matchCounts[i];
-                        }
-                    } else {
-                        for (let i = blockIndex; i < blockCount; i++) {
-                            relevantCount += matchCounts[i];
-                        }
-                    }
-                    return relevantCount >= params.limit || !keepIterating();
-                };
-                if (stopIterating()) return;
+                // // Check if we should stop iterating based on match counts and direction
+                // let stopIterating = () => {
+                //     let relevantCount = 0;
+                //     if (params.searchFromStart) {
+                //         for (let i = 0; i <= blockIndex; i++) {
+                //             relevantCount += matchCounts[i];
+                //         }
+                //     } else {
+                //         for (let i = blockIndex; i < blockCount; i++) {
+                //             relevantCount += matchCounts[i];
+                //         }
+                //     }
+                //     return relevantCount >= params.limit || !keepIterating();
+                // };
+                // if (stopIterating()) return;
+                if (!keepIterating()) return;
 
                 let debugOffsets = {
                     startOffset: 0,
@@ -580,38 +593,17 @@ export class BufferUnitIndex {
                     const step = iterateForward ? 1 : -1;
 
                     for (let i = startIdx; iterateForward ? i < endIdx : i > endIdx; i += step) {
-                        // No matchCount-based cap inside the block. Buffer
-                        // order within a block is not guaranteed to follow the
-                        // search direction (blocks are time-ordered, buffers
-                        // inside them are not), so a mid-block stop on
-                        // `relevantCount >= limit` would drop earlier-time
-                        // buffers we haven't reached yet. Block-level
-                        // `stopIterating` is the safe granularity; here we
-                        // only honor cross-file `keepIterating`, which applies
-                        // to the whole file at once.
+                        // See the note at the top of this function for why
+                        // the matchCount-based stop is gone (commented out).
+                        // if (stopIterating()) break;
                         if (!keepIterating()) break;
                         await results.limitGroup?.wait();
 
                         const buffer = await this.getBufferFromBlock(blockReader, i);
                         if (matchesPattern(buffer)) {
-                            // Only count matches the caller actually kept.
-                            // `onResult` routes through
-                            // `FindProgressTracker.addResult`, which can reject
-                            // for reasons opaque to us — most notably
-                            // time-range filtering. Counting rejected emits
-                            // would let a stretch of out-of-window matches at
-                            // the start of a file blow the per-file cap and
-                            // short-circuit the scan before we reach the
-                            // in-window region. The cost is that we keep
-                            // matching and calling onResult through those
-                            // out-of-window blocks (mild inefficiency); we
-                            // can't skip ahead because buffers are scanned
-                            // linearly and we don't know up front which
-                            // entries the caller will reject.
-                            if (config.onResult(buffer)) {
-                                matchCount++;
-                                matchCounts[blockIndex]++;
-                            }
+                            config.onResult(buffer);
+                            // matchCount++;
+                            // matchCounts[blockIndex]++;
                         }
                     }
                 } catch (e: any) {

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -286,11 +286,11 @@ export class IndexedLogs<T> {
             let added = false;
             const tryAdd = async (nodeId: string, preferredOnly = false) => {
                 if (added) return;
-                let hasLogger = await timeoutToUndefinedSilent(2500, IndexedLogShimController.nodes[nodeId].hasLogger(this.config.name));
+                let hasLogger = await timeoutToUndefinedSilent(10_000, IndexedLogShimController.nodes[nodeId].hasLogger(this.config.name));
                 if (!hasLogger) return false;
                 // NOTE: Prefer to do the searching on the move logs service. However, if it's not available, any service can do searching. It just might lag that server... 
                 if (preferredOnly) {
-                    let metadata = await timeoutToUndefinedSilent(2500, NodeCapabilitiesController.nodes[nodeId].getMetadata());
+                    let metadata = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[nodeId].getMetadata());
                     if (!metadata?.entryPoint.includes("movelogs")) return false;
                 }
                 added = true;
@@ -400,13 +400,13 @@ export class IndexedLogs<T> {
         onResult: (match: T) => void;
         onResults?: (results: IndexedLogResults) => Promise<boolean>;
     }): Promise<IndexedLogResults> {
-
         let { params } = config;
         if (params.pathOverrides && config.params.only === "public") {
             // Fine, if they provided path overrides, and they are all public, just read them, as they'll resolve the same no matter where we read from
         } else if (config.params.forceReadProduction && !isPublic()) {
             let machineNodes = await this.getMachineNodes();
             if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
+            console.log(`Picking machine node ${machineNodes[0]} to read from`);
             return await this.clientFind({
                 ...config,
                 nodeId: machineNodes[0],

package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts

@@ -157,6 +157,7 @@ export class MCPIndexedLogs {
         direction: Direction;
         columns: string[];
         limit?: number;
+        forceRefresh?: boolean;
     }): Promise<SearchResult> {
         let limit = config.limit ?? 100;
         let startTime = normalizeTime(config.startTime, "startTime");
@@ -176,7 +177,7 @@ export class MCPIndexedLogs {
         let machineId = config.machine === "local" ? getOwnMachineId() : config.machine;
 
         let moveStart = Date.now();
-        let moveOutcome = await this.ensureMovedThrough(machineId, endTime);
+        let moveOutcome = await this.ensureMovedThrough(machineId, endTime, config.forceRefresh);
         console.log(`[search] ensureMovedThrough ${moveOutcome} in ${formatTime(Date.now() - moveStart)}`);
 
         let loggers = await getLoggers2Async();
@@ -203,6 +204,7 @@ export class MCPIndexedLogs {
                 loggerName,
                 startTime,
                 endTime,
+                forceRefresh: config.forceRefresh,
             });
             totalPathsSeen += paths.length;
 
@@ -479,9 +481,9 @@ export class MCPIndexedLogs {
     // (e.g. older versions still running). Records moved-through up to
     // now - MOVE_GRACE so we skip this on subsequent calls covering the same
     // window.
-    private async ensureMovedThrough(machineId: string, endTime: number): Promise<"cached" | "no-node" | "moved"> {
+    private async ensureMovedThrough(machineId: string, endTime: number, forceRefresh?: boolean): Promise<"cached" | "no-node" | "moved"> {
         let lastMoved = this.movedThroughByMachine.get(machineId) ?? 0;
-        if (lastMoved >= endTime) return "cached";
+        if (!forceRefresh && lastMoved >= endTime) return "cached";
 
         let nodeIds = await this.findRemoteNodesOnMachine(machineId);
         if (nodeIds.length === 0) {
@@ -493,6 +495,14 @@ export class MCPIndexedLogs {
             let answered = false;
             for (let nodeId of nodeIds) {
                 try {
+                    if (forceRefresh) {
+                        console.log(`MCPIndexedLogs: forceRefresh — flushing ${loggerName} on ${nodeId} unconditionally`);
+                        await IndexedLogShimController.nodes[nodeId].forceMoveLogsToPublic({
+                            indexedLogsName: loggerName,
+                        });
+                        answered = true;
+                        break;
+                    }
                     let hasPending = await timeoutToUndefinedSilent(
                         5000,
                         IndexedLogShimController.nodes[nodeId].hasPendingInRange({
@@ -564,6 +574,7 @@ export class MCPIndexedLogs {
         loggerName: LoggerName;
         startTime: number;
         endTime: number;
+        forceRefresh?: boolean;
     }): Promise<TimeFilePath[]> {
         let bucketStart = Math.floor(config.startTime / timeInHour) * timeInHour;
         let bucketEnd = Math.ceil(config.endTime / timeInHour) * timeInHour;
@@ -575,6 +586,10 @@ export class MCPIndexedLogs {
             if (now - v.time > PATHS_CACHE_TTL) this.pathsCache.delete(k);
         }
 
+        if (config.forceRefresh) {
+            this.pathsCache.delete(key);
+        }
+
         let cached = this.pathsCache.get(key);
         if (cached && now - cached.time <= PATHS_CACHE_TTL) {
             return cached.paths;

package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogsEntry.ts

@@ -56,6 +56,7 @@ Note: each segment between operators ideally has at least 4 contiguous character
                 columns: { type: "array", items: { type: "string" }, description: "Which fields to project onto each row. Use [] to get just metadata; use allColumns from a prior result to pick more." },
                 limit: { type: "number", default: 100 },
                 logTypes: { type: "string", description: "Optional pipe-separated list restricting which log streams to scan. Allowed values: log, info, warn, error. Examples: \"warn|error\", \"log\". Omit (default) to search all four." },
+                forceRefresh: { type: "boolean", description: "If true, bypass the path-cache (TimeFileTree.findAllPaths) and re-walk the archive folders. Use this when recent log files appear missing because the cache is stale." },
             },
             required: ["query", "machine", "startTime", "endTime", "direction", "columns"],
         },