querysub 0.460.0 → 0.462.0

package/src/deployManager/machineApplyMainCode.ts

@@ -30,8 +30,8 @@ import { fsExistsAsync } from "../fs";
 // to PIPE_FILE_LINE_LIMIT lines it is truncated down to PIPE_FILE_LINE_KEEP. The
 // file size is checked every PIPE_FILE_LINE_KEEP lines, so the file stays within
 // [PIPE_FILE_LINE_KEEP, PIPE_FILE_LINE_LIMIT] lines.
-const PIPE_FILE_LINE_LIMIT = 2_000;
-const PIPE_FILE_LINE_KEEP = 1_000;
+const PIPE_FILE_LINE_LIMIT = 10_000;
+const PIPE_FILE_LINE_KEEP = 5_000;
 
 
 const getMemoryInfo = measureWrap(async function getMemoryInfo(): Promise<{ value: number; max: number } | undefined> {

package/src/deployManager/setupMachineMain.ts

@@ -12,7 +12,32 @@ Querysub;
 
 const pinnedNodeVersion = 22;
 
-async function getGitHubApiKey(repoUrl: string, sshRemote: string): Promise<string> {
+function getGitHubKeyCachePath(repoUrl: string): string {
+    let repoOwner = "";
+    let repoName = "";
+    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
+    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
+    if (sshMatch) {
+        repoOwner = sshMatch[1];
+        repoName = sshMatch[2];
+    } else if (httpsMatch) {
+        repoOwner = httpsMatch[1];
+        repoName = httpsMatch[2];
+    }
+    return os.homedir() + `/githubkey_${repoOwner}_${repoName}.json`;
+}
+
+async function verifyGitHubApiKey(apiKey: string): Promise<boolean> {
+    const response = await fetch("https://api.github.com/user", {
+        headers: {
+            "Authorization": `Bearer ${apiKey}`,
+            "Accept": "application/vnd.github+json"
+        }
+    });
+    return response.ok;
+}
+
+async function getGitHubApiKey(repoUrl: string, sshRemote: string, forceRefresh = false): Promise<string> {
     // Parse repository info from URL
     let repoOwner = "";
     let repoName = "";
@@ -30,15 +55,18 @@ async function getGitHubApiKey(repoUrl: string, sshRemote: string): Promise<stri
         repoName = httpsMatch[2];
     }
 
-    const cacheFile = os.homedir() + `/githubkey_${repoOwner}_${repoName}.json`;
+    const cacheFile = getGitHubKeyCachePath(repoUrl);
 
     // Check if we have a cached key
-    if (await fsExistsAsync(cacheFile)) {
+    if (!forceRefresh && await fsExistsAsync(cacheFile)) {
         try {
             const cached = JSON.parse(fs.readFileSync(cacheFile, "utf8"));
             if (cached.apiKey) {
-                console.log(`✅ Using cached GitHub API key from ${cacheFile}`);
-                return cached.apiKey;
+                if (await verifyGitHubApiKey(cached.apiKey)) {
+                    console.log(`✅ Using cached GitHub API key from ${cacheFile}`);
+                    return cached.apiKey;
+                }
+                console.warn(`⚠️  Cached GitHub API key at ${cacheFile} failed verification (401 Bad credentials) — requesting a new one`);
             }
         } catch {
             // Invalid cache file, we'll ask for a new key
@@ -84,8 +112,6 @@ async function getGitHubApiKey(repoUrl: string, sshRemote: string): Promise<stri
 }
 
 async function addDeployKeyToGitHub(sshPublicKey: string, keyTitle: string, repoUrl: string, sshRemote: string): Promise<void> {
-    const apiKey = await getGitHubApiKey(repoUrl, sshRemote);
-
     // Parse repository info from URL to get owner/repo for the API endpoint
     let repoOwner = "";
     let repoName = "";
@@ -104,26 +130,42 @@ async function addDeployKeyToGitHub(sshPublicKey: string, keyTitle: string, repo
 
     let url = `https://api.github.com/repos/${repoOwner}/${repoName}/keys`;
     console.log(url);
-    const response = await fetch(url, {
-        method: "POST",
-        headers: {
-            "Authorization": `Bearer ${apiKey}`,
-            "Accept": "application/vnd.github+json",
-            "Content-Type": "application/json"
-        },
-        body: JSON.stringify({
-            title: keyTitle,
-            key: sshPublicKey,
-            read_only: true
-        })
-    });
 
-    if (!response.ok) {
+    let forceRefresh = false;
+    while (true) {
+        const apiKey = await getGitHubApiKey(repoUrl, sshRemote, forceRefresh);
+        const response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${apiKey}`,
+                "Accept": "application/vnd.github+json",
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                title: keyTitle,
+                key: sshPublicKey,
+                read_only: true
+            })
+        });
+
+        if (response.ok) {
+            console.log("✅ Deploy key added to GitHub repository");
+            return;
+        }
+
         const errorText = await response.text();
+        if (response.status === 401 && !forceRefresh) {
+            console.warn(`⚠️  GitHub API rejected credentials (401). Invalidating cached token and re-prompting.`);
+            try {
+                fs.unlinkSync(getGitHubKeyCachePath(repoUrl));
+            } catch {
+                // Cache file may not exist; ignore
+            }
+            forceRefresh = true;
+            continue;
+        }
         throw new Error(`Failed to add deploy key to GitHub repository: ${response.status} ${errorText}`);
     }
-
-    console.log("✅ Deploy key added to GitHub repository");
 }
 
 async function setupRepositoryOnRemote(sshRemote: string, gitURLLive: string, gitRefLive: string): Promise<void> {

package/src/diagnostics/charts/Chart.tsx

@@ -0,0 +1,240 @@
+import preact from "preact";
+import { qreact } from "../../4-dom/qreact";
+import { css } from "../../4-dom/css";
+import { JSXFormatter, formatValue } from "../../5-diagnostics/GenericFormat";
+import { MeasuredDiv } from "../../library-components/MeasuredDiv";
+
+export type FieldDef = {
+    title?: string;
+    /** If this value is sampled, for example, checking the current CPU usage, disk usage, etc, this field must be set to true. */
+    isSampled?: boolean;
+    isDateType?: boolean;
+    formatter?: JSXFormatter;
+};
+
+export type ChartData<T> = {
+    rows: T[];
+    columns: { [columnName in keyof T]?: FieldDef };
+};
+
+const DEFAULT_BAR_WIDTH_PX = 100;
+const Y_LABEL_TARGET_COUNT = 6;
+const Y_LABEL_MAX_WIDTH_PX = 120;
+const X_LABEL_MAX_WIDTH_PX = 140;
+const X_LABEL_FONT_SIZE_PX = 11;
+const Y_LABEL_FONT_SIZE_PX = 11;
+const GRID_LINE_COLOR = "hsla(0, 0%, 0%, 0.12)";
+const BAR_GAP_PX = 2;
+const BAR_COLOR = "hsl(210, 60%, 55%)";
+const MIN_VISIBLE_BAR_FRAC = 0.005;
+
+function getNiceNumber(num: number): number {
+    if (num <= 0) return 1;
+    let zeros = Math.floor(Math.log10(num));
+    let fraction = num / 10 ** zeros;
+    if (fraction <= 1) fraction = 1;
+    else if (fraction <= 2) fraction = 2;
+    else if (fraction <= 5) fraction = 5;
+    else fraction = 10;
+    return fraction * 10 ** zeros;
+}
+
+export class ChartBar<T> extends qreact.Component<{
+    data: ChartData<T>;
+    xAxis: keyof T;
+    yAxis: keyof T;
+    // Defaults to 100px
+    //  - is only roughly followed. At render time, we determine the width of the entire screen, and then we assume that we're going to fill the entire width, and then we divide that by this and round up, and that's how many bars we try to show. That way if they resize the window, we can utilize CSS to dynamically scale so nothing gets misaligned.
+    barWidthPx?: number;
+}> {
+    state = {
+        availableWidth: typeof window !== "undefined" ? window.innerWidth : 1200,
+    };
+
+    render() {
+        const { data, xAxis, yAxis } = this.props;
+        const barWidthPx = this.props.barWidthPx ?? DEFAULT_BAR_WIDTH_PX;
+        const rows = data.rows;
+
+        const xCol = data.columns[xAxis];
+        const yCol = data.columns[yAxis];
+        const yFormatter: JSXFormatter = yCol?.formatter ?? "guess";
+        const xFormatter: JSXFormatter = xCol?.formatter ?? (xCol?.isDateType ? "date" : "guess");
+
+        const targetBarCount = Math.max(1, Math.ceil(this.state.availableWidth / barWidthPx));
+        // Aggregate rows into at most targetBarCount bars (sum y values inside each bucket).
+        let displayBars: { xValue: unknown; yValue: number; rowsCount: number }[] = [];
+        if (rows.length <= targetBarCount) {
+            for (let r of rows) {
+                displayBars.push({ xValue: r[xAxis], yValue: Number(r[yAxis]) || 0, rowsCount: 1 });
+            }
+        } else {
+            // Bucket adjacent rows together. Each bucket gets ceil(rows.length / targetBarCount) rows.
+            let perBucket = Math.ceil(rows.length / targetBarCount);
+            for (let i = 0; i < rows.length; i += perBucket) {
+                let slice = rows.slice(i, i + perBucket);
+                let sum = 0;
+                for (let r of slice) sum += Number(r[yAxis]) || 0;
+                // Use the middle row's x value as the bucket label.
+                let mid = slice[Math.floor(slice.length / 2)];
+                displayBars.push({ xValue: mid[xAxis], yValue: sum, rowsCount: slice.length });
+            }
+        }
+
+        let maxY = 0;
+        for (let b of displayBars) {
+            if (b.yValue > maxY) maxY = b.yValue;
+        }
+        if (maxY <= 0) maxY = 1;
+        maxY = getNiceNumber(maxY);
+
+        // Build y labels from 0 to maxY, descending so top-of-chart is maxY.
+        let yLabelValues: number[] = [];
+        for (let i = 0; i < Y_LABEL_TARGET_COUNT; i++) {
+            let frac = 1 - i / (Y_LABEL_TARGET_COUNT - 1);
+            yLabelValues.push(frac * maxY);
+        }
+
+        const formatY = (v: number) => formatValue(v, yFormatter, { columnName: yAxis as string });
+        const formatX = (v: unknown) => formatValue(v, xFormatter, { columnName: xAxis as string });
+
+        const yLabelMaxWidth = `min(${Y_LABEL_MAX_WIDTH_PX}px, 15vw)` as const;
+        const xLabelMaxWidth = `min(${X_LABEL_MAX_WIDTH_PX}px, 12vw)` as const;
+
+        // Outer grid: [yLabels | chartArea] / [        | xLabels]
+        // yLabels column: shrinks to content but capped via max-width on labels.
+        // chartArea: contains horizontal grid lines (absolute) and a nested grid of bars.
+
+        // Y-axis labels: each label sits ON its grid line. We render them as a flex column
+        // with space-between and translate each by -50% so the text is centered on the line.
+        let yAxisLabelsNode = (
+            <div class={
+                css.relative.fillHeight.vbox0.justifyContent("space-between").alignEnd
+                    .paddingRight(8)
+                    .fontSize(Y_LABEL_FONT_SIZE_PX)
+            }>
+                {yLabelValues.map((v, i) => (
+                    <div class={
+                        css.relative
+                            .maxWidth(yLabelMaxWidth as any)
+                            .overflowHidden
+                            .whiteSpace("nowrap")
+                            .textOverflow("ellipsis")
+                            .textAlign("right")
+                            .top(i === 0 ? 0 : i === yLabelValues.length - 1 ? 0 : "-0.5em" as any)
+                    } title={String(v)}>
+                        {formatY(v)}
+                    </div>
+                ))}
+            </div>
+        );
+
+        // Horizontal grid lines, one per label position. Use a vertical flex with
+        // space-between, just like the labels, so they line up exactly.
+        let gridLinesNode = (
+            <div class={
+                css.absolute.pos(0, 0).fillBoth.pointerEvents("none")
+                    .vbox0.justifyContent("space-between")
+            }>
+                {yLabelValues.map(() => (
+                    <div class={css.fillWidth.height(1).background(GRID_LINE_COLOR)} />
+                ))}
+            </div>
+        );
+
+        let barsNode = (
+            <div class={
+                css.relative.fillBoth.display("grid")
+                    .gridTemplateColumns(`repeat(${displayBars.length}, 1fr)`)
+                    .alignItems("end")
+                    .columnGap(BAR_GAP_PX)
+            }>
+                {displayBars.map(bar => {
+                    let frac = bar.yValue / maxY;
+                    if (!isFinite(frac) || frac < 0) frac = 0;
+                    if (frac > 1) frac = 1;
+                    // Make tiny non-zero bars at least minimally visible.
+                    if (bar.yValue > 0 && frac < MIN_VISIBLE_BAR_FRAC) frac = MIN_VISIBLE_BAR_FRAC;
+                    let xStr = (() => {
+                        let f = formatX(bar.xValue);
+                        if (typeof f === "string") return f;
+                        if (typeof f === "number") return String(f);
+                        return String(bar.xValue);
+                    })();
+                    let yStr = (() => {
+                        let f = formatY(bar.yValue);
+                        if (typeof f === "string") return f;
+                        if (typeof f === "number") return String(f);
+                        return String(bar.yValue);
+                    })();
+                    return (
+                        <div
+                            class={
+                                css.fillWidth
+                                    .height(`${frac * 100}%` as any)
+                                    .background(BAR_COLOR)
+                                    .filter("brightness(1.15)", "hover")
+                            }
+                            title={`${xCol?.title ?? String(xAxis)}: ${xStr}\n${yCol?.title ?? String(yAxis)}: ${yStr}`}
+                        />
+                    );
+                })}
+            </div>
+        );
+
+        // X-axis labels: same column grid as bars so each label sits under its bar.
+        let xAxisLabelsNode = (
+            <div class={
+                css.display("grid")
+                    .gridTemplateColumns(`repeat(${displayBars.length}, 1fr)`)
+                    .columnGap(BAR_GAP_PX)
+                    .paddingTop(4)
+                    .fontSize(X_LABEL_FONT_SIZE_PX)
+            }>
+                {displayBars.map(bar => {
+                    let xStr = (() => {
+                        let f = formatX(bar.xValue);
+                        if (typeof f === "string") return f;
+                        if (typeof f === "number") return String(f);
+                        return String(bar.xValue);
+                    })();
+                    return (
+                        <div class={
+                            css.textAlign("center")
+                                .maxWidth(xLabelMaxWidth as any)
+                                .overflowHidden
+                                .whiteSpace("nowrap")
+                                .textOverflow("ellipsis")
+                        } title={xStr}>
+                            {formatX(bar.xValue)}
+                        </div>
+                    );
+                })}
+            </div>
+        );
+
+        return (
+            <MeasuredDiv
+                onNewSize={(width) => {
+                    if (width !== this.state.availableWidth) {
+                        this.state.availableWidth = width;
+                    }
+                }}
+                class={
+                    css.fillBoth.display("grid")
+                        .gridTemplateColumns(`auto 1fr`)
+                        .gridTemplateRows(`1fr auto`)
+                        .minHeight(200)
+                }
+            >
+                {yAxisLabelsNode}
+                <div class={css.relative.fillBoth.overflowHidden}>
+                    {gridLinesNode}
+                    {barsNode}
+                </div>
+                <div />
+                {xAxisLabelsNode}
+            </MeasuredDiv>
+        );
+    }
+}

package/src/diagnostics/grossStats/GrossStatsPage.tsx

@@ -3,9 +3,8 @@ import { qreact } from "../../4-dom/qreact";
 import { css } from "../../4-dom/css";
 import { Querysub } from "../../4-querysub/QuerysubController";
 import { SocketFunction } from "socket-function/SocketFunction";
-import { formatNumber } from "socket-function/src/formatting/format";
+import { formatDateTime, formatNumber } from "socket-function/src/formatting/format";
 import { timeInHour, timeInMinute } from "socket-function/src/misc";
-import { cacheArgsEqual } from "socket-function/src/caching";
 import { t } from "../../2-proxy/schema2";
 import { getSyncedController } from "../../library-components/SyncedController";
 import { GrossStatsController } from "./GrossStatsController";
@@ -14,12 +13,12 @@ import {
     GrossStatsField,
     GrossStatsBucket,
 } from "../../5-diagnostics/gross-stats/grossStats";
+import { ChartBar, ChartData } from "../charts/Chart";
 export { GrossStatsController } from "./GrossStatsController";
 
 module.hotreload = true;
 
 const REFRESH_INTERVAL_MS = 5 * timeInMinute;
-const CHART_WIDTH = 1200;
 const CHART_HEIGHT = 300;
 
 const TIME_RANGES: { label: string; ms: number }[] = [
@@ -58,65 +57,33 @@ function shortNodeId(nodeId: string): string {
 
 let grossStatsController = getSyncedController(GrossStatsController);
 
-// Render a stacked bar chart to a PNG data URL. Cached by argument identity:
-//  selectedField/rangeMs/width/height are primitives; nodeIds and bucketsArrays are
-//  expected to be stable references (the bucketsArrays come from syncedController and
-//  only change when refreshed).
-const renderChartPNG = cacheArgsEqual((
+type ChartRow = { time: number; value: number };
+
+function buildChartRows(
     selectedField: GrossStatsField,
     rangeMs: number,
-    width: number,
-    height: number,
-    nodeIds: readonly string[],
-    bucketsArrays: readonly GrossStatsBucket[][],
-): { pngUrl: string; maxTotal: number } => {
-    let canvas = document.createElement("canvas");
-    canvas.width = width;
-    canvas.height = height;
-    let ctx = canvas.getContext("2d")!;
-    ctx.fillStyle = "hsl(0, 0%, 97%)";
-    ctx.fillRect(0, 0, width, height);
-
+    bucketsArrays: GrossStatsBucket[][],
+): { rows: ChartRow[]; peak: number } {
     let now = Date.now();
     let windowStart = now - rangeMs;
-
-    let perPixel: { perNode: number[]; total: number; }[] = [];
-    for (let x = 0; x < width; x++) {
-        perPixel.push({ perNode: new Array(nodeIds.length).fill(0), total: 0 });
-    }
-    let maxTotal = 0;
-    for (let i = 0; i < nodeIds.length; i++) {
-        let buckets = bucketsArrays[i];
+    // Sum across selected nodes, keyed by bucket time (each bucket is one minute).
+    let sumByTime = new Map<number, number>();
+    for (let buckets of bucketsArrays) {
         if (!buckets) continue;
         for (let bucket of buckets) {
             if (bucket.time < windowStart) continue;
-            let xFloat = (bucket.time - windowStart) / rangeMs * width;
-            let x = Math.min(width - 1, Math.max(0, Math.floor(xFloat)));
-            let cell = perPixel[x];
-            let val = bucket.deltas[selectedField] || 0;
-            cell.perNode[i] += val;
-            cell.total += val;
-            if (cell.total > maxTotal) maxTotal = cell.total;
+            let v = bucket.deltas[selectedField] || 0;
+            sumByTime.set(bucket.time, (sumByTime.get(bucket.time) || 0) + v);
         }
     }
-    if (maxTotal <= 0) maxTotal = 1;
-
-    for (let x = 0; x < width; x++) {
-        let cell = perPixel[x];
-        if (cell.total === 0) continue;
-        let y = height;
-        for (let i = 0; i < nodeIds.length; i++) {
-            let val = cell.perNode[i];
-            if (!val) continue;
-            let segHeight = (val / maxTotal) * height;
-            ctx.fillStyle = colorForNode(i);
-            ctx.fillRect(x, y - segHeight, 1, segHeight);
-            y -= segHeight;
-        }
+    let times = Array.from(sumByTime.keys()).sort((a, b) => a - b);
+    let rows: ChartRow[] = times.map(t => ({ time: t, value: sumByTime.get(t)! }));
+    let peak = 0;
+    for (let r of rows) {
+        if (r.value > peak) peak = r.value;
     }
-
-    return { pngUrl: canvas.toDataURL(), maxTotal };
-}, 5);
+    return { rows, peak };
+}
 
 export class GrossStatsPage extends qreact.Component {
     refreshTimer: ReturnType<typeof setInterval> | undefined;
@@ -137,7 +104,7 @@ export class GrossStatsPage extends qreact.Component {
                 <span>Range:</span>
                 {TIME_RANGES.map(r =>
                     <button
-                        className={css.pad2(8, 4) + (state.rangeMs === r.ms ? " " + css.hsl(210, 70, 60).hslcolor(0, 0, 100).bold : "")}
+                        className={css.pad2(8, 4) + (state.rangeMs === r.ms ? " " + css.hsl(210, 70, 60).hslcolor(0, 0, 100).boldStyle : "")}
                         onClick={() => Querysub.commit(() => { state.rangeMs = r.ms; })}
                     >{r.label}</button>
                 )}
@@ -146,7 +113,7 @@ export class GrossStatsPage extends qreact.Component {
                 <span>Field:</span>
                 {GROSS_STATS_FIELDS.map(f =>
                     <button
-                        className={css.pad2(8, 4) + (state.selectedField === f ? " " + css.hsl(140, 60, 50).hslcolor(0, 0, 100).bold : "")}
+                        className={css.pad2(8, 4) + (state.selectedField === f ? " " + css.hsl(140, 60, 50).hslcolor(0, 0, 100).boldStyle : "")}
                         onClick={() => Querysub.commit(() => { state.selectedField = f; })}
                     >{f}</button>
                 )}
@@ -254,44 +221,42 @@ export class GrossStatsPage extends qreact.Component {
         let bucketsArrays: GrossStatsBucket[][] = selectedNodeIds.map(n => bucketsByNode.get(n) ?? []);
         let anyLoading = !result;
 
-        let chart = renderChartPNG(
-            state.selectedField,
-            state.rangeMs,
-            CHART_WIDTH,
-            CHART_HEIGHT,
-            selectedNodeIds,
-            bucketsArrays,
-        );
-        let now = Date.now();
-        let windowStart = now - state.rangeMs;
+        let { rows, peak } = buildChartRows(state.selectedField, state.rangeMs, bucketsArrays);
+
+        let chartData: ChartData<ChartRow> = {
+            rows,
+            columns: {
+                time: {
+                    title: "Time",
+                    isDateType: true,
+                    formatter: (v) => formatDateTime(Number(v)),
+                },
+                value: {
+                    title: state.selectedField,
+                    formatter: (v) => formatNumber(Number(v)),
+                },
+            },
+        };
 
         return <div className={css.vbox(8).pad2(8).fillWidth}>
             <h2>Cluster Stats</h2>
             {this.renderControls(allNodeIds)}
-            <div className={css.vbox(4)}>
-                <div className={css.hbox(8).bold}>
-                    <span>peak: {formatNumber(chart.maxTotal)} {state.selectedField} / minute</span>
+            <div className={css.vbox(4).fillWidth}>
+                <div className={css.hbox(8).boldStyle}>
+                    <span>peak: {formatNumber(peak)} {state.selectedField} / minute</span>
                 </div>
-                <img
-                    src={chart.pngUrl}
-                    width={CHART_WIDTH}
-                    height={CHART_HEIGHT}
-                    className={css.hsl(0, 0, 100)}
+                <div
+                    className={css.fillWidth.height(CHART_HEIGHT).hsl(0, 0, 100)}
                     style={anyLoading ? { opacity: 0.5 } : undefined}
-                />
-                <div className={css.hbox(0).fillWidth}>
-                    <span className={css.flexShrink0}>{formatLocalTime(windowStart)}</span>
-                    <span className={css.fillBoth}></span>
-                    <span className={css.flexShrink0}>{formatLocalTime(now)}</span>
+                >
+                    <ChartBar<ChartRow>
+                        data={chartData}
+                        xAxis="time"
+                        yAxis="value"
+                    />
                 </div>
             </div>
             {this.renderTable(selectedNodeIds, bucketsArrays)}
         </div>;
     }
 }
-
-function formatLocalTime(ms: number): string {
-    let d = new Date(ms);
-    let pad = (n: number) => String(n).padStart(2, "0");
-    return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
-}

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -286,11 +286,11 @@ export class IndexedLogs<T> {
             let added = false;
             const tryAdd = async (nodeId: string, preferredOnly = false) => {
                 if (added) return;
-                let hasLogger = await timeoutToUndefinedSilent(2500, IndexedLogShimController.nodes[nodeId].hasLogger(this.config.name));
+                let hasLogger = await timeoutToUndefinedSilent(10_000, IndexedLogShimController.nodes[nodeId].hasLogger(this.config.name));
                 if (!hasLogger) return false;
                 // NOTE: Prefer to do the searching on the move logs service. However, if it's not available, any service can do searching. It just might lag that server... 
                 if (preferredOnly) {
-                    let metadata = await timeoutToUndefinedSilent(2500, NodeCapabilitiesController.nodes[nodeId].getMetadata());
+                    let metadata = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[nodeId].getMetadata());
                     if (!metadata?.entryPoint.includes("movelogs")) return false;
                 }
                 added = true;
@@ -400,13 +400,13 @@ export class IndexedLogs<T> {
         onResult: (match: T) => void;
         onResults?: (results: IndexedLogResults) => Promise<boolean>;
     }): Promise<IndexedLogResults> {
-
         let { params } = config;
         if (params.pathOverrides && config.params.only === "public") {
             // Fine, if they provided path overrides, and they are all public, just read them, as they'll resolve the same no matter where we read from
         } else if (config.params.forceReadProduction && !isPublic()) {
             let machineNodes = await this.getMachineNodes();
             if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
+            console.log(`Picking machine node ${machineNodes[0]} to read from`);
             return await this.clientFind({
                 ...config,
                 nodeId: machineNodes[0],

package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts

@@ -157,6 +157,7 @@ export class MCPIndexedLogs {
         direction: Direction;
         columns: string[];
         limit?: number;
+        forceRefresh?: boolean;
     }): Promise<SearchResult> {
         let limit = config.limit ?? 100;
         let startTime = normalizeTime(config.startTime, "startTime");
@@ -176,7 +177,7 @@ export class MCPIndexedLogs {
         let machineId = config.machine === "local" ? getOwnMachineId() : config.machine;
 
         let moveStart = Date.now();
-        let moveOutcome = await this.ensureMovedThrough(machineId, endTime);
+        let moveOutcome = await this.ensureMovedThrough(machineId, endTime, config.forceRefresh);
         console.log(`[search] ensureMovedThrough ${moveOutcome} in ${formatTime(Date.now() - moveStart)}`);
 
         let loggers = await getLoggers2Async();
@@ -203,6 +204,7 @@ export class MCPIndexedLogs {
                 loggerName,
                 startTime,
                 endTime,
+                forceRefresh: config.forceRefresh,
             });
             totalPathsSeen += paths.length;
 
@@ -479,9 +481,9 @@ export class MCPIndexedLogs {
     // (e.g. older versions still running). Records moved-through up to
     // now - MOVE_GRACE so we skip this on subsequent calls covering the same
     // window.
-    private async ensureMovedThrough(machineId: string, endTime: number): Promise<"cached" | "no-node" | "moved"> {
+    private async ensureMovedThrough(machineId: string, endTime: number, forceRefresh?: boolean): Promise<"cached" | "no-node" | "moved"> {
         let lastMoved = this.movedThroughByMachine.get(machineId) ?? 0;
-        if (lastMoved >= endTime) return "cached";
+        if (!forceRefresh && lastMoved >= endTime) return "cached";
 
         let nodeIds = await this.findRemoteNodesOnMachine(machineId);
         if (nodeIds.length === 0) {
@@ -493,6 +495,14 @@ export class MCPIndexedLogs {
             let answered = false;
             for (let nodeId of nodeIds) {
                 try {
+                    if (forceRefresh) {
+                        console.log(`MCPIndexedLogs: forceRefresh — flushing ${loggerName} on ${nodeId} unconditionally`);
+                        await IndexedLogShimController.nodes[nodeId].forceMoveLogsToPublic({
+                            indexedLogsName: loggerName,
+                        });
+                        answered = true;
+                        break;
+                    }
                     let hasPending = await timeoutToUndefinedSilent(
                         5000,
                         IndexedLogShimController.nodes[nodeId].hasPendingInRange({
@@ -564,6 +574,7 @@ export class MCPIndexedLogs {
         loggerName: LoggerName;
         startTime: number;
         endTime: number;
+        forceRefresh?: boolean;
     }): Promise<TimeFilePath[]> {
         let bucketStart = Math.floor(config.startTime / timeInHour) * timeInHour;
         let bucketEnd = Math.ceil(config.endTime / timeInHour) * timeInHour;
@@ -575,6 +586,10 @@ export class MCPIndexedLogs {
             if (now - v.time > PATHS_CACHE_TTL) this.pathsCache.delete(k);
         }
 
+        if (config.forceRefresh) {
+            this.pathsCache.delete(key);
+        }
+
         let cached = this.pathsCache.get(key);
         if (cached && now - cached.time <= PATHS_CACHE_TTL) {
             return cached.paths;