querysub 0.456.0 → 0.458.0

package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts

@@ -1,4 +1,4 @@
-import { timeInHour, timeInMinute, timeInSecond, timeoutToError, timeoutToUndefined, timeoutToUndefinedSilent } from "socket-function/src/misc";
+import { sort, timeInHour, timeInMinute, timeInSecond, timeoutToError, timeoutToUndefined, timeoutToUndefinedSilent } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
 import { getMachineId } from "../../../-a-auth/certs";
 import { getAllNodeIds, getOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
@@ -27,32 +27,6 @@ const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
 const LOGGER_NAMES = ["logs/log", "logs/info", "logs/warn", "logs/error"] as const;
 type LoggerName = typeof LOGGER_NAMES[number];
 
-// Public-facing short names callers pass in via the `logTypes` parameter, mapped
-//  to the internal logger names above.
-const EXTERNAL_TO_INTERNAL_LOGGER: Record<string, LoggerName> = {
-    "log": "logs/log",
-    "info": "logs/info",
-    "warn": "logs/warn",
-    "error": "logs/error",
-};
-
-// Parses the caller's `logTypes` string (e.g. "warn|error") into the matching
-//  internal logger names. Empty / undefined means "all four" (no restriction).
-function parseLogTypes(value: string | undefined): readonly LoggerName[] {
-    if (!value) return LOGGER_NAMES;
-    let parts = value.split("|").map(s => s.trim().toLowerCase()).filter(s => s);
-    if (parts.length === 0) return LOGGER_NAMES;
-    let result: LoggerName[] = [];
-    for (let p of parts) {
-        let internal = EXTERNAL_TO_INTERNAL_LOGGER[p];
-        if (!internal) {
-            throw new Error(`logTypes: unknown log type ${JSON.stringify(p)}; expected one of log, info, warn, error (separated by "|")`);
-        }
-        if (!result.includes(internal)) result.push(internal);
-    }
-    return result;
-}
-
 // Per-logger accounting for one search. Byte counts are raw buffer sizes.
 export type LoggerStats = {
     // Files in range matching the requested machine.
@@ -168,11 +142,8 @@ function createProgressLogger(): (message: string) => void {
 }
 
 export class MCPIndexedLogs {
-    // `${machineId}|${loggerName}` -> latest timestamp guaranteed to already be
-    //  moved-to-public for that specific (machine, logger). Keyed per-logger so
-    //  a search scoped to only some loggers doesn't poison the cache for the
-    //  others.
-    private movedThroughByMachineLogger = new Map<string, number>();
+    // machineId -> latest timestamp guaranteed to already be moved-to-public.
+    private movedThroughByMachine = new Map<string, number>();
 
     // Cache: `${type}|${loggerName}|${startBucket}|${endBucket}` -> { time, paths }.
     // Buckets are hour-aligned start/end so adjacent searches reuse work.
@@ -186,15 +157,14 @@ export class MCPIndexedLogs {
         direction: Direction;
         columns: string[];
         limit?: number;
-        // Optional pipe-separated list restricting which log streams to scan
-        //  (e.g. "warn|error", "log"). Omit / empty = all four streams.
-        logTypes?: string;
     }): Promise<SearchResult> {
         let limit = config.limit ?? 100;
         let startTime = normalizeTime(config.startTime, "startTime");
         let endTime = normalizeTime(config.endTime, "endTime");
-        let enabledLoggers = parseLogTypes(config.logTypes);
-        console.log(`[search] query=${JSON.stringify(config.query)} | machine=${config.machine} | startTime=${formatDateTime(startTime)} | endTime=${formatDateTime(endTime)} | direction=${config.direction} | columns=[${config.columns.join(",")}] | limit=${config.limit ?? "(default)"} | logTypes=${config.logTypes ?? "(all)"}`);
+        // `time` is always projected — the final sort needs it, and callers can't
+        // meaningfully read a log row without knowing when it happened.
+        let columns = config.columns.includes("time") ? config.columns : ["time", ...config.columns];
+        console.log(`[search] query=${JSON.stringify(config.query)} | machine=${config.machine} | startTime=${formatDateTime(startTime)} | endTime=${formatDateTime(endTime)} | direction=${config.direction} | columns=[${config.columns.join(",")}] | limit=${config.limit ?? "(default)"}`);
         let now = Date.now();
         if (endTime > now - END_TIME_MIN_AGE) {
             throw new Error(`endTime must be at least ${formatTime(END_TIME_MIN_AGE)} in the past (got ${formatTime(now - endTime)} ago)`);
@@ -206,7 +176,7 @@ export class MCPIndexedLogs {
         let machineId = config.machine === "local" ? getOwnMachineId() : config.machine;
 
         let moveStart = Date.now();
-        let moveOutcome = await this.ensureMovedThrough(machineId, endTime, enabledLoggers);
+        let moveOutcome = await this.ensureMovedThrough(machineId, endTime);
         console.log(`[search] ensureMovedThrough ${moveOutcome} in ${formatTime(Date.now() - moveStart)}`);
 
         let loggers = await getLoggers2Async();
@@ -223,7 +193,7 @@ export class MCPIndexedLogs {
 
         let pathsStart = Date.now();
         let totalPathsSeen = 0;
-        await Promise.all(enabledLoggers.map(async (loggerName) => {
+        await Promise.all(LOGGER_NAMES.map(async (loggerName) => {
             let logger = this.getLoggerByName(loggers, loggerName);
             let archives = logger.debugGetCachedLogs({ type: useType });
 
@@ -270,6 +240,9 @@ export class MCPIndexedLogs {
         }));
         console.log(`[search] read ${allFiles.length} files in ${formatTime(Date.now() - searchStart)}`);
 
+        let dir = config.direction === "fromStart" ? 1 : -1;
+        sort(readFiles, x => x.entry.path.startTime * dir);
+
         // Phase 2: scan the already-read files in time order, applying a moving
         // cutoff once we have `limit` rows: any unprocessed file whose entire range
         // is past the cutoff cannot contribute results we'd keep.
@@ -282,7 +255,6 @@ export class MCPIndexedLogs {
             scanCount++;
             logScanProgress(`[search] scanning files ${scanCount}/${readFiles.length}`);
 
-            if (resultRows.length >= limit) break;
             if (buffers === undefined) continue;
 
             if (cutoff !== undefined) {
@@ -303,7 +275,7 @@ export class MCPIndexedLogs {
                 limit,
                 queryBuffer,
                 matchesPattern,
-                columns: config.columns,
+                columns,
                 startTime,
                 endTime,
                 stats,
@@ -326,18 +298,26 @@ export class MCPIndexedLogs {
         let totals = createEmptyLoggerStats();
         for (let name of LOGGER_NAMES) addLoggerStats(totals, fileCounts[name]);
 
-        let limitHit = resultRows.length >= limit;
-        console.log(`[search] done in ${formatTime(Date.now() - searchStart)} (filesScanned=${totals.scanned}/${allFiles.length} scannedBytes=${formatNumber(totals.scannedBytes)}B blocksMatched=${totals.blocksMatched} blocksRead=${totals.blocksRead} blockBytesRead=${formatNumber(totals.blockBytesRead)}B results=${resultRows.length} limit=${limit}${limitHit ? " HIT" : ""})`);
+        // Files from different loggers can overlap in time, so rows come out of
+        // phase 2 only roughly time-ordered. Sort by row time in the scan
+        // direction and slice to `limit`. The slice is the *only* place the
+        // global limit is enforced on the returned set — per-file/per-block caps
+        // upstream are bounded by `limit` to keep memory sane but don't define
+        // truncation by themselves.
+        sort(resultRows, r => Number(r.time) * dir);
+        let totalMatched = resultRows.length;
+        if (totalMatched > limit) resultRows = resultRows.slice(0, limit);
+
+        let limitHit = totalMatched > limit;
+        console.log(`[search] done in ${formatTime(Date.now() - searchStart)} (filesScanned=${totals.scanned}/${allFiles.length} scannedBytes=${formatNumber(totals.scannedBytes)}B blocksMatched=${totals.blocksMatched} blocksRead=${totals.blocksRead} blockBytesRead=${formatNumber(totals.blockBytesRead)}B matched=${totalMatched} returned=${resultRows.length} limit=${limit}${limitHit ? " HIT" : ""})`);
         console.log(`[search] buffer types: stream=${stats.typeCounts.stream} bulk=${stats.typeCounts.bulk}`);
         console.log(`[search] timing: readFiles=${formatTime(totals.readFilesMs)} findMatchingBlocks=${formatTime(totals.findMatchingBlocksMs)} getBlockBuffers=${formatTime(totals.getBlockBuffersMs)}`);
 
         // Trim the internal LoggerStats down to just total + scanned. The rest
         // (bytes/blocks/timing) stays in the console.log above and is NOT
-        // returned — see the warning on SearchResult. We only emit entries for
-        // the loggers we actually searched, so a caller who scoped to
-        // `warn|error` doesn't see misleading 0s for the loggers they skipped.
+        // returned — see the warning on SearchResult.
         let files: Record<string, { total: number; scanned: number }> = {};
-        for (let name of enabledLoggers) {
+        for (let name of LOGGER_NAMES) {
             files[name] = { total: fileCounts[name].total, scanned: fileCounts[name].scanned };
         }
 
@@ -387,8 +367,13 @@ export class MCPIndexedLogs {
         sink: SearchSink;
     }): Promise<void> {
         let { entry, indexBuf, dataBuf, direction, limit, queryBuffer, matchesPattern, columns, startTime, endTime, stats, sink } = scan;
-        let { resultRows, loggerStats } = sink;
+        let { loggerStats } = sink;
         let p = entry.path;
+        // Per-file cap. Blocks are scanned in time order, so anything past the
+        // first `limit` rows from this file would lose the caller's final
+        // sort+slice anyway. Tracked locally (not against the shared sink)
+        // so a noisy earlier file doesn't starve overlapping later files.
+        let rowsFromThisFile = 0;
 
         // Region 1: the index scan that picks candidate blocks.
         let findStart = Date.now();
@@ -421,7 +406,7 @@ export class MCPIndexedLogs {
         }
 
         for (let block of blocks) {
-            if (resultRows.length >= limit) break;
+            if (rowsFromThisFile >= limit) break;
 
             // Region 2: decoding the candidate block's buffers.
             let buffers: Buffer[] | undefined;
@@ -441,8 +426,10 @@ export class MCPIndexedLogs {
 
             let ordered = direction === "fromStart" ? buffers : [...buffers].reverse();
             for (let buf of ordered) {
-                if (resultRows.length >= limit) break;
+                if (rowsFromThisFile >= limit) break;
+                let before = sink.resultRows.length;
                 this.appendRow({ buf, matchesPattern, columns, startTime, endTime, sink });
+                if (sink.resultRows.length > before) rowsFromThisFile++;
             }
         }
         loggerStats.getBlockBuffersMs += Date.now() - blockStart;
@@ -485,20 +472,16 @@ export class MCPIndexedLogs {
         sink.loggerStats.rows++;
     }
 
-    // For each requested logger, asks each remote node on the target machine
-    // whether it has pending logs overlapping [0, endTime]. The first node that
-    // answers without throwing wins; if it says yes, we ask the same node to
-    // flush. We iterate because not every node necessarily exposes the new
-    // endpoints (e.g. older versions still running). Records moved-through up
-    // to now - MOVE_GRACE per (machine, logger) so we skip this on subsequent
-    // calls covering the same window. Only the loggers listed in `loggers` are
-    // touched; the others aren't queried or flushed.
-    private async ensureMovedThrough(machineId: string, endTime: number, loggers: readonly LoggerName[]): Promise<"cached" | "no-node" | "moved"> {
-        let needed = loggers.filter(name => {
-            let lastMoved = this.movedThroughByMachineLogger.get(`${machineId}|${name}`) ?? 0;
-            return lastMoved < endTime;
-        });
-        if (needed.length === 0) return "cached";
+    // For each logger, asks each remote node on the target machine whether it
+    // has pending logs overlapping [0, endTime]. The first node that answers
+    // without throwing wins; if it says yes, we ask the same node to flush.
+    // We iterate because not every node necessarily exposes the new endpoints
+    // (e.g. older versions still running). Records moved-through up to
+    // now - MOVE_GRACE so we skip this on subsequent calls covering the same
+    // window.
+    private async ensureMovedThrough(machineId: string, endTime: number): Promise<"cached" | "no-node" | "moved"> {
+        let lastMoved = this.movedThroughByMachine.get(machineId) ?? 0;
+        if (lastMoved >= endTime) return "cached";
 
         let nodeIds = await this.findRemoteNodesOnMachine(machineId);
         if (nodeIds.length === 0) {
@@ -506,7 +489,7 @@ export class MCPIndexedLogs {
             return "no-node";
         }
 
-        for (let loggerName of needed) {
+        for (let loggerName of LOGGER_NAMES) {
             let answered = false;
             for (let nodeId of nodeIds) {
                 try {
@@ -534,10 +517,7 @@ export class MCPIndexedLogs {
             }
         }
 
-        let recordTime = Date.now() - MOVE_GRACE;
-        for (let loggerName of needed) {
-            this.movedThroughByMachineLogger.set(`${machineId}|${loggerName}`, recordTime);
-        }
+        this.movedThroughByMachine.set(machineId, Date.now() - MOVE_GRACE);
         return "moved";
     }
 

package/src/diagnostics/logs/diskLogger.ts

@@ -10,6 +10,9 @@ import { isPublic } from "../../config";
 import type { IndexedLogs } from "./IndexedLogs/IndexedLogs";
 // IMPORTANT! We can't have any real imports here, because we are depended on so early in startup!
 
+let logWriteCount = 0;
+export function getLogWriteCount() { return logWriteCount; }
+
 if (isNode()) {
     // Delayed setup, as we depend on diskLogger early, and we don't want to force high level
     //  modules to be required before their level
@@ -150,6 +153,7 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
             } else {
                 errorLogs.append(logObj);
             }
+            logWriteCount++;
         }
 
     } catch (e: any) {

package/src/diagnostics/managementPages.tsx

@@ -167,6 +167,12 @@ export async function registerManagementPages2(config: {
         controllerName: "SyncTestController",
         getModule: () => import("./SyncTestPage"),
     });
+    inputPages.push({
+        title: "Gross Stats",
+        componentName: "GrossStatsPage",
+        controllerName: "GrossStatsController",
+        getModule: () => import("./grossStats/GrossStatsPage"),
+    });
     inputPages.push(...config.pages);
 
     // NOTE: We don't store the UI in the database (here, or anywhere else, at least
@@ -313,6 +319,7 @@ export function renderIsManagementUser() {
 
 const ErrorWarning = createLazyComponent(() => import("./logs/errorNotifications2/ErrorWarning"))("ErrorWarning");
 const LaunchTrackingHeader = createLazyComponent(() => import("../deployManager/LaunchTrackingHeader"))("LaunchTrackingHeader");
+const GrossStatsInfo = createLazyComponent(() => import("./grossStats/GrossStatsInfo"))("GrossStatsInfo");
 
 class ManagementRoot extends qreact.Component {
     state = {
@@ -386,6 +393,7 @@ class ManagementRoot extends qreact.Component {
                             <PathDistributionInfo />
                             <ValuePathWarning />
                             <LaunchTrackingHeader />
+                            <GrossStatsInfo />
                         </div>}
                     </div>
                     {currentPage &&