querysub 0.456.0 → 0.457.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.456.0",
+  "version": "0.457.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-a-archives/archivesBackBlaze.ts

@@ -516,8 +516,9 @@ export class ArchivesBackblaze {
         fnc: (api: B2Api) => Promise<T>,
         retries = 3
     ): Promise<T> {
-        let api = await this.getBucketAPI();
+        let api: B2Api | undefined;
         try {
+            api = await this.getBucketAPI();
             return await fnc(api);
         } catch (err: any) {
             if (retries <= 0) throw err;
@@ -570,19 +571,21 @@ export class ArchivesBackblaze {
             }
 
             if (err.stack.includes(`getaddrinfo ENOTFOUND`)) {
-                let urlObj = new URL(api.apiUrl);
-                let hostname = urlObj.hostname;
-                let lookupAddresses = await new Promise(resolve => {
-                    dns.lookup(hostname, (err, addresses) => {
-                        resolve(addresses);
+                if (api) {
+                    let urlObj = new URL(api.apiUrl);
+                    let hostname = urlObj.hostname;
+                    let lookupAddresses = await new Promise(resolve => {
+                        dns.lookup(hostname, (err, addresses) => {
+                            resolve(addresses);
+                        });
                     });
-                });
-                let resolveAddresses = await new Promise(resolve => {
-                    dns.resolve4(hostname, (err, addresses) => {
-                        resolve(addresses);
+                    let resolveAddresses = await new Promise(resolve => {
+                        dns.resolve4(hostname, (err, addresses) => {
+                            resolve(addresses);
+                        });
                     });
-                });
-                console.error(`[${context}] getaddrinfo ENOTFOUND ${hostname}`, { lookupAddresses, resolveAddresses, apiUrl: api.apiUrl, fullError: err.stack });
+                    console.error(`[${context}] getaddrinfo ENOTFOUND ${hostname}`, { lookupAddresses, resolveAddresses, apiUrl: api.apiUrl, fullError: err.stack });
+                }
             }
 
             // NOTE: The AI thought case that happens when we run out of retries, that's stupid. This obviously isn't the case. This is the case when it's a normal error, as in the file doesn't exist, we need to throw. We absolutely should not warn here. Warning here wouldn't be anything. It would just be saying, oh, we checked if a file and it didn't, which is normal, which is why we check if a file exists. 

package/src/-h-path-value-serialize/PathValueSerializer.ts

@@ -997,23 +997,23 @@ class PathValueSerializer {
     public getPathValue(pathValue: PathValue | undefined, noMutate?: "noMutate"): unknown {
         if (!pathValue) return undefined;
         if (pathValue.isValueLazy) {
-            // NOTE: If this throws, it likely means you used atomicObjectWrite.
-            //  Use atomicObjectWriteNoFreeze or doAtomicWrites instead.
-            if (!noMutate) {
-                try {
-                    pathValue.isValueLazy = false;
-                } catch { }
-            }
             let buffer = this.lazyValues.get(pathValue.value as {});
             if (!buffer) {
-                console.error(`Expected lazy value to have a buffer, but it didn't. Lazy ref has a typeof ${typeof pathValue.value} (${String(pathValue.value)})`);
-                return pathValue.value;
+                throw new Error(`Expected lazy value to have a buffer, but it didn't. Lazy ref has a typeof ${typeof pathValue.value} (${String(pathValue.value)})`);
             }
             let newValue = recursiveFreeze(cbor.decode(buffer));
-            if (!noMutate) {
+            if (!noMutate && !pathValue.isValueLazy) {
+                // NOTE: If this throws, it likely means you used atomicObjectWrite.
+                //  Use atomicObjectWriteNoFreeze or doAtomicWrites instead.
                 try {
+                    pathValue.isValueLazy = false;
                     pathValue.value = newValue;
-                } catch { }
+                } catch {
+                    // In theory, it could be that you can set the is value lazy property but not the value property. In which case we want to reset the isValazy property back. Otherwise, we will no longer try to deserialize it, and we will emit the lazy placeholder instead of the actual value next time.
+                    try {
+                        pathValue.isValueLazy = true;
+                    } catch { }
+                }
             }
             return newValue;
         }
@@ -1027,8 +1027,7 @@ class PathValueSerializer {
                 // NOTE: Did you pass a raw PathValue and then try to use PathValueSerializer with it?
                 //  - Instead you should pass a buffer serialized with pathValueSerializer.serialize and
                 //      deserialized with pathValueSerializer.deserialize.
-                console.error(`Expected lazy value to have a buffer, but it didn't. Lazy ref has a typeof ${typeof pathValue.value} (${String(pathValue.value)})`);
-                return pathValue.value;
+                throw new Error(`Expected lazy value to have a buffer, but it didn't. Lazy ref has a typeof ${typeof pathValue.value} (${String(pathValue.value)})`);
             }
             return buffer;
         }
@@ -1036,8 +1035,11 @@ class PathValueSerializer {
     }
 
     private getBuffer(pathValue: PathValue): Buffer {
-        let buffer = pathValue.isValueLazy && this.lazyValues.get(pathValue.value as {});
-        if (buffer) return buffer;
+        if (pathValue.isValueLazy) {
+            let buffer = this.lazyValues.get(pathValue.value as {});
+            if (!buffer) throw new Error(`Expected lazy value to have a buffer, but it didn't. Lazy ref has a typeof ${typeof pathValue.value} (${String(pathValue.value)})`);
+            return buffer;
+        }
         return cborEncoder().encode(pathValue.value);
     }
     public compareValuePaths(a: PathValue | undefined, b: PathValue | undefined) {

package/src/0-path-value-core/pathValueCore.ts

@@ -746,7 +746,7 @@ class AuthorityPathValueStorage {
 
         if (isDebugLogEnabled() && !config?.doNotArchive) {
             for (let value of newValues) {
-                auditLog("INGEST VALUE", { path: value.path, timeId: value.time.time });
+                auditLog("INGEST VALUE", { path: value.path, timeId: value.time.time, batchTime: now });
             }
         }
 

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -620,7 +620,17 @@ export class IndexedLogs<T> {
                     dataReader,
                     params: {
                         ...config.params,
-                        limit: config.params.limit - results.matchCount,
+                        // Per-file cap = global `limit`, NOT `limit - matchCount`.
+                        // Subtracting the running count would starve parallel
+                        // files: once one file fills matchCount, every other
+                        // in-flight file gets limit=0 and stops scanning, even
+                        // though `isSourceRelevant` may still want their earlier
+                        // matches to displace the kept top-K. Within a single
+                        // file blocks are time-ordered in the scan direction,
+                        // so capping at `limit` per file is safe — anything
+                        // past the first `limit` matches in this file couldn't
+                        // survive the top-K sort.
+                        limit: config.params.limit,
                     },
                     keepIterating: () => !results.cancel && progressTracker.isSourceRelevant(path),
                     onResult: (match: Buffer) => {

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -38,7 +38,7 @@ import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
 import { formatSearchString } from "./LogViewerParams";
 
 let excludePendingResults = new URLParam("excludePendingResults", false);
-let limitURL = new URLParam("limit", 16);
+let limitURL = new URLParam("limit", 200);
 let enableLogsURL = new URLParam("enableLogs", true);
 let enableInfosURL = new URLParam("enableInfos", true);
 let enableWarningsURL = new URLParam("enableWarnings", true);
@@ -46,7 +46,6 @@ let enableErrorsURL = new URLParam("enableErrors", true);
 
 let savedPathsURL = new URLParam("savedPaths", "");
 let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
-let useRelativeTimeURL = new URLParam("useRelativeTime", true);
 
 // NOTE: Because this doesn't cache properly, it lags a lot, so we shouldn't try to use it. 
 let showLifecycleColumnURL = new URLParam("showlifecycle", false);
@@ -269,7 +268,8 @@ export class LogViewer3 extends qreact.Component {
         let updateResults = throttleFunction(100, () => {
             if (this.searchSequenceNumber !== currentSequenceNumber) return;
             Querysub.commitLocal(() => {
-                this.state.results = results;
+                // LIMIT, as the other values will be fragmented and so are confusing.
+                this.state.results = results.slice(0, limitURL.value);
             });
         });
 
@@ -495,17 +495,19 @@ export class LogViewer3 extends qreact.Component {
 
         for (let field of selectedFields) {
             let column: ColumnType<unknown, LogDatum> = {};
-            if (field === "time") {
-                column.formatter = (x: unknown) => useRelativeTimeURL.value ? formatDateJSX(Number(x)) : <span title={formatDateTimeDetailed(Number(x))}>{formatDateTime(Number(x))}</span>;
-            } else if (field === "__machineId") {
-                column.formatter = (x: unknown, context) => {
-                    if (!context?.row || !context.row.__machineId) return <ObjectDisplay value={x} />;
-                    return <MachineThreadInfo machineId={context.row.__machineId} threadId={context.row.__threadId || undefined} />;
-                };
-            }
-            if (!column.formatter) {
-                column.formatter = (x: unknown) => <ObjectDisplay value={x} />;
+            if (field === "time" || field === "timeId") {
+                column.formatter = (x: unknown) => <span title={formatDateTimeDetailed(Number(x))}>{formatDateTimeDetailed(Number(x))}</span>;
             }
+            // ObjectDisplay is too slow.
+            // else if (field === "__machineId") {
+            //     column.formatter = (x: unknown, context) => {
+            //         if (!context?.row || !context.row.__machineId) return <ObjectDisplay value={x} />;
+            //         return <MachineThreadInfo machineId={context.row.__machineId} threadId={context.row.__threadId || undefined} />;
+            //     };
+            // }
+            // if (!column.formatter) {
+            //     column.formatter = (x: unknown) => <ObjectDisplay value={x} />;
+            // }
             columns[field] = column;
         }
 
@@ -622,11 +624,6 @@ export class LogViewer3 extends qreact.Component {
                         selectedFieldsURL.value = newValues;
                     }}
                 />
-                <InputLabelURL
-                    label="Use Relative Time"
-                    checkbox
-                    url={useRelativeTimeURL}
-                />
                 <InputLabelURL
                     label="Show Lifecycles"
                     checkbox

package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts

@@ -1,4 +1,4 @@
-import { timeInHour, timeInMinute, timeInSecond, timeoutToError, timeoutToUndefined, timeoutToUndefinedSilent } from "socket-function/src/misc";
+import { sort, timeInHour, timeInMinute, timeInSecond, timeoutToError, timeoutToUndefined, timeoutToUndefinedSilent } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
 import { getMachineId } from "../../../-a-auth/certs";
 import { getAllNodeIds, getOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
@@ -27,32 +27,6 @@ const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
 const LOGGER_NAMES = ["logs/log", "logs/info", "logs/warn", "logs/error"] as const;
 type LoggerName = typeof LOGGER_NAMES[number];
 
-// Public-facing short names callers pass in via the `logTypes` parameter, mapped
-//  to the internal logger names above.
-const EXTERNAL_TO_INTERNAL_LOGGER: Record<string, LoggerName> = {
-    "log": "logs/log",
-    "info": "logs/info",
-    "warn": "logs/warn",
-    "error": "logs/error",
-};
-
-// Parses the caller's `logTypes` string (e.g. "warn|error") into the matching
-//  internal logger names. Empty / undefined means "all four" (no restriction).
-function parseLogTypes(value: string | undefined): readonly LoggerName[] {
-    if (!value) return LOGGER_NAMES;
-    let parts = value.split("|").map(s => s.trim().toLowerCase()).filter(s => s);
-    if (parts.length === 0) return LOGGER_NAMES;
-    let result: LoggerName[] = [];
-    for (let p of parts) {
-        let internal = EXTERNAL_TO_INTERNAL_LOGGER[p];
-        if (!internal) {
-            throw new Error(`logTypes: unknown log type ${JSON.stringify(p)}; expected one of log, info, warn, error (separated by "|")`);
-        }
-        if (!result.includes(internal)) result.push(internal);
-    }
-    return result;
-}
-
 // Per-logger accounting for one search. Byte counts are raw buffer sizes.
 export type LoggerStats = {
     // Files in range matching the requested machine.
@@ -168,11 +142,8 @@ function createProgressLogger(): (message: string) => void {
 }
 
 export class MCPIndexedLogs {
-    // `${machineId}|${loggerName}` -> latest timestamp guaranteed to already be
-    //  moved-to-public for that specific (machine, logger). Keyed per-logger so
-    //  a search scoped to only some loggers doesn't poison the cache for the
-    //  others.
-    private movedThroughByMachineLogger = new Map<string, number>();
+    // machineId -> latest timestamp guaranteed to already be moved-to-public.
+    private movedThroughByMachine = new Map<string, number>();
 
     // Cache: `${type}|${loggerName}|${startBucket}|${endBucket}` -> { time, paths }.
     // Buckets are hour-aligned start/end so adjacent searches reuse work.
@@ -186,15 +157,14 @@ export class MCPIndexedLogs {
         direction: Direction;
         columns: string[];
         limit?: number;
-        // Optional pipe-separated list restricting which log streams to scan
-        //  (e.g. "warn|error", "log"). Omit / empty = all four streams.
-        logTypes?: string;
     }): Promise<SearchResult> {
         let limit = config.limit ?? 100;
         let startTime = normalizeTime(config.startTime, "startTime");
         let endTime = normalizeTime(config.endTime, "endTime");
-        let enabledLoggers = parseLogTypes(config.logTypes);
-        console.log(`[search] query=${JSON.stringify(config.query)} | machine=${config.machine} | startTime=${formatDateTime(startTime)} | endTime=${formatDateTime(endTime)} | direction=${config.direction} | columns=[${config.columns.join(",")}] | limit=${config.limit ?? "(default)"} | logTypes=${config.logTypes ?? "(all)"}`);
+        // `time` is always projected — the final sort needs it, and callers can't
+        // meaningfully read a log row without knowing when it happened.
+        let columns = config.columns.includes("time") ? config.columns : ["time", ...config.columns];
+        console.log(`[search] query=${JSON.stringify(config.query)} | machine=${config.machine} | startTime=${formatDateTime(startTime)} | endTime=${formatDateTime(endTime)} | direction=${config.direction} | columns=[${config.columns.join(",")}] | limit=${config.limit ?? "(default)"}`);
         let now = Date.now();
         if (endTime > now - END_TIME_MIN_AGE) {
             throw new Error(`endTime must be at least ${formatTime(END_TIME_MIN_AGE)} in the past (got ${formatTime(now - endTime)} ago)`);
@@ -206,7 +176,7 @@ export class MCPIndexedLogs {
         let machineId = config.machine === "local" ? getOwnMachineId() : config.machine;
 
         let moveStart = Date.now();
-        let moveOutcome = await this.ensureMovedThrough(machineId, endTime, enabledLoggers);
+        let moveOutcome = await this.ensureMovedThrough(machineId, endTime);
         console.log(`[search] ensureMovedThrough ${moveOutcome} in ${formatTime(Date.now() - moveStart)}`);
 
         let loggers = await getLoggers2Async();
@@ -223,7 +193,7 @@ export class MCPIndexedLogs {
 
         let pathsStart = Date.now();
         let totalPathsSeen = 0;
-        await Promise.all(enabledLoggers.map(async (loggerName) => {
+        await Promise.all(LOGGER_NAMES.map(async (loggerName) => {
             let logger = this.getLoggerByName(loggers, loggerName);
             let archives = logger.debugGetCachedLogs({ type: useType });
 
@@ -270,6 +240,9 @@ export class MCPIndexedLogs {
         }));
         console.log(`[search] read ${allFiles.length} files in ${formatTime(Date.now() - searchStart)}`);
 
+        let dir = config.direction === "fromStart" ? 1 : -1;
+        sort(readFiles, x => x.entry.path.startTime * dir);
+
         // Phase 2: scan the already-read files in time order, applying a moving
         // cutoff once we have `limit` rows: any unprocessed file whose entire range
         // is past the cutoff cannot contribute results we'd keep.
@@ -282,7 +255,6 @@ export class MCPIndexedLogs {
             scanCount++;
             logScanProgress(`[search] scanning files ${scanCount}/${readFiles.length}`);
 
-            if (resultRows.length >= limit) break;
             if (buffers === undefined) continue;
 
             if (cutoff !== undefined) {
@@ -303,7 +275,7 @@ export class MCPIndexedLogs {
                 limit,
                 queryBuffer,
                 matchesPattern,
-                columns: config.columns,
+                columns,
                 startTime,
                 endTime,
                 stats,
@@ -326,18 +298,26 @@ export class MCPIndexedLogs {
         let totals = createEmptyLoggerStats();
         for (let name of LOGGER_NAMES) addLoggerStats(totals, fileCounts[name]);
 
-        let limitHit = resultRows.length >= limit;
-        console.log(`[search] done in ${formatTime(Date.now() - searchStart)} (filesScanned=${totals.scanned}/${allFiles.length} scannedBytes=${formatNumber(totals.scannedBytes)}B blocksMatched=${totals.blocksMatched} blocksRead=${totals.blocksRead} blockBytesRead=${formatNumber(totals.blockBytesRead)}B results=${resultRows.length} limit=${limit}${limitHit ? " HIT" : ""})`);
+        // Files from different loggers can overlap in time, so rows come out of
+        // phase 2 only roughly time-ordered. Sort by row time in the scan
+        // direction and slice to `limit`. The slice is the *only* place the
+        // global limit is enforced on the returned set — per-file/per-block caps
+        // upstream are bounded by `limit` to keep memory sane but don't define
+        // truncation by themselves.
+        sort(resultRows, r => Number(r.time) * dir);
+        let totalMatched = resultRows.length;
+        if (totalMatched > limit) resultRows = resultRows.slice(0, limit);
+
+        let limitHit = totalMatched > limit;
+        console.log(`[search] done in ${formatTime(Date.now() - searchStart)} (filesScanned=${totals.scanned}/${allFiles.length} scannedBytes=${formatNumber(totals.scannedBytes)}B blocksMatched=${totals.blocksMatched} blocksRead=${totals.blocksRead} blockBytesRead=${formatNumber(totals.blockBytesRead)}B matched=${totalMatched} returned=${resultRows.length} limit=${limit}${limitHit ? " HIT" : ""})`);
         console.log(`[search] buffer types: stream=${stats.typeCounts.stream} bulk=${stats.typeCounts.bulk}`);
         console.log(`[search] timing: readFiles=${formatTime(totals.readFilesMs)} findMatchingBlocks=${formatTime(totals.findMatchingBlocksMs)} getBlockBuffers=${formatTime(totals.getBlockBuffersMs)}`);
 
         // Trim the internal LoggerStats down to just total + scanned. The rest
         // (bytes/blocks/timing) stays in the console.log above and is NOT
-        // returned — see the warning on SearchResult. We only emit entries for
-        // the loggers we actually searched, so a caller who scoped to
-        // `warn|error` doesn't see misleading 0s for the loggers they skipped.
+        // returned — see the warning on SearchResult.
         let files: Record<string, { total: number; scanned: number }> = {};
-        for (let name of enabledLoggers) {
+        for (let name of LOGGER_NAMES) {
             files[name] = { total: fileCounts[name].total, scanned: fileCounts[name].scanned };
         }
 
@@ -387,8 +367,13 @@ export class MCPIndexedLogs {
         sink: SearchSink;
     }): Promise<void> {
         let { entry, indexBuf, dataBuf, direction, limit, queryBuffer, matchesPattern, columns, startTime, endTime, stats, sink } = scan;
-        let { resultRows, loggerStats } = sink;
+        let { loggerStats } = sink;
         let p = entry.path;
+        // Per-file cap. Blocks are scanned in time order, so anything past the
+        // first `limit` rows from this file would lose the caller's final
+        // sort+slice anyway. Tracked locally (not against the shared sink)
+        // so a noisy earlier file doesn't starve overlapping later files.
+        let rowsFromThisFile = 0;
 
         // Region 1: the index scan that picks candidate blocks.
         let findStart = Date.now();
@@ -421,7 +406,7 @@ export class MCPIndexedLogs {
         }
 
         for (let block of blocks) {
-            if (resultRows.length >= limit) break;
+            if (rowsFromThisFile >= limit) break;
 
             // Region 2: decoding the candidate block's buffers.
             let buffers: Buffer[] | undefined;
@@ -441,8 +426,10 @@ export class MCPIndexedLogs {
 
             let ordered = direction === "fromStart" ? buffers : [...buffers].reverse();
             for (let buf of ordered) {
-                if (resultRows.length >= limit) break;
+                if (rowsFromThisFile >= limit) break;
+                let before = sink.resultRows.length;
                 this.appendRow({ buf, matchesPattern, columns, startTime, endTime, sink });
+                if (sink.resultRows.length > before) rowsFromThisFile++;
             }
         }
         loggerStats.getBlockBuffersMs += Date.now() - blockStart;
@@ -485,20 +472,16 @@ export class MCPIndexedLogs {
         sink.loggerStats.rows++;
     }
 
-    // For each requested logger, asks each remote node on the target machine
-    // whether it has pending logs overlapping [0, endTime]. The first node that
-    // answers without throwing wins; if it says yes, we ask the same node to
-    // flush. We iterate because not every node necessarily exposes the new
-    // endpoints (e.g. older versions still running). Records moved-through up
-    // to now - MOVE_GRACE per (machine, logger) so we skip this on subsequent
-    // calls covering the same window. Only the loggers listed in `loggers` are
-    // touched; the others aren't queried or flushed.
-    private async ensureMovedThrough(machineId: string, endTime: number, loggers: readonly LoggerName[]): Promise<"cached" | "no-node" | "moved"> {
-        let needed = loggers.filter(name => {
-            let lastMoved = this.movedThroughByMachineLogger.get(`${machineId}|${name}`) ?? 0;
-            return lastMoved < endTime;
-        });
-        if (needed.length === 0) return "cached";
+    // For each logger, asks each remote node on the target machine whether it
+    // has pending logs overlapping [0, endTime]. The first node that answers
+    // without throwing wins; if it says yes, we ask the same node to flush.
+    // We iterate because not every node necessarily exposes the new endpoints
+    // (e.g. older versions still running). Records moved-through up to
+    // now - MOVE_GRACE so we skip this on subsequent calls covering the same
+    // window.
+    private async ensureMovedThrough(machineId: string, endTime: number): Promise<"cached" | "no-node" | "moved"> {
+        let lastMoved = this.movedThroughByMachine.get(machineId) ?? 0;
+        if (lastMoved >= endTime) return "cached";
 
         let nodeIds = await this.findRemoteNodesOnMachine(machineId);
         if (nodeIds.length === 0) {
@@ -506,7 +489,7 @@ export class MCPIndexedLogs {
             return "no-node";
         }
 
-        for (let loggerName of needed) {
+        for (let loggerName of LOGGER_NAMES) {
             let answered = false;
             for (let nodeId of nodeIds) {
                 try {
@@ -534,10 +517,7 @@ export class MCPIndexedLogs {
             }
         }
 
-        let recordTime = Date.now() - MOVE_GRACE;
-        for (let loggerName of needed) {
-            this.movedThroughByMachineLogger.set(`${machineId}|${loggerName}`, recordTime);
-        }
+        this.movedThroughByMachine.set(machineId, Date.now() - MOVE_GRACE);
         return "moved";
     }