querysub 0.377.0 → 0.379.0

package/.cursorrules

@@ -10,6 +10,8 @@ When running a command in the current project, don't "cd" to it. It's redundant,
 
 Don't use redundant comments. If it's a single line and the function name says the same thing that the comment is going to say, you don't need the comment.
 
+When you're inline passing a lambda as an argument, never specify the type. The type should always be inferred. Specifying the type is bad, and it's hard coding, and we don't like hard coding. 
+
 NEVER EVER pass state to qreact.Component as a template parameter. It should ALSO be declared like so (inside the class):
     state = t.state({
         num: t.number,

package/bin/error-watch-public.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+process.argv.push("--nobreak");
+process.argv.push("--public");
+
+require("typenode");
+require("../src/diagnostics/logs/errorNotifications2/errorWatchEntry.ts");

package/bin/error-watch.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+
+process.argv.push("--local");
+
+require("typenode");
+require("../src/diagnostics/logs/errorNotifications2/errorWatchEntry.ts");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.377.0",
+  "version": "0.379.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -12,9 +12,10 @@
     "servershardedtest": "yarn server --authority ./pathremain.json & yarn server --authority ./patha.json & yarn server --authority ./pathb.json & yarn server --authority ./pathc.json & yarn server --authority ./pathd.json",
     "type": "yarn tsc --noEmit",
     "depend": "yarn --silent depcruise src --include-only \"^src\" --config --output-type dot | dot -T svg > dependency-graph.svg",
-    "t": "yarn typenode ./src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts",
+    "t": "yarn typenode ./test.ts",
+    "t2": "yarn typenode ./test2.ts",
     "test-wat": "yarn typenode ./src/wat/watCompiler.ts",
-    "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx",
+    "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications2/errorWatchEntry.ts",
     "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx",
     "build-native": "cd src/diagnostics/logs/IndexedLogs && node-gyp rebuild"
   },
@@ -33,7 +34,9 @@
     "join": "./bin/join.js",
     "join-public": "./bin/join-public.js",
     "movelogs": "./bin/movelogs.js",
-    "addsuperuser": "./bin/addsuperuser.js"
+    "addsuperuser": "./bin/addsuperuser.js",
+    "error-watch": "./bin/error-watch.js",
+    "error-watch-public": "./bin/error-watch-public.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -113,6 +113,13 @@ export function isNodeIdOnOwnMachineId(nodeId: string): boolean {
     return certs.getMachineId(nodeId) === getOwnMachineId() || nodeId.startsWith("127-0-0-1.");
 }
 
+/** Often, when getting all nodes, you're going to receive duplicates of all local nodes with the duplicate containing this prefix. Using the local ID is faster as it will actually use the local IP.
+    TODO: We need a robust way to give a deduplicated set that prefers the local ids. At the moment we just filter out the id locals if we can't have duplicates.
+ */
+export function isNodeIdLocal(nodeId: string): boolean {
+    return nodeId.startsWith("127-0-0-1.");
+}
+
 let nodeOverrides: string[] | undefined;
 let beforeGetNodeAllId = async () => { };
 export async function getAllNodeIds() {

package/src/-g-core-values/NodeCapabilities.ts

@@ -16,15 +16,14 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { isNoNetwork } from "../config";
 import { getDebuggerUrl } from "../diagnostics/listenOnDebugger";
 import { hackDevtoolsWebsocketForward } from "./oneTimeForward";
-import { getOwnMachineId, decodeNodeId, decodeNodeIdAssert } from "../-a-auth/certs";
+import { getOwnMachineId, decodeNodeId, decodeNodeIdAssert, getMachineId } from "../-a-auth/certs";
 import { sort } from "socket-function/src/misc";
 import { getPathStr2 } from "../path";
+import { PromiseObj } from "../promise";
 setImmediate(() => {
     import("../diagnostics/MachineThreadInfo");
 });
 
-let loadTime = Date.now();
-
 let controllerNodeIdCache = new Map<string, string | Promise<string | undefined>>();
 
 // NOTE: If this becomes slow (because we are just trying all servers), we could start to store capabilities
@@ -39,32 +38,45 @@ export async function getControllerNodeId(
     if (cached && typeof cached !== "string") {
         cached = await cached;
     }
+    // TODO: We are relying on is node connected to do a lot of work here. If the node's connected, but somehow doesn't work, we can get in an unrecoverable state, even if there are plenty of other working nodes.
     if (cached && !SocketFunction.isNodeConnected(cached)) {
         controllerNodeIdCache.delete(controller._classGuid);
         cached = undefined;
     }
     if (cached) return cached;
-    let promise = getInternal();
+    let promise = getInternal(retryCount);
     controllerNodeIdCache.set(controller._classGuid, promise);
     return promise;
 
-    async function getInternal() {
-
+    async function getInternal(retryCount: number) {
         let nodeIdsToTest = await getAllNodeIds();
         // Shuffle, so we aren't always using the same node!
         nodeIdsToTest = shuffle(nodeIdsToTest, Date.now());
-        for (let nodeId of nodeIdsToTest) {
+        let resolvedNode = new PromiseObj<string>();
+
+        if (!quiet) {
+            console.log(`Checking ${nodeIdsToTest.length} nodes for capability ${controller._classGuid}`);
+        }
+        let allFinished = Promise.all(nodeIdsToTest.map(async nodeId => {
             if (await doesNodeExposeController(nodeId, controller)) {
                 if (!quiet) {
                     let duration = Date.now() - initialTime;
                     console.log(green(`Resolved capability ${controller._classGuid} with node ${nodeId}, in ${formatTime(duration)}`));
                 }
-                return nodeId;
+                resolvedNode.resolve(nodeId);
             }
+        }));
+        let result = await Promise.race([allFinished, resolvedNode.promise]);
+        if (typeof result === "string") {
+            return result;
         }
+
         if (retryCount > 0) {
+            if (!quiet) {
+                console.log(`Did not find capability in ${nodeIdsToTest.length} nodes, retrying... ${controller._classGuid}`);
+            }
             await delay(2000);
-            return getControllerNodeId(controller, quiet, retryCount - 1, initialTime);
+            return getInternal(retryCount - 1);
         }
         if (!quiet) {
             console.warn(yellow(`Could not find a node that exposes controller ${controller._classGuid}, tried: ${JSON.stringify(nodeIdsToTest)}`));
@@ -81,7 +93,11 @@ export async function getControllerNodeIdList(
     await Promise.all(nodeIdsToTest.map(async nodeId => {
         let result = await doesNodeExposeController(nodeId, controller);
         if (result) {
-            passedNodeIds.set(nodeId, result);
+            let entryPoint = await NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
+            passedNodeIds.set(nodeId, {
+                machineId: getMachineId(nodeId),
+                entryPoint,
+            });
         }
     }));
 
@@ -99,13 +115,11 @@ export async function getControllerNodeIdList(
 }
 
 
-export async function doesNodeExposeController(reconnectNodeId: string, controller: SocketRegistered<{}>): Promise<{ machineId: string, entryPoint: string } | false> {
+export async function doesNodeExposeController(reconnectNodeId: string, controller: SocketRegistered<{}>): Promise<boolean> {
     let exposedControllers = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[reconnectNodeId].getExposedControllers());
-    let machineId = await getOwnMachineId();
-    let entryPoint = await new NodeCapabilitiesControllerBase().getEntryPoint();
 
     if (exposedControllers?.includes(controller._classGuid)) {
-        return { machineId, entryPoint };
+        return true;
     }
     return false;
 }

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -40,10 +40,6 @@ async function main() {
 
     await Querysub.hostService("PathFunctionRunnerMain");
 
-    if (!isPublic()) {
-        void IndexedLogs.runLogMoveLoop();
-    }
-
     // Use a fairly high stick time (the default is 10s), because having wait to sync data is very slow,
     //  and the function runner SHOULD have more memory than the clients, and much faster network speeds
     //  (it should be on the same local network as the path value authorities).

package/src/diagnostics/MachineThreadInfo.tsx

@@ -40,6 +40,7 @@ export class MachineThreadInfo extends qreact.Component<{
     machineId: string;
     threadId?: string;
     onlyShowThread?: boolean;
+    noButtons?: boolean;
 }> {
     static renderInProgress = true;
 
@@ -93,7 +94,17 @@ export class MachineThreadInfo extends qreact.Component<{
                             <>
                                 <div className={css.vbox(4)}>
                                     <strong>Entry Point</strong>
-                                    <div>{threadInfo.entrypoint}</div>
+                                    <div>
+                                        {threadInfo.entrypoint && (
+                                            <a
+                                                href={`cursor://file/${threadInfo.entrypoint}`}
+                                                className={css.color("hsl(210, 100%, 80%)").textDecoration("none").textDecoration("underline", "hover")}
+                                            >
+                                                {threadInfo.entrypoint}
+                                            </a>
+                                        )}
+                                        {!threadInfo.entrypoint && threadInfo.entrypoint}
+                                    </div>
                                 </div>
 
                                 <div className={css.vbox(4)}>
@@ -112,9 +123,9 @@ export class MachineThreadInfo extends qreact.Component<{
                     </div>
 
                     <div className={css.hbox(12)}>
-                        <Button onClick={() => this.handleAttach(threadInfo)}>
+                        {threadInfo && <Button onClick={() => this.handleAttach(threadInfo)}>
                             Attach Debugger
-                        </Button>
+                        </Button>}
 
                         <Button onClick={this.handleRefresh}>
                             Refresh
@@ -145,8 +156,8 @@ export class MachineThreadInfo extends qreact.Component<{
 
         return (
             <div
-                className={css.button.hbox(4).pad2(4).hsla(0, 0, 100, 0.5)}
-                onClick={(e) => {
+                className={(this.props.noButtons && "" || css.button) + css.hbox(4).pad2(4).hsla(0, 0, 100, 0.5)}
+                onClick={this.props.noButtons ? undefined : (e) => {
                     e.stopPropagation();
                     this.showExpandedView(machineInfo, threadInfo);
                 }}
@@ -155,10 +166,22 @@ export class MachineThreadInfo extends qreact.Component<{
                     {machineInfo?.hostname || this.props.machineId}
                 </span>}
                 {!this.props.onlyShowThread && this.props.threadId && " | "}
-                {this.props.threadId && <span title={this.props.threadId}>
-                    {/* TODO: Create a mapping from entrypoints to nice names. The full folder path will likely be too large, but just the filename is too similar (it's server.ts both both the PathValueServer and the application server). */}
-                    {threadInfo?.entrypoint || this.props.threadId}
-                </span>}
+                {this.props.threadId && (
+                    threadInfo?.entrypoint ? (
+                        <a
+                            href={`cursor://file/${threadInfo.entrypoint}`}
+                            title={this.props.threadId}
+                            className={css.color("hsl(210, 100%, 80%)").textDecoration("none").textDecoration("underline", "hover")}
+                            onClick={(e) => e.stopPropagation()}
+                        >
+                            {threadInfo.entrypoint}
+                        </a>
+                    ) : (
+                        <span title={this.props.threadId}>
+                            {this.props.threadId}
+                        </span>
+                    )
+                )}
             </div>
         );
     }
@@ -213,7 +236,7 @@ class MachineThreadInfoBase {
 
     // This function is only intended to be called between nodes
     public async getNodeInfo() {
-        const hostname = await errorToUndefinedSilent(runPromise("hostname"));
+        const hostname = await errorToUndefinedSilent(runPromise("hostname", { quiet: true }));
         return {
             hostname: hostname?.trim() || "",
             entrypoint: process.argv[1] || "",

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -26,6 +26,9 @@ import { LimitGroup } from "../../../functional/limitProcessing";
 import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
 import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
 import { getLoggers2Async } from "../diskLogger";
+import { watchAllValues } from "../errorNotifications2/logWatcher";
+// 
+watchAllValues;
 
 export type TimeFilePathWithSize = TimeFilePath & {
     size: number;
@@ -33,7 +36,8 @@ export type TimeFilePathWithSize = TimeFilePath & {
     sourceName: string;
 };
 
-let loggerByName = new Map<string, IndexedLogs<unknown>>();
+export let loggerByName = new Map<string, IndexedLogs<unknown>>();
+
 
 
 export class IndexedLogs<T> {
@@ -218,11 +222,30 @@ export class IndexedLogs<T> {
         return groups;
     }
 
+    private errorWatchers = new Set<(datum: T) => void>();
+    public watchErrors(callback: (datum: T) => void): () => void {
+        console.log(blue(`Watching errors: ${this.config.name}`));
+        this.errorWatchers.add(callback);
+        return () => {
+            this.errorWatchers.delete(callback);
+        };
+    }
+
     public append(datum: T) {
         this.getCurrentLogStream().append(datum);
         if (IndexedLogs.shouldRunLoop) {
             this.runLogMoverLoop();
         }
+
+        for (let callback of this.errorWatchers.values()) {
+            console.log(blue(`Calling error callback: ${this.config.name}`));
+            try {
+                callback(datum);
+            } catch (e) {
+                this.errorWatchers.delete(callback);
+                console.warn("Error in watchErrors callback, removing callback (likely just a client disconnect)", e);
+            }
+        }
     }
 
     private machineNodeCache = new Map<string, string>();

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -4,7 +4,7 @@ import { css } from "../../../4-dom/css";
 import { t } from "../../../2-proxy/schema2";
 import { Button } from "../../../library-components/Button";
 import { InputLabel, InputLabelURL } from "../../../library-components/InputLabel";
-import { getLoggers2, getLoggers2Async, LogDatum } from "../diskLogger";
+import { getLoggers2Async, LogDatum } from "../diskLogger";
 import { list, timeInDay, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
 import { formatDateTime, formatDateTimeDetailed, formatNumber, formatTime, formatPercent } from "socket-function/src/formatting/format";
 import { IndexedLogs, TimeFilePathWithSize } from "./IndexedLogs";
@@ -69,7 +69,11 @@ export class LogViewer3 extends qreact.Component {
     private searchSequenceNumber = 0;
 
     componentDidMount(): void {
-        void this.loadPaths();
+        if (searchTextURL.value) {
+            void this.search();
+        } else {
+            void this.loadPaths();
+        }
     }
 
     getPaths(): TimeFilePathWithSize[] {

package/src/diagnostics/logs/diskLogger.ts

@@ -7,6 +7,7 @@ import { formatTime } from "socket-function/src/formatting/format";
 import { addEpsilons } from "../../bits";
 import { getPathStr2 } from "../../path";
 import { isPublic } from "../../config";
+import type { IndexedLogs } from "./IndexedLogs/IndexedLogs";
 // IMPORTANT! We can't have any real imports here, because we are depended on so early in startup!
 
 if (isNode()) {
@@ -65,32 +66,35 @@ export const LOG_LINE_LIMIT_FLAG = String.fromCharCode(44534) + "LOGS_LINE_LIMIT
 /** If this key exists in the logged object, as in a key in one of the objects logged, then we will use the value of it as the limit ID. This is useful as it allows us to either override a limit or limit something independently from other logs in the file. */
 export const LOG_LINE_LIMIT_ID = "LIMIT_LINE_ID";
 
-export const getLoggers2 = lazy(function () {
-    const { IndexedLogs } = require("./IndexedLogs/IndexedLogs") as typeof import("./IndexedLogs/IndexedLogs");
-
-    if (!IndexedLogs) {
-        setImmediate(() => {
-            getLoggers2.reset();
-        });
-        return undefined;
-    }
-
-    return {
-        logLogs: new IndexedLogs<LogDatum>({ name: "logs/log", getTime: x => x.time }),
-        warnLogs: new IndexedLogs<LogDatum>({ name: "logs/warn", getTime: x => x.time }),
-        infoLogs: new IndexedLogs<LogDatum>({ name: "logs/info", getTime: x => x.time }),
-        errorLogs: new IndexedLogs<LogDatum>({ name: "logs/error", getTime: x => x.time }),
-    };
-});
+let loggersObj: {
+    logLogs: IndexedLogs<LogDatum>;
+    warnLogs: IndexedLogs<LogDatum>;
+    infoLogs: IndexedLogs<LogDatum>;
+    errorLogs: IndexedLogs<LogDatum>;
+} | undefined;
 export const getLoggers2Async = lazy(async () => {
     const { IndexedLogs } = await import("./IndexedLogs/IndexedLogs");
-    return {
+    loggersObj = {
         logLogs: new IndexedLogs<LogDatum>({ name: "logs/log", getTime: x => x.time }),
         warnLogs: new IndexedLogs<LogDatum>({ name: "logs/warn", getTime: x => x.time }),
         infoLogs: new IndexedLogs<LogDatum>({ name: "logs/info", getTime: x => x.time }),
         errorLogs: new IndexedLogs<LogDatum>({ name: "logs/error", getTime: x => x.time }),
     };
+    return loggersObj;
 });
+export function getLogLogs(): Promise<IndexedLogs<LogDatum>> {
+    return getLoggers2Async().then(x => x.logLogs);
+}
+export function getWarnLogs(): Promise<IndexedLogs<LogDatum>> {
+    return getLoggers2Async().then(x => x.warnLogs);
+}
+export function getInfoLogs(): Promise<IndexedLogs<LogDatum>> {
+    return getLoggers2Async().then(x => x.infoLogs);
+}
+export function getErrorLogs(): Promise<IndexedLogs<LogDatum>> {
+    return getLoggers2Async().then(x => x.errorLogs);
+}
+
 
 // NOTE: If any message (first param) starts with this, we don't log it to the disk. VERY useful for multi-line logging where it wouldn't make sense in the logs
 // NOTE: This is visible, otherwise it's easy to accidentally copy it, and not know why the text is behaving strangely (not === other seemingly equal text, etc).
@@ -105,10 +109,6 @@ export function addGlobalContext(fnc: () => { [key: string]: unknown }) {
     globalContextParts.push(fnc);
 }
 
-let startupDone = false;
-void Promise.resolve().then(() => {
-    startupDone = true;
-});
 
 
 
@@ -136,15 +136,14 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
         // NOTE: Local logs now log to their local disk instead of backblaze, so we can log even if in development (they just won't show up on the remote server).
         //if (isPublic())
         {
-            let loggers = startupDone ? getLoggers2() : undefined;
-            if (!loggers) {
-                getLoggers2.reset();
-                setImmediate(() => {
+            let promise = getLoggers2Async();
+            if (!loggersObj) {
+                void promise.finally(() => {
                     logDiskDontShim(type, ...args);
                 });
                 return;
             }
-            const { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
+            const { logLogs, warnLogs, infoLogs, errorLogs } = loggersObj;
             if (type === "log") {
                 logLogs.append(logObj);
             } else if (type === "warn") {

package/src/diagnostics/logs/diskShimConsoleLogs.ts

@@ -20,6 +20,10 @@ export function shimConsoleLogs() {
         let console = globalThis.console;
         let originalFnc = console[fncName];
         console[fncName] = (...args: any[]) => {
+            if (!(globalThis as any).shimmedConsoleLogs) {
+                if (fncName === "info") return;
+                return originalFnc(...args);
+            }
             try {
                 if (
                     args.length > 0