querysub 0.20.0 → 0.23.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.20.0",
+    "version": "0.23.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -351,6 +351,8 @@ async function runMainSyncLoops(discoveryReady: PromiseObj<void>) {
         await timeoutToUndefinedSilent(timeInSecond * 5, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId())));
     }));
 
+    console.log(magenta(`Node discovery is ready and first heartbeat + sync is done`));
+
     await runInfinitePoll(HEARTBEAT_INTERVAL, async function nodeDiscoverHeartbeat() {
         // If we waited too long, other nodes might think we are dead. In which case, we SHOULD terminate.
         if (!isNoNetwork()) {

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -230,6 +230,7 @@ class NodePathAuthorities {
                     return;
                 }
 
+                let time = Date.now();
                 let createTime = await errorToUndefinedSilent(PathController.nodes[nodeId].getCreateTime());
                 if (createTime === undefined) {
                     // Don't log for 127-0-0-1, as it usually fails, and is mostly a development optimization
@@ -238,7 +239,8 @@ class NodePathAuthorities {
                     }
                     return;
                 }
-                console.log(blue(`Identifying ${nodeId} as a path authority`));
+                time = Date.now() - time;
+                console.log(blue(`Identifying ${nodeId} as a path authority. ping latency = ${formatTime(time)}`));
                 this.authorities.set(nodeId, {
                     nodeId,
                     self: nodeId.startsWith("127-0-0-1."),

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -23,7 +23,7 @@ import { LOCAL_DOMAIN_PATH } from "../0-path-value-core/NodePathAuthorities";
 import { registerPeriodic } from "../diagnostics/periodic";
 import { remoteWatcher } from "../1-path-client/RemoteWatcher";
 import { Schema2, Schema2Fncs } from "./schema2";
-import { devDebugbreak, getDomain } from "../config";
+import { devDebugbreak, getDomain, isPublic } from "../config";
 
 import type { CallSpec } from "../3-path-functions/PathFunctionRunner";
 import type { FunctionMetadata } from "../3-path-functions/syncSchema";
@@ -1813,9 +1813,9 @@ export function registerSchemaPrefix(config: {
     if (!list) {
         schemaPrefixes.push(list = { len, prefixes: new Map() });
     }
-    if (isNode()) {
+    if (isNode() && !globalThis.isHotReloading?.()) {
         if (list.prefixes.has(config.prefixPathStr)) {
-            throw new Error(`Prefix matches only work due to only having one version of the code ever load (except for during hot reloading development clientside). If we try to render multiple versions of the same code serverside (ex, because we pre-loaded new code during a deploy), it will break, as we can't distinguish them. ADD GIT HASH TO PREFIX SCHEMAS TO FIX THIS!`);
+            throw new Error(`Prefix matches only work due to only having one version of the code ever load (except for during hot reloading development clientside). If we try to render multiple versions of the same code serverside (ex, because we pre-loaded new code during a deploy), it will break, as we can't distinguish them. ADD GIT HASH TO PREFIX SCHEMAS TO FIX THIS! Tried to double load ${config.prefixPathStr}`);
         }
     }
     list.prefixes.set(config.prefixPathStr, config.schema);

package/src/3-path-functions/pathFunctionLoader.ts

@@ -16,7 +16,7 @@ import { isNode, isNodeTrue, nextId, timeInSecond } from "socket-function/src/mi
 import { getPathStr2, getPathStr3 } from "../path";
 import { consistentHash } from "../misc/hash";
 import { setExternalHotReloading } from "socket-function/hot/HotReloadController";
-import { isPublic } from "../config";
+import { devDebugbreak, isPublic } from "../config";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { getControllerNodeId, getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
@@ -281,6 +281,10 @@ async function getModuleFromSpecBase(
         deployPath = packagePath + "deploy.ts";
     }
 
+    if (path.startsWith("/root")) {
+        devDebugbreak();
+    }
+
     console.log(blue(`require(${JSON.stringify(path)})`));
 
     // Set functionSpec for the next synchronous evaluation

package/src/5-diagnostics/nodeMetadata.ts

@@ -6,10 +6,11 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { cache } from "socket-function/src/caching";
 import { TimeGrouper } from "./TimeGrouper";
 import { runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { timeInMinute } from "socket-function/src/misc";
+import { throttleFunction, timeInMinute } from "socket-function/src/misc";
 import { formatNumber, formatPercent, formatTime } from "socket-function/src/formatting/format";
 import os from "os";
 import { isNode } from "typesafecss";
+import { diskLog } from "../diagnostics/logs/diskLogger";
 
 // Undefined means we infer the column
 // Null means the column is removed
@@ -40,8 +41,12 @@ export function registerNodeMetadata(getter: ExtraMetadata) {
 
 let stateValues = cache((title: string, format: (value: number) => string): { (value: number): void } => {
     let timeGrouper = new TimeGrouper();
+    let logToDisk = throttleFunction(timeInMinute, (value: number) => {
+        diskLog(title, { value, formatted: format(value) });
+    });
     function onValue(value: number) {
         timeGrouper.onValueChanged(value);
+        void logToDisk(value);
     }
     registerNodeMetadata({
         columnName: title,
@@ -54,8 +59,12 @@ let stateValues = cache((title: string, format: (value: number) => string): { (v
 });
 let eventValues = cache((title: string, format: (value: number) => string): { (value: number): void } => {
     let timeGrouper = new TimeGrouper();
+    let logToDisk = throttleFunction(timeInMinute, (value: number) => {
+        diskLog(title, { value, formatted: format(value) });
+    });
     function onValue(value: number) {
         timeGrouper.onValueChanged(value);
+        void logToDisk(value);
     }
     registerNodeMetadata({
         columnName: title,
@@ -81,11 +90,11 @@ if (isNode()) {
         const free = os.freemem();
         const total = os.totalmem();
 
-        let constrainedMemory = process.constrainedMemory();
-
         logNodeStateStats("System|Memory Free", x => formatNumber(x) + "B")(free);
         logNodeStateStats("System|Memory Free %", formatPercent)(free / total);
-        if (constrainedMemory !== undefined) {
+
+        let constrainedMemory = process.constrainedMemory();
+        if (constrainedMemory !== undefined && constrainedMemory < 2 ** 48) {
             logNodeStateStats("System|Constrained Memory", x => formatNumber(x) + "B")(constrainedMemory);
         }
 
@@ -97,7 +106,8 @@ if (isNode()) {
         logNodeStateStats("System|CPU Count", formatNumber)(cpuCount);
 
         let load = os.loadavg();
-        logNodeStateStats("System|Average Free Cores", formatPercent)((1 - load[0]) * os.cpus().length);
+        logNodeStateStats("System|Average Load", formatPercent)(load[0]);
+        logNodeStateStats("System|Average Free Cores", formatPercent)((os.cpus().length - load[0]));
 
     });
 }

package/src/diagnostics/NodeViewer.tsx

@@ -54,7 +54,7 @@ type NodeData = {
         columns: ColumnsType;
         row: RowType;
     };
-
+    loadTime?: number;
 };
 
 module.hotreload = true;
@@ -81,6 +81,7 @@ export class NodeViewer extends qreact.Component {
                 let ourIP = await controller.getCallerIP();
 
                 await Promise.allSettled(nodeIds.map(async nodeId => {
+                    let time = Date.now();
                     let data: NodeData = { nodeId };
                     try {
                         data.table = await controller.getMiscInfo(nodeId);
@@ -100,6 +101,8 @@ export class NodeViewer extends qreact.Component {
                     } catch (e: any) {
                         data.apiError = "Error: " + e.stack;
                     }
+                    time = Date.now() - time;
+                    data.loadTime = time;
                     Querysub.commit(() => {
                         this.state.nodes[nodeId] = atomicObjectWrite(data);
                     });
@@ -182,7 +185,7 @@ export class NodeViewer extends qreact.Component {
         }
 
         let builtinGroups = {
-            "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "Heap", "All Memory", "Blocking Lag", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
+            "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "Heap", "All Memory", "Blocking Lag", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
         };
         // Column => group
         let builtInGroupsLookup = new Map<string, string>();
@@ -263,7 +266,6 @@ export class NodeViewer extends qreact.Component {
                                     formatter: (obj) => {
                                         let str = String(obj);
                                         if (str.startsWith("http")) return formatValue(obj);
-                                        https://noproxy.querysub.com:1111/?page=test&showingmanagement&managementpage=DiskLoggerPage&nodeId=b4d19fba5f79d48c3.b68f46ffaffad0636.querysub.com%3A41863&filter=%22%5C%22__nodeId%5C%22%3A%5C%22b654f522bf3a67c49.b68f46ffaffad0636.querysub.com%3A33473%5C%22%22
 
                                         return (
                                             <ATag values={[
@@ -277,6 +279,7 @@ export class NodeViewer extends qreact.Component {
                                     },
                                 },
                                 ip: {},
+                                loadTime: { formatter: "timeSpan" },
                                 ...x.table?.columns,
                                 //capabilities: null,
                                 apiError: { formatter: "error" },
@@ -353,8 +356,12 @@ class NodeViewerControllerBase {
         await forwardPort({ internalPort: externalPort, externalPort: externalPort, });
 
         tlsServer.on("secureConnection", (socket) => {
-            if (socket.remoteAddress !== callerIP) {
-                console.error(`Rejecting connection from ${socket.remoteAddress}, expected ${callerIP}`);
+            let matchedIP = (
+                socket.remoteAddress === callerIP ||
+                callerIP === "127.0.0.1" && socket.remoteAddress === ourIP
+            );
+            if (!matchedIP) {
+                console.error(`Rejecting connection from ${socket.remoteAddress}, expected ${callerIP} ${callerIP === "127.0.0.1" && `or ${ourIP}` || ""}`);
                 socket.end();
                 return;
             }
@@ -375,7 +382,7 @@ class NodeViewerControllerBase {
         let ourDomain = ourIP.replaceAll(".", "-") + "." + getDomain();
         await setRecord("A", ourDomain, ourIP);
 
-        console.log(`${ourDomain}:${finalPort} => ${baseNodeIPResolved}:${externalPort} => 127.0.0.1:${internalPort}, for ${callerIP}`);
+        console.log(`Inspect forward ${ourDomain}:${finalPort} => ${baseNodeIPResolved}:${externalPort} => 127.0.0.1:${internalPort}, for ${callerIP}`);
 
         return internalInspectURL.replace(`ws=127.0.0.1:${internalPort}`, `wss=${ourDomain}:${finalPort}`);
     }

package/src/diagnostics/logs/DiskLoggerPage.tsx

@@ -182,7 +182,7 @@ export class DiskLoggerPage extends qreact.Component {
                         if (!canHaveChildren(value)) {
                             // Add to search as well
                             if (filterURL.value.trim()) {
-                                filterURL.value += " | ";
+                                filterURL.value += " & ";
                             }
                             filterURL.value += String(value);
                         }
@@ -244,17 +244,17 @@ export class DiskLoggerPage extends qreact.Component {
         };
 
         table.columns["remaining"] = {
+            title: "Object",
             formatter(value, context) {
                 return <ObjectDisplay
                     value={context?.row}
-                    excludedFields={fullSelectedFields}
                     onClickKey={path => addPath(path)}
                     onClickValue={(path, value) => {
                         addPath(path);
                         if (!canHaveChildren(value)) {
                             // Add to search as well
                             if (filterURL.value.trim()) {
-                                filterURL.value += " | ";
+                                filterURL.value += " & ";
                             }
                             filterURL.value += String(value);
                         }

package/src/diagnostics/logs/diskLogger.ts

@@ -1,6 +1,6 @@
 
 import { batchFunction, runInfinitePoll } from "socket-function/src/batching";
-import { timeInDay, timeInHour } from "socket-function/src/misc";
+import { nextId, timeInDay, timeInHour } from "socket-function/src/misc";
 import { getStorageDir, getStorageFolder } from "../../fs";
 import fs from "fs";
 import { canHaveChildren } from "socket-function/src/types";
@@ -188,9 +188,11 @@ function safeCopyObject<T>(obj: T): T {
         return { errorCopying: e.messsage } as any;
     }
 }
+let __baseThreadId = nextId();
 function packageLogObj(args: unknown[]): LogObj {
     let logObj: LogObj = {
         time: Date.now(),
+        __baseThreadId,
     };
     for (let part of globalContextParts) {
         Object.assign(logObj, safeCopyObject(part()));

package/src/diagnostics/trackResources.ts

@@ -76,7 +76,7 @@ function logResourcesNow() {
 }
 
 registerMeasureInfo(() => {
-    return formatNumber(getUsedHeapSize()) + "B";
+    return formatNumber(getHeapSize()) + "B";
 });
 
 registerPeriodic(logResourcesNow);