querysub 0.24.0 → 0.26.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.24.0",
+    "version": "0.26.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-c-identity/IdentityController.ts

@@ -14,6 +14,7 @@ import { getShortNumber } from "../bits";
 import { measureFnc, measureWrap } from "socket-function/src/profiling/measure";
 import { timeoutToError } from "../errors";
 import { delay } from "socket-function/src/batching";
+import { formatTime } from "socket-function/src/formatting/format";
 
 let callerInfo = new Map<CallerContext, {
     reconnectNodeId: string | undefined;
@@ -114,6 +115,7 @@ class IdentityControllerBase {
     //  NodeJS will throw/ignore the peer cert if it isn't trusted).
     @measureFnc
     public async changeIdentity(signature: string, payload: ChangeIdentityPayload) {
+        let time = Date.now();
         const caller = SocketFunction.getCaller();
         // Verify it wasn't signed too long ago (to make signature stealing WAY more difficult).
         const signedThreshold = Date.now() - 1000 * 30;
@@ -152,6 +154,8 @@ class IdentityControllerBase {
             pubKey,
             pubKeyShort: getShortNumber(pubKey),
         });
+
+        console.log(`Authenticated identity for ${caller.nodeId} to ${reconnectNodeId || caller.nodeId} in ${formatTime(Date.now() - time)}, at ${Date.now()}`);
     }
 }
 
@@ -182,13 +186,11 @@ const changeIdentityOnce = cache(measureWrap(async function changeIdentityOnce(c
         mountedPort: getNodeIdLocation(SocketFunction.mountedNodeId)?.port,
     };
     let signature = sign(threadKeyCert, payload);
-    await measureWrap(async function callChangeIdentity() {
-        await timeoutToError(
-            10 * 1000,
-            IdentityController.nodes[nodeId].changeIdentity(signature, payload),
-            () => new Error(`Timeout calling changeIdentity for ${nodeId}`)
-        );
-    }, `callChangeIdentity|${nodeId}`)();
+    await timeoutToError(
+        10 * 1000,
+        IdentityController.nodes[nodeId].changeIdentity(signature, payload),
+        () => new Error(`Timeout calling changeIdentity for ${nodeId}`)
+    );
 }));
 SocketFunction.addGlobalClientHook(async function identityHook(context) {
     if (context.call.classGuid === IdentityController._classGuid) return;

package/src/-d-trust/NetworkTrust2.ts

@@ -1,7 +1,7 @@
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { getCommonName, getIdentityCA, getMachineId, getOwnMachineId } from "../-a-auth/certs";
 import { getArchives } from "../-a-archives/archives";
-import { isNode, timeInSecond } from "socket-function/src/misc";
+import { isNode, throttleFunction, timeInSecond } from "socket-function/src/misc";
 import { SocketFunctionHook } from "socket-function/SocketFunctionTypes";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { IdentityController_getMachineId } from "../-c-identity/IdentityController";
@@ -10,7 +10,9 @@ import { getNodeIdDomainMaybeUndefined, getNodeIdLocation } from "socket-functio
 import { trustCertificate } from "socket-function/src/certStore";
 import { isClient, isServer } from "../config2";
 import debugbreak from "debugbreak";
-import { devDebugbreak, getDomain, isDevDebugbreak } from "../config";
+import { devDebugbreak, getDomain, isDevDebugbreak, isPublic } from "../config";
+import { formatTime } from "socket-function/src/formatting/format";
+import { runInSerial } from "socket-function/src/batching";
 
 // Cache the untrust list, to prevent bugs from causing too many backend reads (while also allowing
 //  bad servers which make request before their trust is verified from staying broken).
@@ -41,10 +43,10 @@ export const requiresNetworkTrustHook: SocketFunctionHook = async config => {
 };
 export const assertIsNetworkTrusted = requiresNetworkTrustHook;
 
-
+let lastArchivesTrusted: string[] | undefined;
 let trustedCache = new Set<string>();
 let untrustedCache = new Map<string, number>();
-export async function isTrusted(machineId: string) {
+export const isTrusted = runInSerial(async function isTrusted(machineId: string) {
     // See the comment in requiresNetworkTrustHook for why clients have to trust all callers.
     if (isClient()) return true;
 
@@ -56,15 +58,23 @@ export async function isTrusted(machineId: string) {
         return false;
     }
 
+    if (machineId.includes("..")) {
+        console.warn(`Invalid machineId in isTrust call: ${machineId}`);
+        return false;
+    }
+
     // Find is faster than get, and usually we don't need the full certificate
     let trustedMachineIds = await archives().find("");
+    lastArchivesTrusted = trustedMachineIds.slice();
     // Always trust ourself
     trustedMachineIds.push(getOwnMachineId());
     // IF developing, trust localhost. This allows us to develop without port forwards,
-    //  on our services, which is INCREASES security, and prevents dev machines from being
+    //  on our services, which INCREASES security, and prevents dev machines from being
     //  connected to by attackers (as dev machines might reveal unfinished content, or even
     //  have security vulnerabilities).
-    if (isDevDebugbreak()) {
+    //  - Don't trust this on public, as in theory an attacker MIGHT be able to connect
+    //      from localhost (but not have disk read/write access)? Maybe...
+    if (!isPublic()) {
         trustedMachineIds.push("127-0-0-1." + getDomain());
     }
 
@@ -80,7 +90,7 @@ export async function isTrusted(machineId: string) {
     } else {
         return true;
     }
-}
+});
 
 export async function isNodeTrusted(nodeId: string) {
     let domainName = getNodeIdDomainMaybeUndefined(nodeId);
@@ -103,12 +113,24 @@ const loadServerCert = cache(async (machineId: string) => {
 const ensureWeAreTrusted = lazy(measureWrap(async () => {
     let machineKeyCert = getIdentityCA();
     let machineId = getCommonName(machineKeyCert.cert);
-    let inArchives = await archives().get(machineId);
-    if (!inArchives) {
+
+    await isTrusted(machineId);
+    if (!lastArchivesTrusted?.includes(machineId)) {
         await archives().set(machineId, machineKeyCert.cert);
     }
 }));
 
+async function loadTrustCerts(nodeId: string) {
+    let location = getNodeIdLocation(nodeId);
+    if (location) {
+        let machineId = getMachineId(location.address);
+        let isTrustedBool = await isTrusted(machineId);
+        if (isTrustedBool) {
+            await loadServerCert(machineId);
+        }
+    }
+};
+
 export const isTrustedByNode = cache(async function isTrustedByNode(nodeId: string) {
     return await TrustedController.nodes[nodeId].isTrusted();
 });
@@ -128,8 +150,8 @@ const TrustedController = SocketFunction.register(
 );
 
 
-if (isNode()) {
 
+if (isNode()) {
     // We have to be trusted if we make calls to a trusted endpoint, OR our mounting
     //  (really only if we are mounting a trusted endpoint, but we don't actually know that)
     requiresNetworkTrustHook.clientHook = async config => {
@@ -138,13 +160,11 @@ if (isNode()) {
 
     // Load the remote certificate, in the almost certain case it isn't a real certificate, and is just internal
     SocketFunction.addGlobalClientHook(async config => {
-        await ensureWeAreTrusted();
-        let location = getNodeIdLocation(config.call.nodeId);
-        if (location) {
-            let machineId = getMachineId(location.address);
-            if (await isTrusted(machineId)) {
-                await loadServerCert(machineId);
-            }
-        }
+        await measureWrap(async function checkTrust() {
+            await Promise.all([
+                ensureWeAreTrusted(),
+                loadTrustCerts(config.call.nodeId)
+            ]);
+        })();
     });
 }

package/src/-e-certs/EdgeCertController.ts

@@ -169,7 +169,7 @@ export async function publishMachineARecords() {
             //  and not set public (such as the FunctionRunner). SO... just ignore this, leaving the records as
             //  public, even though the current run doesn't have public set.
             if (process.argv[1].includes("server.ts")) {
-                console.log(yellow(`Current process is not marked as public, but machine previous had public services. NOT publishing A records to point to 127.0.0.1, which will make service inaccessible if the port is not forwarded open on the public ip.`));
+                console.log(yellow(`Current process is not marked as public, but machine previous had public services. NOT publishing A records to point to 127.0.0.1, which will make service inaccessible if the port is not port forwarded (or open). I recommend using noproxy.DOMAIN.com to access the server. 127-0-0-1.DOMAIN.com might work as well.`));
             }
             return;
         }

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -194,12 +194,11 @@ class NodePathAuthorities {
     private async watchAuthorityPaths() {
         await onNodeDiscoveryReady();
 
-        onReadyReady = (nodeId, time) => {
+        onReadReady = (nodeId, time) => {
             let obj = this.authorities.get(nodeId);
             if (!obj) {
-                // HACK: Sometimes when a node is first added we can't identify it yet. I THINK this is because
-                //  we haven't learned to trust the key, or... something? Hmm...
-                // ingestNewNodeIds([nodeId], []);
+                // Might as well use this to add the node, if we don't know about it yet.
+                ingestNewNodeIds([nodeId], []);
                 return;
             }
             obj.isReadReady = time;
@@ -936,7 +935,7 @@ function authoritiesMightOverlap(other: AuthorityPath, current: AuthorityPath):
 
 
 
-let onReadyReady = (nodeId: string, time: number) => { };
+let onReadReady = (nodeId: string, time: number) => { };
 class PathControllerBase {
     public async getAuthorityPaths() {
         return pathValueAuthority2.getSelfAuthorities();
@@ -953,7 +952,7 @@ class PathControllerBase {
     public async broadcastReadReady(time: number) {
         let nodeIdCaller = IdentityController_getCurrentReconnectNodeIdAssert();
         console.log(magenta(`Received ready broadcast`), { nodeIdCaller });
-        onReadyReady(nodeIdCaller, time);
+        onReadReady(nodeIdCaller, time);
     }
 }
 const PathController = SocketFunction.register(

package/src/3-path-functions/pathFunctionLoader.ts

@@ -281,10 +281,6 @@ async function getModuleFromSpecBase(
         deployPath = packagePath + "deploy.ts";
     }
 
-    if (path.startsWith("/root")) {
-        devDebugbreak();
-    }
-
     console.log(blue(`require(${JSON.stringify(path)})`));
 
     // Set functionSpec for the next synchronous evaluation

package/src/4-querysub/QuerysubController.ts

@@ -236,7 +236,7 @@ export async function baseAddCall(call: CallSpec, nodeId: string, cancel: () =>
     }
 }
 
-class QuerysubControllerBase {
+export class QuerysubControllerBase {
     public async watch(config: WatchConfig) {
         for (let path of config.paths) {
             Querysub.assertDomainAllowed(path);

package/src/4-querysub/querysubPrediction.ts

@@ -8,7 +8,7 @@ import { CallSpec, FunctionResult, FunctionSpec, debugCallSpec, functionSchema,
 import { getModuleFromConfig, setGitURLMapping } from "../3-path-functions/pathFunctionLoader";
 import { logErrors } from "../errors";
 import { getParentPathStr, getPathFromStr, getPathStr1, getPathStr2 } from "../path";
-import { Querysub, QuerysubController, baseAddCall, callWaitOn, querysubNodeId } from "./QuerysubController";
+import { Querysub, QuerysubController, QuerysubControllerBase, baseAddCall, callWaitOn, querysubNodeId } from "./QuerysubController";
 import { Benchmark } from "../diagnostics/benchmark";
 import { parseArgs } from "../3-path-functions/PathFunctionHelpers";
 import { runInSerial } from "socket-function/src/batching";
@@ -27,9 +27,18 @@ const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
 //  do too much for us if we already have the fully resolved path...
 //  - Although using it DOES allow permissions checks to work nicely, so, eh... maybe it is fine to use pathFunctionLoader?
 const addModuleToLoader = cacheJSONArgsEqual(async (spec: FunctionSpec): Promise<void> => {
-    let nodeId = await querysubNodeId();
-    if (!nodeId) throw new Error("No querysub node found");
-    let path = await QuerysubController.nodes[nodeId].getModulePath({ functionSpec: spec });
+    let controller: QuerysubControllerBase;
+    if (isNode()) {
+        // NOTE: If we are on node, then the require WON'T run over the network, so we need to use
+        //  our local module path, not the remove one.
+        controller = new QuerysubControllerBase();
+    } else {
+        let nodeId = await querysubNodeId();
+        if (!nodeId) throw new Error("No querysub node found");
+        controller = QuerysubController.nodes[nodeId] as any;
+    }
+
+    let path = await controller.getModulePath({ functionSpec: spec });
     setGitURLMapping({ spec, resolvedPath: path });
 });
 
@@ -503,6 +512,9 @@ export async function getCallWrites(config: {
     let exportPath = getPathFromStr(functionSpec.exportPathStr);
     let exportObj = module.exports;
     for (let path of exportPath) {
+        if (!(path in exportObj)) {
+            throw new Error(`Export not found for call prediction: ${path}, in ${call.DomainName}.${call.ModuleId}.${call.FunctionId}. Have ${Object.keys(exportObj)}`);
+        }
         exportObj = exportObj[path];
     }
     let baseFunction = exportObj as Function;

package/src/5-diagnostics/synchronousLagTracking.ts

@@ -4,6 +4,7 @@ import { registerMeasureInfo } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
 import { monitorEventLoopDelay } from "perf_hooks";
 import { registerNodeMetadata } from "./nodeMetadata";
+import { getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 
 const POLL_INTERVAL = 350;
 
@@ -29,6 +30,8 @@ export function trackSynchronousLag() {
                 return [`< ${formatTime(max / 1e6)}`, `99% < ${formatTime(p99 / 1e6)}`, `50% < ${formatTime(p50 / 1e6)}`];
             }
         });
+
+        registerMeasureInfo(() => getOwnNodeId());
     } else {
         let lastLag = POLL_INTERVAL;
         let time = Date.now();

package/src/diagnostics/NodeViewer.tsx

@@ -185,7 +185,7 @@ export class NodeViewer extends qreact.Component {
         }
 
         let builtinGroups = {
-            "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "Heap", "All Memory", "Blocking Lag", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
+            "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "Heap", "Buffers", "All Memory", "Blocking Lag", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
         };
         // Column => group
         let builtInGroupsLookup = new Map<string, string>();

package/src/diagnostics/trackResources.ts

@@ -55,6 +55,14 @@ function getUsedHeapSize() {
         return (window.performance as any).memory.usedJSHeapSize;
     }
 }
+function getBufferUsage() {
+    if (isNode()) {
+        let usage = process.memoryUsage();
+        return usage.arrayBuffers;
+    } else {
+        return 0;
+    }
+}
 
 function logResourcesNow() {
     let resourcesWithCounts = resources.map(resource => ({ ...resource, count: resource.getCount() }));
@@ -67,6 +75,7 @@ function logResourcesNow() {
         }
     } else {
         logNodeStateStats("Heap", formatNumber)(getUsedHeapSize());
+        logNodeStateStats("Buffers", formatNumber)(getBufferUsage());
         logNodeStateStats("All Memory", formatNumber)(getHeapSize());
         for (let resource of resourcesWithCounts) {
             if (resource.count === 0) continue;