socket-function 0.8.39 → 0.9.0

package/SocketFunction.ts

@@ -201,6 +201,8 @@ export class SocketFunction {
     public static mountedNodeId: string = "";
     public static mountedIP: string = "";
     private static hasMounted = false;
+    private static onMountCallback: () => void = () => { };
+    public static mountPromise: Promise<void> = new Promise(r => this.onMountCallback = r);
     public static async mount(config: SocketServerConfig) {
         if (this.mountedNodeId) {
             throw new Error("SocketFunction already mounted, mounting twice in one thread is not allowed.");
@@ -223,6 +225,7 @@ export class SocketFunction {
                 await callback();
             }
         }
+        this.onMountCallback();
         return this.mountedNodeId;
     }
 
@@ -246,6 +249,10 @@ export class SocketFunction {
         return getNodeId(location.address, location.port);
     }
 
+    public static locationNode() {
+        return SocketFunction.connect({ address: location.hostname, port: +location.port });
+    }
+
     public static addGlobalHook(hook: SocketFunctionHook<SocketExposedInterface>) {
         registerGlobalHook(hook as SocketFunctionHook);
     }

package/hot/HotReloadController.ts

@@ -5,12 +5,25 @@ module.allowclient = true;
 import { SocketFunction } from "../SocketFunction";
 import { cache, lazy } from "../src/caching";
 import * as fs from "fs";
+import debugbreak from "debugbreak";
+import { isNode } from "../src/misc";
+import { red } from "../src/formatting/logColors";
 
-/** Hot reloads server and client files, just trigger a refresh clientside,
- *      while triggering per file re-evaluation and export updates serverside.
- *      - Requires HotReloadController to be exposed both serverside and clientside.
+/** Enables some hot reload functionality.
+ *      - Triggers a refresh clientside
+ *      - Triggers a reload server, for modules marked with `module.hotreload`
  */
-export function watchFilesAndTriggerHotReloading() {
+export function watchFilesAndTriggerHotReloading(noAutomaticBrowserWatch = false) {
+
+    SocketFunction.expose(HotReloadController);
+    if (!isNode()) {
+        if (!noAutomaticBrowserWatch) {
+            HotReloadController.nodes[SocketFunction.locationNode()]
+                .watchFiles()
+                .catch(e => console.error("watchFiles error", e))
+                ;
+        }
+    }
     setInterval(() => {
         for (let module of Object.values(require.cache)) {
             if (!module) continue;
@@ -19,6 +32,14 @@ export function watchFilesAndTriggerHotReloading() {
     }, 5000);
 }
 
+declare global {
+    namespace NodeJS {
+        interface Module {
+            hotreload?: boolean;
+            noserverhotreload?: boolean;
+        }
+    }
+}
 
 const hotReloadModule = cache((module: NodeJS.Module) => {
     if (!module.updateContents) return;
@@ -26,6 +47,23 @@ const hotReloadModule = cache((module: NodeJS.Module) => {
         if (curr.mtime.getTime() === prev.mtime.getTime()) return;
         console.log(`Hot reloading due to change: ${module.filename}`);
         module.updateContents?.();
+        if (isNode()) {
+            if (
+                module.hotreload
+                // A fairly big hack (as this could just be in a string, or something similar), but... it also VERY useful
+                || module.moduleContents?.includes("\nmodule.hotreload = true;" + "\n")
+                || module.moduleContents?.includes("\r\nmodule.hotreload = true;" + "\r\n")
+            ) {
+                console.log(`Reloading ${module.id}`);
+                try {
+                    module.loaded = false;
+                    module.load(module.id);
+                } catch (e) {
+                    console.error(red(`Error hot reloading ${module.id}`));
+                    console.error(e);
+                }
+            }
+        }
         triggerClientSideReload();
     });
 });
@@ -50,10 +88,7 @@ class HotReloadControllerBase {
     // TODO: Also hot reload when we reconnect to the server, as it is likely setup will need to
     //  be rerun in that case as well (for example, we need to call watchFiles again!)
     async watchFiles() {
-        let callerId = HotReloadController.context.caller?.nodeId;
-        if (!callerId) {
-            throw new Error("No nodeId?");
-        }
+        let callerId = SocketFunction.getCaller().nodeId;
         clientWatcherNodes.add(callerId);
     }
     async fileUpdated() {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "socket-function",
-	"version": "0.8.39",
+	"version": "0.9.0",
 	"main": "index.js",
 	"license": "MIT",
 	"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -9,8 +9,8 @@
 		"mobx": "^6.6.2",
 		"node-forge": "https://github.com/sliftist/forge#name",
 		"preact": "^10.10.6",
-		"rdtsc-now": "^0.3.0",
-		"typenode": "^4.9.4-c",
+		"rdtsc-now": "^0.3.1",
+		"typenode": "^4.9.4-g",
 		"ws": "^8.8.0"
 	},
 	"scripts": {

package/require/require.js

@@ -1,4 +1,6 @@
 (function () {
+    //# sourceURL=require.js
+
     // Globals
     Object.assign(window, {
         process: {

package/src/CallFactory.ts

@@ -10,8 +10,9 @@ import { getClientNodeId, getNodeIdLocation, registerNodeClient } from "./nodeCa
 import debugbreak from "debugbreak";
 import { lazy } from "./caching";
 import { JSONLACKS } from "./JSONLACKS/JSONLACKS";
-import { red } from "./formatting/logColors";
-import { isSplitableArray } from "./fixLargeNetworkCalls";
+import { red, yellow } from "./formatting/logColors";
+import { isSplitableArray, markArrayAsSplitable } from "./fixLargeNetworkCalls";
+import { delay } from "./batching";
 
 const MIN_RETRY_DELAY = 1000;
 
@@ -56,6 +57,8 @@ export interface SenderInterface {
     addEventListener(event: "message", listener: (data: ws.RawData | ws.MessageEvent | string) => void): void;
 
     readyState: number;
+
+    ping?(): void;
 }
 
 export async function createCallFactory(
@@ -82,6 +85,18 @@ export async function createCallFactory(
     //  in return calls.
     let nextSeqNum = Date.now() + Math.random();
 
+    // NOTE: I'm not sure if this is needed, I thought it was, but... now I think
+    //  it probably isn't...
+    // if (webSocketBase?.readyState === 1 /* OPEN */ && webSocketBase.ping) {
+    //     // Heartbeat loop, otherwise onDisconnect is never called.
+    //     ((async () => {
+    //         while (webSocketBase?.readyState === 1 /* OPEN */ && webSocketBase.ping) {
+    //             await delay(1000 * 60);
+    //             webSocketBase.ping?.();
+    //         }
+    //     }))().catch(() => { });
+    // }
+
     let lastConnectionAttempt = 0;
 
     let callerContext: CallerContextBase = {
@@ -120,6 +135,7 @@ export async function createCallFactory(
             if (data.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
                 let splitArgIndex = call.args.findIndex(isSplitableArray);
                 if (splitArgIndex >= 0) {
+                    console.log(yellow(`Splitting large call due to large args: ${call.classGuid}.${call.functionName}`));
                     let SPLIT_GROUPS = 10;
                     let splitArg = call.args[splitArgIndex] as unknown[];
                     let subCalls = list(SPLIT_GROUPS).map(index => {
@@ -127,12 +143,15 @@ export async function createCallFactory(
                         let end = Math.floor((index + 1) / SPLIT_GROUPS * splitArg.length);
                         return splitArg.slice(start, end);
                     }).filter(x => x.length > 0);
-                    for (let splitList of subCalls) {
+
+                    let calls = subCalls.map(async splitList => {
                         let subCall = { ...call };
                         subCall.args = subCall.args.slice();
-                        subCall.args[splitArgIndex] = splitList;
+                        subCall.args[splitArgIndex] = markArrayAsSplitable(splitList);
                         await callFactory.performCall(subCall);
-                    }
+                    });
+                    await Promise.allSettled(calls);
+                    await Promise.all(calls);
                     // Eh... we COULD return the array of results, but... then the result would sometimes be an array,
                     //  some times not, so, it is better to return a string which will make it more clear why it sometimes varies.
                     return "CALLS_SPLIT_DUE_TO_LARGE_ARGS";
@@ -355,9 +374,15 @@ export async function createCallFactory(
             }
             throw new Error(`Unhandled data type ${typeof message}`);
         } catch (e: any) {
-            debugbreak(1);
-            debugger;
-            console.error(e.stack);
+            // NOTE: I'm looking for all types of errors here (specifically, .send errors), in case
+            //  there are errors I should be handling.
+            if (e.stack.startsWith("Error: Cannot send data to") && e.stack.includes("as the connection has closed")) {
+                // This is fine, just ignore it
+            } else {
+                debugbreak(2);
+                debugger;
+                console.error(e.stack);
+            }
         }
     }
 

package/src/batching.ts

@@ -113,15 +113,18 @@ export async function runInfinitePollCallAtStart(
     delayTime: number,
     fnc: () => Promise<void> | void
 ) {
-    void (async () => {
-        while (true) {
-            await delay(delayTime);
-            try {
-                await fnc();
-            } catch (e: any) {
-                console.error(`Error in infinite poll ${fnc.name} (continuing poll loop)\n${e.stack}`);
+    try {
+        return await fnc();
+    } finally {
+        void (async () => {
+            while (true) {
+                await delay(delayTime);
+                try {
+                    await fnc();
+                } catch (e: any) {
+                    console.error(`Error in infinite poll ${fnc.name} (continuing poll loop)\n${e.stack}`);
+                }
             }
-        }
-    })();
-    return await fnc();
+        })();
+    }
 }

package/src/caching.ts

@@ -3,7 +3,6 @@ import { AnyFunction, Args, canHaveChildren } from "./types";
 
 export function lazy<T>(factory: () => T): () => T {
     let value: { value: T } | undefined = undefined;
-
     return () => {
         if (!value) {
             value = { value: factory() };
@@ -33,8 +32,10 @@ export function cacheEmptyArray<T>(array: T[]): T[] {
 
 export function cache<Output, Key>(getValue: (key: Key) => Output): {
     (key: Key): Output;
+    // NOTE: If you want to clear all, just make a new cache!
     clear(key: Key): void;
     forceSet(key: Key, value: Output): void;
+    getAllKeys(): Key[];
 } {
     let startingCalculating = new Set<Key>();
     let values = new Map<Key, Output>();
@@ -61,6 +62,9 @@ export function cache<Output, Key>(getValue: (key: Key) => Output): {
         values.set(key, value);
         startingCalculating.add(key);
     };
+    cache.getAllKeys = () => {
+        return [...values.keys()];
+    };
     return cache;
 }
 

package/src/callHTTPHandler.ts

@@ -158,7 +158,7 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
 
 
         // NOTE: Our ETag caching is only to reduce data sent on the wire, we evaluate the calls
-        //  every time (so it is strictly a wire cache, not a computation cache)
+        //  every time (so it is strictly a wire cache for HTTP, not a computation cache)
         if (SocketFunction.httpETagCache) {
             response.setHeader("cache-control", "private, max-age=0, must-revalidate");
             let hash = sha256Hash(resultBuffer);

package/src/certStore.ts

@@ -1,5 +1,5 @@
 import * as tls from "tls";
-import { sha256Hash } from "./misc";
+import { isNode, sha256Hash } from "./misc";
 
 let trustedCerts = new Set<string>();
 let watchCallbacks = new Set<(certs: string[]) => void>();
@@ -15,8 +15,12 @@ export function trustCertificate(cert: string | Buffer) {
     }
 }
 export function getTrustedCertificates(): string[] {
-    //console.log(`trustedCerts = ${Array.from(trustedCerts).map(x => sha256Hash(x).slice(0, 10))}`);
-    return tls.rootCertificates.concat(Array.from(trustedCerts));
+    let certs: string[] = [];
+    if (isNode()) {
+        certs.push(...tls.rootCertificates);
+    }
+    certs.push(...Array.from(trustedCerts));
+    return certs;
 }
 
 export function watchTrustedCertificates(callback: (certs: string[]) => void) {

package/src/fixLargeNetworkCalls.ts

@@ -5,5 +5,5 @@ export function markArrayAsSplitable<T>(data: T[]): T[] {
 }
 export function isSplitableArray<T>(data: T): data is T & (unknown[]) {
     if (!Array.isArray(data)) return false;
-    return (data as any)[arrayIsSplitable] === true;
+    return !!(data as any)[arrayIsSplitable];
 }

package/src/misc.ts

@@ -139,10 +139,34 @@ export function getKeys(obj: unknown) {
     }
     return keyArray;
 }
+export function getStringKeys<T extends {}>(obj: T): ((keyof T) & string)[] {
+    return Object.keys(obj) as any;
+}
 
 if (isNode()) {
     // TODO: Find a better place for this...
     process.on("unhandledRejection", async (reason: any, promise) => {
         console.error(`Uncaught promise rejection: ${String(reason.stack || reason)}`);
     });
+}
+
+export function keyBy<T, K>(arr: T[], getKey: (value: T) => K): Map<K, T> {
+    let map = new Map<K, T>();
+    for (let item of arr) {
+        map.set(getKey(item), item);
+    }
+    return map;
+}
+export function keyByArray<T, K>(arr: T[], getKey: (value: T) => K): Map<K, T[]> {
+    let map = new Map<K, T[]>();
+    for (let item of arr) {
+        let key = getKey(item);
+        let arr = map.get(key);
+        if (!arr) {
+            arr = [];
+            map.set(key, arr);
+        }
+        arr.push(item);
+    }
+    return map;
 }

package/src/profiling/measure.ts

@@ -1,6 +1,6 @@
 import debugbreak from "debugbreak";
 import { formatTime, formatNumber } from "../formatting/format";
-import { red, yellow, blue } from "../formatting/logColors";
+import { red, yellow, blue, magenta } from "../formatting/logColors";
 
 import { getOwnTime, getPendingOwnTimeInstances, getPendingOwnTimeObjs, OwnTimeObj } from "./getOwnTime";
 import { addToStats, addToStatsValue, createStatsValue, getStatsTop, StatsValue } from "./stats";
@@ -108,9 +108,9 @@ export function logMeasureTable(
     let totalTime = entries.map(x => getTime(x).sum).reduce((a, b) => a + b, 0);
 
     console.log();
-    let title = yellow(`Profiled ${formatTime(totalTime)}`);
+    let title = yellow(`Profiled ${formatTime(totalTime)} (logged at ${new Date().toISOString()})`);
     if (name) {
-        title += ` ${blue(name)}`;
+        title = `(${blue(name)}) ${title}`;
     }
     console.log(title);
     function percent(value: number) {
@@ -142,7 +142,7 @@ export function logMeasureTable(
 
         let ownTimeTop = getStatsTop(getTime(entry));
         if (ownTimeTop.topHeavy) {
-            output += red(`    TOP ${percent(ownTimeTop.valueFraction)} of the time is owned by ${percent(ownTimeTop.countFraction)} of the calls`);
+            output += red(`    TOP ${percent(ownTimeTop.valueFraction)} of the time is owned by ${percent(ownTimeTop.countFraction)} of the calls (${formatTime(ownTimeTop.value / ownTimeTop.count)} per * ${formatNumber(ownTimeTop.count)} = ${formatTime(ownTimeTop.value)})`);
         }
 
         if (entry.stillOpenCount > 0) {

package/src/webSocketServer.ts

@@ -12,6 +12,10 @@ import debugbreak from "debugbreak";
 import { getNodeId } from "./nodeCache";
 import crypto from "crypto";
 import { Watchable } from "./misc";
+import { delay, runInfinitePoll } from "./batching";
+import { magenta } from "./formatting/logColors";
+import { yellow } from "./formatting/logColors";
+import { green } from "./formatting/logColors";
 
 export type SocketServerConfig = (
     https.ServerOptions & {
@@ -27,6 +31,11 @@ export type SocketServerConfig = (
         public?: boolean;
         ip?: string;
 
+        // NOTE: Any same origin accesses are allowed (header.origin === header.host)
+        // For example, to allow "letx.ca" to access the server (when the hosted domain
+        //  may be, "querysub.com", for example), use ["letx.ca"]
+        allowHostnames?: string[];
+
         /** If the SNI matches this domain, we use a different key/cert. */
         SNICerts?: {
             [domain: string]: Watchable<https.ServerOptions>;
@@ -58,6 +67,11 @@ export async function startSocketServer(
         });
         let httpsServer = await httpServerPromise;
 
+        let allowedHostnames = new Set<string>();
+        for (let hostname of config.allowHostnames || []) {
+            allowedHostnames.add(hostname);
+        }
+
         watchTrustedCertificates(() => {
             lastOptions.ca = getTrustedCertificates();
             httpsServer.setSecureContext(lastOptions);
@@ -99,8 +113,8 @@ export async function startSocketServer(
                 try {
                     let host = new URL("ws://" + request.headers["host"]).hostname;
                     let origin = new URL(originHeader).hostname;
-                    if (host !== origin) {
-                        throw new Error(`Invalid cross thread request, ${JSON.stringify(host)} !== ${JSON.stringify(origin)}`);
+                    if (host !== origin && !allowedHostnames.has(origin)) {
+                        throw new Error(`Invalid cross domain request, ${JSON.stringify(host)} !== ${JSON.stringify(origin)} (also not config.allowedHostnames ${JSON.stringify(config.allowHostnames)})`);
                     }
                 } catch (e) {
                     console.error(e);
@@ -213,7 +227,7 @@ export async function startSocketServer(
     }
 
     if (!SocketFunction.silent) {
-        console.log(`Trying to listening on ${host}:${port}`);
+        console.log(yellow(`Trying to listening on ${host}:${port}`));
     }
     realServer.listen(port, host);
 
@@ -222,7 +236,7 @@ export async function startSocketServer(
     port = (realServer.address() as net.AddressInfo).port;
     let nodeId = getNodeId(getCommonName(config.cert), port);
     if (!SocketFunction.silent) {
-        console.log(`Started Listening on ${nodeId}`);
+        console.log(green(`Started Listening on ${nodeId}`));
     }
 
     return nodeId;

package/src/websocketFactory.ts

@@ -38,7 +38,7 @@ export function createWebsocketFactory(): (nodeId: string) => SenderInterface {
                 console.log(`Connecting to ${address}:${port}`);
             }
             let webSocket = new ws.WebSocket(`wss://${address}:${port}`, {
-                ca: tls.rootCertificates.concat(getTrustedCertificates()),
+                ca: getTrustedCertificates()
             });
             let result = Object.assign(webSocket, { socket: undefined as tls.TLSSocket | undefined });
             webSocket.once("upgrade", e => {

package/test/shared.ts

@@ -8,19 +8,13 @@ class TestBase {
     memberVariable = 5;
 
     async add(lhs: number, rhs: number) {
-        let caller = Test.context.caller?.nodeId;
-        if (!caller) {
-            throw new Error("No caller?");
-        }
+        let caller = SocketFunction.getCaller().nodeId;
         console.log(`Caller is ${caller}`);
         return lhs + rhs;
     }
 
     async callMe() {
-        let caller = Test.context.caller?.nodeId;
-        if (!caller) {
-            throw new Error("No caller?");
-        }
+        let caller = SocketFunction.getCaller().nodeId;
         console.log(`Caller is ${caller}`);
         void (async () => {
             let seqNum = 1;