socket-function 0.8.37 → 0.8.39

package/SocketFunction.ts

@@ -9,6 +9,7 @@ import { Args, MaybePromise } from "./src/types";
 import { setDefaultHTTPCall } from "./src/callHTTPHandler";
 import debugbreak from "debugbreak";
 import { lazy } from "./src/caching";
+import { delay } from "./src/batching";
 
 module.allowclient = true;
 
@@ -28,10 +29,16 @@ type PickByType<T, Value> = {
 
 export class SocketFunction {
     public static logMessages = false;
+
+    public static MAX_MESSAGE_SIZE = 1024 * 1024 * 32;
     public static compression: undefined | {
         type: "gzip";
     };
+
     public static httpETagCache = false;
+    public static silent = true;
+
+    public static WIRE_WARN_TIME = 100;
 
     private static onMountCallbacks = new Map<string, (() => MaybePromise<void>)[]>();
     public static exposedClasses = new Set<string>();
@@ -70,7 +77,7 @@ export class SocketFunction {
             return shape as any as SocketExposedShape;
         });
 
-        setImmediate(() => {
+        void Promise.resolve().then(() => {
             registerClass(classGuid, instance as SocketExposedInterface, getShape());
         });
 
@@ -82,14 +89,14 @@ export class SocketFunction {
                 console.log(`START\t\t\t${classGuid}.${functionName}`);
             }
             try {
-                let callFactory = await getCreateCallFactory(nodeId, SocketFunction.mountedNodeId);
+                let callFactory = await getCreateCallFactory(nodeId);
 
                 let shapeObj = getShape()[functionName];
                 if (!shapeObj) {
                     throw new Error(`Function ${functionName} is not in shape`);
                 }
 
-                let hookResult = await runClientHooks(call, shapeObj as SocketExposedShape[""]);
+                let hookResult = await runClientHooks(call, shapeObj as SocketExposedShape[""], callFactory.connectionId);
 
                 if ("overrideResult" in hookResult) {
                     return hookResult.overrideResult;
@@ -109,7 +116,7 @@ export class SocketFunction {
             _classGuid: classGuid,
         };
 
-        setImmediate(() => {
+        void Promise.resolve().then(() => {
             let onMount = getDefaultHooks?.().onMount;
             if (onMount) {
                 let callbacks = SocketFunction.onMountCallbacks.get(classGuid);
@@ -191,12 +198,22 @@ export class SocketFunction {
         }
     }
 
-    public static mountedNodeId: string = "NOTMOUNTED";
+    public static mountedNodeId: string = "";
+    public static mountedIP: string = "";
     private static hasMounted = false;
     public static async mount(config: SocketServerConfig) {
-        if (this.mountedNodeId !== "NOTMOUNTED") {
+        if (this.mountedNodeId) {
             throw new Error("SocketFunction already mounted, mounting twice in one thread is not allowed.");
         }
+
+        this.mountedIP = config.public ? "0.0.0.0" : "127.0.0.1";
+        if (config.ip) {
+            this.mountedIP = config.ip;
+        }
+
+        // Wait for any additionals functions to expose themselves
+        await delay("immediate");
+
         this.mountedNodeId = await startSocketServer(config);
         this.hasMounted = true;
         for (let classGuid of SocketFunction.exposedClasses) {

package/SocketFunctionTypes.ts

@@ -57,6 +57,7 @@ export type ClientHookContext<ExposedType extends SocketExposedInterface = Socke
     call: FullCallType;
     // If the result is overriden, we continue evaluating hooks BUT DO NOT perform the final call
     overrideResult?: unknown;
+    connectionId: { nodeId: string };
 };
 export interface SocketFunctionClientHook<ExposedType extends SocketExposedInterface = SocketExposedInterface> {
     (config: ClientHookContext<ExposedType>): MaybePromise<void>;
@@ -85,5 +86,6 @@ export type CallerContextBase = {
     // The nodeId they contacted. This is useful to determine their intention (otherwise
     //  requests can be redirected to us and would accept them, even though they are being
     //  blatantly MITMed).
+    //  IF they are the server, calling us back, then this will just be ""
     localNodeId: string;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "socket-function",
-	"version": "0.8.37",
+	"version": "0.8.39",
 	"main": "index.js",
 	"license": "MIT",
 	"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -9,7 +9,8 @@
 		"mobx": "^6.6.2",
 		"node-forge": "https://github.com/sliftist/forge#name",
 		"preact": "^10.10.6",
-		"typenode": "^4.9.4-a",
+		"rdtsc-now": "^0.3.0",
+		"typenode": "^4.9.4-c",
 		"ws": "^8.8.0"
 	},
 	"scripts": {
@@ -21,6 +22,7 @@
 		"@types/node-forge": "^1.3.1",
 		"@types/ws": "^8.5.3",
 		"debugbreak": "^0.6.5",
+		"pegjs": "^0.10.0",
 		"typedev": "^0.1.1"
 	}
 }

package/require/RequireController.ts

@@ -13,6 +13,11 @@ declare global {
             /** Indicates the module is allowed clientside. */
             allowclient?: boolean;
 
+            /** Causes the module to not preload, requiring `await import()` for it to load correctly
+             *      - Shouldn't be set recursively, otherwise nested packages will break.
+             */
+            lazyload?: boolean;
+
             /** Indicates the module is definitely not allowed clientside */
             serveronly?: boolean;
 
@@ -118,7 +123,8 @@ class RequireControllerBase {
         let modules: {
             [resolvedPath: string]: SerializedModule;
         } = Object.create(null);
-        function addModule(module: NodeJS.Module) {
+        function addModule(module: NodeJS.Module, rootImport = false) {
+            if (!rootImport && module.lazyload) return;
             if (!module.requireControllerSeqNum) {
                 module.requireControllerSeqNum = nextModuleSeqNum++;
             }
@@ -223,7 +229,7 @@ class RequireControllerBase {
                 clientModule = createNotFoundModule(`Module ${pathRequest} (resolved to ${resolvedPath}) is not allowed clientside (set module.allowclient in it, or call setFlag when it is imported).`);
             }
 
-            addModule(clientModule);
+            addModule(clientModule, true);
         }
 
         return { requestsResolvedPaths, modules, requireSeqNumProcessId };

package/require/require.js

@@ -16,7 +16,6 @@
         global: window,
     });
 
-
     // Not real modules, as we just define their exports
     const builtInModuleExports = {
         worker_threads: {
@@ -243,6 +242,15 @@
             }
 
             let resolvedPath = serializedModule.requests[request];
+            if (resolvedPath !== "NOTALLOWEDCLIENTSIDE" && !serializedModules[resolvedPath]) {
+                if (!asyncIsFine) {
+                    console.warn(`Accessed unexpected module %c${request}%c in %c${module.id}%c\n\tTreating it as an async require.\n\tAll modules require synchronously clientside must be required serverside at a module level.`,
+                        "color: red", "color: unset",
+                        "color: red", "color: unset",
+                    );
+                }
+                return rootRequire(resolvedPath);
+            }
 
             let exportsOverride = undefined;
             if (resolvedPath === "NOTALLOWEDCLIENTSIDE" || !serializedModules[resolvedPath].allowclient) {
@@ -253,6 +261,7 @@
                             if (property === "__esModule") return undefined;
                             // NOTE: Return a toString that evaluates to "" so we can EXPLICITLY detect non-loaded modules
                             if (property === unloadedModule) return true;
+                            if (property === "default") return exportsOverride;
 
                             throw new Error(`Module ${childId} is serverside only. Tried to access ${property} from ${module.id}`);
                         }
@@ -263,6 +272,7 @@
                             if (property === "__esModule") return undefined;
                             // NOTE: Return a toString that evaluates to "" so we can EXPLICITLY detect non-loaded modules
                             if (property === unloadedModule) return true;
+                            if (property === "default") return exportsOverride;
 
                             serializedModule;
 
@@ -336,6 +346,7 @@
         module.id = resolvedId;
         module.filename = serializedModule?.filename;
         module.exports = {};
+        module.exports.default = module.exports;
         module.children = [];
 
         module.load = load;
@@ -382,19 +393,6 @@
 
             let dirname = module.filename.replace(/\\/g, "/").split("/").slice(0, -1).join("/");
 
-            var __createBinding = (Object.create ? (function (o, m, k, k2) {
-                if (k2 === undefined) k2 = k;
-                Object.defineProperty(o, k2, { enumerable: true, get: function () { return m[k]; } });
-            }) : (function (o, m, k, k2) {
-                if (k2 === undefined) k2 = k;
-                o[k2] = m[k];
-            }));
-            var __setModuleDefault = (Object.create ? (function (o, v) {
-                Object.defineProperty(o, "default", { enumerable: true, value: v });
-            }) : function (o, v) {
-                o["default"] = v;
-            });
-
             let time = Date.now();
             currentModuleEvaluationStack.push(module.filename);
             try {
@@ -404,18 +402,11 @@
                         //  which checks for unloadedModule and returns undefined in that case.
                         __importStar(mod) {
                             if (mod[unloadedModule]) return undefined;
-                            if (mod && mod.__esModule) return mod;
-                            var result = {};
-                            if (mod !== null && mod !== undefined) {
-                                for (var k in mod) {
-                                    if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) {
-                                        __createBinding(result, mod, k);
-                                    }
-                                }
-                            }
-                            __setModuleDefault(result, mod);
-                            return result;
-                        }
+                            return mod;
+                        },
+                        __importDefault(mod) {
+                            return mod.default ? mod : { default: mod };
+                        },
                     },
                     module.exports,
                     module.require,

package/src/CallFactory.ts

@@ -1,7 +1,7 @@
 import { CallerContext, CallerContextBase, CallType, FullCallType } from "../SocketFunctionTypes";
 import * as ws from "ws";
 import { performLocalCall } from "./callManager";
-import { convertErrorStackToError, formatNumberSuffixed, isNode } from "./misc";
+import { convertErrorStackToError, formatNumberSuffixed, isNode, list } from "./misc";
 import { createWebsocketFactory, getTLSSocket } from "./websocketFactory";
 import { SocketFunction } from "../SocketFunction";
 import { gzip } from "zlib";
@@ -9,6 +9,9 @@ import * as tls from "tls";
 import { getClientNodeId, getNodeIdLocation, registerNodeClient } from "./nodeCache";
 import debugbreak from "debugbreak";
 import { lazy } from "./caching";
+import { JSONLACKS } from "./JSONLACKS/JSONLACKS";
+import { red } from "./formatting/logColors";
+import { isSplitableArray } from "./fixLargeNetworkCalls";
 
 const MIN_RETRY_DELAY = 1000;
 
@@ -37,6 +40,7 @@ export interface CallFactory {
     //  Trigger performLocalCall on the other side of the connection
     performCall(call: CallType): Promise<unknown>;
     onNextDisconnect(callback: () => void): void;
+    connectionId: { nodeId: string };
 }
 
 export interface SenderInterface {
@@ -56,8 +60,10 @@ export interface SenderInterface {
 
 export async function createCallFactory(
     webSocketBase: SenderInterface | undefined,
+    // The node id we are connecting to (or that connected to us)
     nodeId: string,
-    localNodeId: string,
+    // The node id that we were contacted on
+    localNodeId = "",
 ): Promise<CallFactory> {
     let niceConnectionName = nodeId;
 
@@ -66,8 +72,6 @@ export async function createCallFactory(
 
     const canReconnect = !!getNodeIdLocation(nodeId);
 
-    let lastReceivedSeqNum = 0;
-
     let pendingCalls: Map<number, {
         data: Buffer;
         call: InternalCallType;
@@ -76,7 +80,7 @@ export async function createCallFactory(
     // NOTE: It is important to make this as random as possible, to prevent
     //  reconnections dues to a process being reset causing seqNum collisions
     //  in return calls.
-    let nextSeqNum = Math.random();
+    let nextSeqNum = Date.now() + Math.random();
 
     let lastConnectionAttempt = 0;
 
@@ -93,6 +97,7 @@ export async function createCallFactory(
     let callFactory: CallFactory = {
         nodeId,
         lastClosed: 0,
+        connectionId: { nodeId },
         onNextDisconnect,
         async performCall(call: CallType) {
             let seqNum = nextSeqNum++;
@@ -105,7 +110,37 @@ export async function createCallFactory(
                 seqNum,
                 compress: !!SocketFunction.compression,
             };
-            let data = Buffer.from(JSON.stringify(fullCall));
+            let time = Date.now();
+            let data = Buffer.from(JSONLACKS.stringify(fullCall));
+            time = Date.now() - time;
+            if (time > SocketFunction.WIRE_WARN_TIME) {
+                console.log(red(`Slow serialize, took ${time}ms to serialize ${data.byteLength} bytes. For ${call.classGuid}.${call.functionName}`));
+            }
+
+            if (data.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
+                let splitArgIndex = call.args.findIndex(isSplitableArray);
+                if (splitArgIndex >= 0) {
+                    let SPLIT_GROUPS = 10;
+                    let splitArg = call.args[splitArgIndex] as unknown[];
+                    let subCalls = list(SPLIT_GROUPS).map(index => {
+                        let start = Math.floor(index / SPLIT_GROUPS * splitArg.length);
+                        let end = Math.floor((index + 1) / SPLIT_GROUPS * splitArg.length);
+                        return splitArg.slice(start, end);
+                    }).filter(x => x.length > 0);
+                    for (let splitList of subCalls) {
+                        let subCall = { ...call };
+                        subCall.args = subCall.args.slice();
+                        subCall.args[splitArgIndex] = splitList;
+                        await callFactory.performCall(subCall);
+                    }
+                    // Eh... we COULD return the array of results, but... then the result would sometimes be an array,
+                    //  some times not, so, it is better to return a string which will make it more clear why it sometimes varies.
+                    return "CALLS_SPLIT_DUE_TO_LARGE_ARGS";
+                }
+
+                throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${data.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
+            }
+
             let resultPromise = new Promise((resolve, reject) => {
                 let callback = (result: InternalReturnType) => {
                     if (SocketFunction.logMessages) {
@@ -137,6 +172,7 @@ export async function createCallFactory(
         registerOnce();
 
         function onClose(error: string) {
+            callFactory.connectionId = { nodeId };
             callFactory.lastClosed = Date.now();
             webSocketPromise = undefined;
             if (!canReconnect) {
@@ -181,7 +217,9 @@ export async function createCallFactory(
         if (newWebSocket.readyState === 0 /* CONNECTING */) {
             await new Promise<void>(resolve => {
                 newWebSocket.addEventListener("open", () => {
-                    console.log(`Connection established to ${niceConnectionName}`);
+                    if (!SocketFunction.silent) {
+                        console.log(`Connection established to ${niceConnectionName}`);
+                    }
                     callFactory.isConnected = true;
                     resolve();
                 });
@@ -248,9 +286,15 @@ export async function createCallFactory(
                     (message as any) = Buffer.from(arrayBuffer);
                 }
 
-                let call = JSON.parse(message.toString()) as InternalCallType | InternalReturnType;
+                let time = Date.now();
+                let call = JSONLACKS.parse(message.toString(), { extended: false }) as InternalCallType | InternalReturnType;
+                time = Date.now() - time;
+
                 if (call.isReturn) {
                     let callbackObj = pendingCalls.get(call.seqNum);
+                    if (time > SocketFunction.WIRE_WARN_TIME) {
+                        console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for receieving result of call to ${callbackObj?.call.classGuid}.${callbackObj?.call.functionName}`));
+                    }
                     if (!callbackObj) {
                         console.log(`Got return for unknown call ${call.seqNum}`);
                         return;
@@ -258,11 +302,9 @@ export async function createCallFactory(
                     call.resultSize = resultSize;
                     callbackObj.callback(call);
                 } else {
-                    if (call.seqNum <= lastReceivedSeqNum) {
-                        console.log(`Received out of sequence call ${call.seqNum}`);
-                        return;
+                    if (time > SocketFunction.WIRE_WARN_TIME) {
+                        console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for call to ${call.classGuid}.${call.functionName}`));
                     }
-                    lastReceivedSeqNum = call.seqNum;
 
                     let response: InternalReturnType;
                     try {
@@ -288,13 +330,24 @@ export async function createCallFactory(
                     let result: Buffer;
                     if (isNode() && call.compress && SocketFunction.compression?.type === "gzip") {
                         response.compressed = true;
-                        result = Buffer.from(JSON.stringify(response));
+                        result = Buffer.from(JSONLACKS.stringify(response));
                         result = await new Promise<Buffer>((resolve, reject) =>
                             gzip(result, (err, result) => err ? reject(err) : resolve(result))
                         );
                         result = Buffer.concat([new Uint8Array([0]), result]);
                     } else {
-                        result = Buffer.from(JSON.stringify(response));
+                        result = Buffer.from(JSONLACKS.stringify(response));
+                    }
+                    if (result.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
+                        response = {
+                            isReturn: true,
+                            result: undefined,
+                            seqNum: call.seqNum,
+                            error: new Error(`Response too large to send (${call.classGuid}.${call.functionName}, size: ${result.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`).stack,
+                            resultSize: resultSize,
+                            compressed: false,
+                        };
+                        result = Buffer.from(JSONLACKS.stringify(response));
                     }
                     await send(result);
                 }
@@ -302,6 +355,8 @@ export async function createCallFactory(
             }
             throw new Error(`Unhandled data type ${typeof message}`);
         } catch (e: any) {
+            debugbreak(1);
+            debugger;
             console.error(e.stack);
         }
     }