socket-function 0.8.36 → 0.8.38

package/SocketFunction.ts

@@ -1,9 +1,9 @@
 /// <reference path="./require/RequireController.ts" />
 
-import { SocketExposedInterface, CallContextType, SocketFunctionHook, SocketFunctionClientHook, SocketExposedShape, SocketRegistered, CallerContext, FullCallType } from "./SocketFunctionTypes";
+import { SocketExposedInterface, SocketFunctionHook, SocketFunctionClientHook, SocketExposedShape, SocketRegistered, CallerContext, FullCallType } from "./SocketFunctionTypes";
 import { exposeClass, registerClass, registerGlobalClientHook, registerGlobalHook, runClientHooks } from "./src/callManager";
 import { SocketServerConfig, startSocketServer } from "./src/webSocketServer";
-import { getCreateCallFactoryLocation, getNodeId, getNodeIdLocation } from "./src/nodeCache";
+import { getCallFactory, getCreateCallFactory, getNodeId, getNodeIdLocation } from "./src/nodeCache";
 import { getCallProxy } from "./src/nodeProxy";
 import { Args, MaybePromise } from "./src/types";
 import { setDefaultHTTPCall } from "./src/callHTTPHandler";
@@ -28,20 +28,32 @@ type PickByType<T, Value> = {
 
 export class SocketFunction {
     public static logMessages = false;
+
+    public static MAX_MESSAGE_SIZE = 1024 * 1024 * 32;
     public static compression: undefined | {
         type: "gzip";
     };
+
     public static httpETagCache = false;
+    public static silent = true;
+
+    public static WIRE_WARN_TIME = 100;
 
     private static onMountCallbacks = new Map<string, (() => MaybePromise<void>)[]>();
     public static exposedClasses = new Set<string>();
 
+    public static callerContext: CallerContext | undefined;
+    public static getCaller(): CallerContext {
+        const caller = SocketFunction.callerContext;
+        if (!caller) throw new Error(`Tried to access caller when not in the synchronous phase of a function call`);
+        return caller;
+    }
+
     // NOTE: We use callbacks we don't run into issues with cyclic dependencies
     //  (ex, using a hook in a controller where the hook also calls the controller).
     public static register<
         ClassInstance extends object,
-        Shape extends SocketExposedShape<SocketExposedInterface, CallContext>,
-        CallContext extends CallContextType
+        Shape extends SocketExposedShape<SocketExposedInterface>,
     >(
         classGuid: string,
         instance: ClassInstance,
@@ -49,7 +61,7 @@ export class SocketFunction {
         defaultHooksFnc?: () => SocketExposedShape[""] & {
             onMount?: () => MaybePromise<void>;
         }
-    ): SocketRegistered<ExtractShape<ClassInstance, Shape>, CallContext> {
+    ): SocketRegistered<ExtractShape<ClassInstance, Shape>> {
         let getDefaultHooks = defaultHooksFnc && lazy(defaultHooksFnc);
         const getShape = lazy(() => {
             let shape = shapeFnc();
@@ -76,14 +88,14 @@ export class SocketFunction {
                 console.log(`START\t\t\t${classGuid}.${functionName}`);
             }
             try {
-                let callFactory = await getCreateCallFactoryLocation(nodeId, SocketFunction.mountedNodeId);
+                let callFactory = await getCreateCallFactory(nodeId);
 
                 let shapeObj = getShape()[functionName];
                 if (!shapeObj) {
                     throw new Error(`Function ${functionName} is not in shape`);
                 }
 
-                let hookResult = await runClientHooks(call, shapeObj as SocketExposedShape[""]);
+                let hookResult = await runClientHooks(call, shapeObj as SocketExposedShape[""], callFactory.connectionId);
 
                 if ("overrideResult" in hookResult) {
                     return hookResult.overrideResult;
@@ -99,7 +111,6 @@ export class SocketFunction {
         });
 
         let output: SocketRegistered = {
-            context: curSocketContext,
             nodes: nodeProxy,
             _classGuid: classGuid,
         };
@@ -119,6 +130,40 @@ export class SocketFunction {
         return output as any;
     }
 
+    public static onNextDisconnect(nodeId: string, callback: () => void) {
+        (async () => {
+            let factory = await getCallFactory(nodeId);
+            if (!factory) {
+                callback();
+                return;
+            }
+
+            factory.onNextDisconnect(callback);
+        })().catch(() => {
+            callback();
+        });
+    }
+    public static getLastDisconnectTime(nodeId: string): number | undefined {
+        let factory = getCallFactory(nodeId);
+        if (!factory) {
+            return undefined;
+        }
+        if (factory instanceof Promise) {
+            return undefined;
+        }
+        return factory.lastClosed;
+    }
+    public static isNodeConnected(nodeId: string): boolean {
+        let factory = getCallFactory(nodeId);
+        if (!factory) {
+            return false;
+        }
+        if (factory instanceof Promise) {
+            return false;
+        }
+        return !!factory.isConnected;
+    }
+
     /** NOTE: Only works if the nodeIs used is from SocketFunction.connect (we can't convert arbitrary nodeIds into urls,
      *      as we have no way of knowing how to contain a nodeId).
      *  */
@@ -152,12 +197,18 @@ export class SocketFunction {
         }
     }
 
-    public static mountedNodeId: string = "NOTMOUNTED";
+    public static mountedNodeId: string = "";
+    public static mountedIP: string = "";
     private static hasMounted = false;
     public static async mount(config: SocketServerConfig) {
-        if (this.mountedNodeId !== "NOTMOUNTED") {
+        if (this.mountedNodeId) {
             throw new Error("SocketFunction already mounted, mounting twice in one thread is not allowed.");
         }
+
+        this.mountedIP = config.public ? "0.0.0.0" : "127.0.0.1";
+        if (config.ip) {
+            this.mountedIP = config.ip;
+        }
         this.mountedNodeId = await startSocketServer(config);
         this.hasMounted = true;
         for (let classGuid of SocketFunction.exposedClasses) {
@@ -190,41 +241,29 @@ export class SocketFunction {
         return getNodeId(location.address, location.port);
     }
 
-    public static addGlobalHook<CallContext extends CallContextType>(hook: SocketFunctionHook<SocketExposedInterface, CallContext>) {
+    public static addGlobalHook(hook: SocketFunctionHook<SocketExposedInterface>) {
         registerGlobalHook(hook as SocketFunctionHook);
     }
-    public static addGlobalClientHook<CallContext extends CallContextType>(hook: SocketFunctionClientHook<SocketExposedInterface, CallContext>) {
+    public static addGlobalClientHook(hook: SocketFunctionClientHook<SocketExposedInterface>) {
         registerGlobalClientHook(hook as SocketFunctionClientHook);
     }
 }
 
 
-const curSocketContext: SocketRegistered["context"] = {
-    curContext: undefined,
-    caller: undefined,
-    getCaller() {
-        const caller = curSocketContext.caller;
-        if (!caller) throw new Error(`Tried to access caller when not in the synchronous phase of a function call`);
-        return caller;
-    }
-};
 let socketContextSeqNum = 1;
 
 export function _setSocketContext<T>(
-    callContext: CallContextType,
     caller: CallerContext,
     code: () => T,
 ) {
     socketContextSeqNum++;
     let seqNum = socketContextSeqNum;
-    curSocketContext.curContext = callContext;
-    curSocketContext.caller = caller;
+    SocketFunction.callerContext = caller;
     try {
         return code();
     } finally {
         if (seqNum === socketContextSeqNum) {
-            curSocketContext.curContext = undefined;
-            curSocketContext.caller = undefined;
+            SocketFunction.callerContext = undefined;
         }
     }
 }

package/SocketFunctionTypes.ts

@@ -2,6 +2,7 @@
 
 module.allowclient = true;
 
+import { SocketFunction } from "./SocketFunction";
 import { getCallObj } from "./src/nodeProxy";
 import { Args, MaybePromise } from "./src/types";
 
@@ -21,14 +22,14 @@ export type SocketExposedInterfaceClass = {
     new(): unknown;
     prototype: unknown;
 };
-export interface SocketExposedShape<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> {
+export interface SocketExposedShape<ExposedType extends SocketExposedInterface = SocketExposedInterface> {
     [functionName: string]: {
         /** Indicates with the same input, we give the same output, forever,
          *      independent of code changes. This only works for data storage.
          */
         dataImmutable?: boolean;
-        hooks?: SocketFunctionHook<ExposedType, CallContext>[];
-        clientHooks?: SocketFunctionClientHook<ExposedType, CallContext>[];
+        hooks?: SocketFunctionHook<ExposedType>[];
+        clientHooks?: SocketFunctionClientHook<ExposedType>[];
     };
 }
 
@@ -41,32 +42,28 @@ export interface FullCallType extends CallType {
     nodeId: string;
 }
 
-export interface SocketFunctionHook<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> {
-    (config: HookContext<ExposedType, CallContext>): MaybePromise<void>;
+export interface SocketFunctionHook<ExposedType extends SocketExposedInterface = SocketExposedInterface> {
+    (config: HookContext<ExposedType>): MaybePromise<void>;
     /** NOTE: This is useful when we need a clientside hook to set up state specifically for our serverside hook. */
-    clientHook?: SocketFunctionClientHook<ExposedType, CallContext>;
+    clientHook?: SocketFunctionClientHook<ExposedType>;
 }
-export type HookContext<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> = {
+export type HookContext<ExposedType extends SocketExposedInterface = SocketExposedInterface> = {
     call: FullCallType;
-    context: SocketRegistered<ExposedType, CallContext>["context"];
     // If the result is overriden, we continue evaluating hooks BUT DO NOT perform the final call
     overrideResult?: unknown;
 };
 
-export type ClientHookContext<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> = {
+export type ClientHookContext<ExposedType extends SocketExposedInterface = SocketExposedInterface> = {
     call: FullCallType;
     // If the result is overriden, we continue evaluating hooks BUT DO NOT perform the final call
     overrideResult?: unknown;
+    connectionId: { nodeId: string };
 };
-export interface SocketFunctionClientHook<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> {
-    (config: ClientHookContext<ExposedType, CallContext>): MaybePromise<void>;
+export interface SocketFunctionClientHook<ExposedType extends SocketExposedInterface = SocketExposedInterface> {
+    (config: ClientHookContext<ExposedType>): MaybePromise<void>;
 }
 
-export type CallContextType = {
-    [key: string]: unknown;
-};
-
-export interface SocketRegistered<ExposedType = any, DynamicCallContext extends CallContextType = CallContextType> {
+export interface SocketRegistered<ExposedType = any> {
     nodes: {
         // NOTE: Don't pass around nodeId to other nodes, instead pass around NetworkLocation (which they
         //  then turn into a nodeId, which they can then check permissions on themself).
@@ -76,12 +73,6 @@ export interface SocketRegistered<ExposedType = any, DynamicCallContext extends
             }
         };
     };
-    context: {
-        // If undefined we are not synchronously in a call
-        curContext: DynamicCallContext | undefined;
-        caller: CallerContext | undefined;
-        getCaller(): CallerContext;
-    };
     _classGuid: string;
 }
 export type CallerContext = Readonly<CallerContextBase>;
@@ -95,5 +86,6 @@ export type CallerContextBase = {
     // The nodeId they contacted. This is useful to determine their intention (otherwise
     //  requests can be redirected to us and would accept them, even though they are being
     //  blatantly MITMed).
+    //  IF they are the server, calling us back, then this will just be ""
     localNodeId: string;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "socket-function",
-	"version": "0.8.36",
+	"version": "0.8.38",
 	"main": "index.js",
 	"license": "MIT",
 	"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -9,7 +9,8 @@
 		"mobx": "^6.6.2",
 		"node-forge": "https://github.com/sliftist/forge#name",
 		"preact": "^10.10.6",
-		"typenode": "^4.9.4-a",
+		"rdtsc-now": "^0.3.0",
+		"typenode": "^4.9.4-b",
 		"ws": "^8.8.0"
 	},
 	"scripts": {
@@ -21,6 +22,7 @@
 		"@types/node-forge": "^1.3.1",
 		"@types/ws": "^8.5.3",
 		"debugbreak": "^0.6.5",
+		"pegjs": "^0.10.0",
 		"typedev": "^0.1.1"
 	}
 }

package/require/require.js

@@ -16,7 +16,6 @@
         global: window,
     });
 
-
     // Not real modules, as we just define their exports
     const builtInModuleExports = {
         worker_threads: {
@@ -253,6 +252,7 @@
                             if (property === "__esModule") return undefined;
                             // NOTE: Return a toString that evaluates to "" so we can EXPLICITLY detect non-loaded modules
                             if (property === unloadedModule) return true;
+                            if (property === "default") return exportsOverride;
 
                             throw new Error(`Module ${childId} is serverside only. Tried to access ${property} from ${module.id}`);
                         }
@@ -263,6 +263,7 @@
                             if (property === "__esModule") return undefined;
                             // NOTE: Return a toString that evaluates to "" so we can EXPLICITLY detect non-loaded modules
                             if (property === unloadedModule) return true;
+                            if (property === "default") return exportsOverride;
 
                             serializedModule;
 
@@ -415,7 +416,12 @@
                             }
                             __setModuleDefault(result, mod);
                             return result;
-                        }
+                        },
+                        __importDefault(mod) {
+                            // If typescript isn't going to complain about importing from a module with no default export,
+                            //  then we'll just change our implementation to work the same way as typescript types...
+                            return mod.default ? mod : { default: mod };
+                        },
                     },
                     module.exports,
                     module.require,

package/src/CallFactory.ts

@@ -9,6 +9,10 @@ import * as tls from "tls";
 import { getClientNodeId, getNodeIdLocation, registerNodeClient } from "./nodeCache";
 import debugbreak from "debugbreak";
 import { lazy } from "./caching";
+import { JSONLACKS } from "./JSONLACKS/JSONLACKS";
+import { red } from "./formatting/logColors";
+
+const MIN_RETRY_DELAY = 1000;
 
 type InternalCallType = FullCallType & {
     seqNum: number;
@@ -28,10 +32,14 @@ type InternalReturnType = {
 
 export interface CallFactory {
     nodeId: string;
+    lastClosed: number;
+    closedForever?: boolean;
+    isConnected?: boolean;
     // NOTE: May or may not have reconnection or retry logic inside of performCall.
     //  Trigger performLocalCall on the other side of the connection
     performCall(call: CallType): Promise<unknown>;
-    closedForever: boolean;
+    onNextDisconnect(callback: () => void): void;
+    connectionId: { nodeId: string };
 }
 
 export interface SenderInterface {
@@ -51,15 +59,17 @@ export interface SenderInterface {
 
 export async function createCallFactory(
     webSocketBase: SenderInterface | undefined,
+    // The node id we are connecting to (or that connected to us)
     nodeId: string,
-    localNodeId: string,
+    // The node id that we were contacted on
+    localNodeId = "",
 ): Promise<CallFactory> {
     let niceConnectionName = nodeId;
 
     const createWebsocket = createWebsocketFactory();
     const registerOnce = lazy(() => registerNodeClient(callFactory));
 
-    let lastReceivedSeqNum = 0;
+    const canReconnect = !!getNodeIdLocation(nodeId);
 
     let pendingCalls: Map<number, {
         data: Buffer;
@@ -69,21 +79,26 @@ export async function createCallFactory(
     // NOTE: It is important to make this as random as possible, to prevent
     //  reconnections dues to a process being reset causing seqNum collisions
     //  in return calls.
-    let nextSeqNum = Math.random();
+    let nextSeqNum = Date.now() + Math.random();
+
+    let lastConnectionAttempt = 0;
 
     let callerContext: CallerContextBase = {
         nodeId,
         localNodeId
     };
 
+    let disconnectCallbacks: (() => void)[] = [];
+    function onNextDisconnect(callback: () => void): void {
+        disconnectCallbacks.push(callback);
+    }
+
     let callFactory: CallFactory = {
         nodeId,
-        closedForever: false,
+        lastClosed: 0,
+        connectionId: { nodeId },
+        onNextDisconnect,
         async performCall(call: CallType) {
-            if (callFactory.closedForever) {
-                throw new Error(`Connection lost to ${niceConnectionName}`);
-            }
-
             let seqNum = nextSeqNum++;
             let fullCall: InternalCallType = {
                 nodeId,
@@ -94,7 +109,12 @@ export async function createCallFactory(
                 seqNum,
                 compress: !!SocketFunction.compression,
             };
-            let data = Buffer.from(JSON.stringify(fullCall));
+            let time = Date.now();
+            let data = Buffer.from(JSONLACKS.stringify(fullCall));
+            time = Date.now() - time;
+            if (time > SocketFunction.WIRE_WARN_TIME) {
+                console.log(red(`Slow serialize, took ${time}ms to serialize ${data.byteLength} bytes. For ${call.classGuid}.${call.functionName}`));
+            }
             let resultPromise = new Promise((resolve, reject) => {
                 let callback = (result: InternalReturnType) => {
                     if (SocketFunction.logMessages) {
@@ -110,6 +130,10 @@ export async function createCallFactory(
                 pendingCalls.set(seqNum, { callback, data, call: fullCall });
             });
 
+            if (data.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
+                throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${data.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
+            }
+
             await send(data);
 
             return await resultPromise;
@@ -126,7 +150,12 @@ export async function createCallFactory(
         registerOnce();
 
         function onClose(error: string) {
+            callFactory.connectionId = { nodeId };
+            callFactory.lastClosed = Date.now();
             webSocketPromise = undefined;
+            if (!canReconnect) {
+                callFactory.closedForever = true;
+            }
             for (let [key, call] of pendingCalls) {
                 pendingCalls.delete(key);
                 call.callback({
@@ -138,6 +167,14 @@ export async function createCallFactory(
                     compressed: false,
                 });
             }
+
+            let callbacks = disconnectCallbacks;
+            disconnectCallbacks = [];
+            for (let callback of callbacks) {
+                try {
+                    callback();
+                } catch { }
+            }
         }
 
         newWebSocket.addEventListener("error", e => {
@@ -158,7 +195,10 @@ export async function createCallFactory(
         if (newWebSocket.readyState === 0 /* CONNECTING */) {
             await new Promise<void>(resolve => {
                 newWebSocket.addEventListener("open", () => {
-                    console.log(`Connection established to ${niceConnectionName}`);
+                    if (!SocketFunction.silent) {
+                        console.log(`Connection established to ${niceConnectionName}`);
+                    }
+                    callFactory.isConnected = true;
                     resolve();
                 });
                 newWebSocket.addEventListener("close", () => resolve());
@@ -166,15 +206,29 @@ export async function createCallFactory(
             });
         } else if (newWebSocket.readyState !== 1 /* OPEN */) {
             onClose(`Websocket received in closed state`);
+            callFactory.isConnected = true;
         }
     }
 
     async function send(data: Buffer) {
-        webSocketPromise = webSocketPromise || tryToReconnect();
+        if (!webSocketPromise) {
+            if (canReconnect) {
+                webSocketPromise = tryToReconnect();
+            } else {
+                throw new Error(`Cannot send data to ${niceConnectionName} as the connection has closed`);
+            }
+        }
         let webSocket = await webSocketPromise;
         webSocket.send(data);
     }
     async function tryToReconnect(): Promise<SenderInterface> {
+        // Don't try to reconnect too often!
+        let timeSinceLastAttempt = Date.now() - lastConnectionAttempt;
+        if (timeSinceLastAttempt < MIN_RETRY_DELAY) {
+            await new Promise(r => setTimeout(r, MIN_RETRY_DELAY - timeSinceLastAttempt));
+        }
+        lastConnectionAttempt = Date.now();
+
         let newWebSocket = createWebsocket(nodeId);
         await initializeWebsocket(newWebSocket);
 
@@ -210,9 +264,15 @@ export async function createCallFactory(
                     (message as any) = Buffer.from(arrayBuffer);
                 }
 
-                let call = JSON.parse(message.toString()) as InternalCallType | InternalReturnType;
+                let time = Date.now();
+                let call = JSONLACKS.parse(message.toString(), { extended: false }) as InternalCallType | InternalReturnType;
+                time = Date.now() - time;
+
                 if (call.isReturn) {
                     let callbackObj = pendingCalls.get(call.seqNum);
+                    if (time > SocketFunction.WIRE_WARN_TIME) {
+                        console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for receieving result of call to ${callbackObj?.call.classGuid}.${callbackObj?.call.functionName}`));
+                    }
                     if (!callbackObj) {
                         console.log(`Got return for unknown call ${call.seqNum}`);
                         return;
@@ -220,11 +280,9 @@ export async function createCallFactory(
                     call.resultSize = resultSize;
                     callbackObj.callback(call);
                 } else {
-                    if (call.seqNum <= lastReceivedSeqNum) {
-                        console.log(`Received out of sequence call ${call.seqNum}`);
-                        return;
+                    if (time > SocketFunction.WIRE_WARN_TIME) {
+                        console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for call to ${call.classGuid}.${call.functionName}`));
                     }
-                    lastReceivedSeqNum = call.seqNum;
 
                     let response: InternalReturnType;
                     try {
@@ -250,13 +308,24 @@ export async function createCallFactory(
                     let result: Buffer;
                     if (isNode() && call.compress && SocketFunction.compression?.type === "gzip") {
                         response.compressed = true;
-                        result = Buffer.from(JSON.stringify(response));
+                        result = Buffer.from(JSONLACKS.stringify(response));
                         result = await new Promise<Buffer>((resolve, reject) =>
                             gzip(result, (err, result) => err ? reject(err) : resolve(result))
                         );
                         result = Buffer.concat([new Uint8Array([0]), result]);
                     } else {
-                        result = Buffer.from(JSON.stringify(response));
+                        result = Buffer.from(JSONLACKS.stringify(response));
+                    }
+                    if (result.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
+                        response = {
+                            isReturn: true,
+                            result: undefined,
+                            seqNum: call.seqNum,
+                            error: new Error(`Response too large to send (${call.classGuid}.${call.functionName}, size: ${result.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`).stack,
+                            resultSize: resultSize,
+                            compressed: false,
+                        };
+                        result = Buffer.from(JSONLACKS.stringify(response));
                     }
                     await send(result);
                 }
@@ -264,6 +333,8 @@ export async function createCallFactory(
             }
             throw new Error(`Unhandled data type ${typeof message}`);
         } catch (e: any) {
+            debugbreak(1);
+            debugger;
             console.error(e.stack);
         }
     }