socket-function 0.5.0

package/.eslintrc.js

@@ -0,0 +1,51 @@
+module.exports = {
+    "root": true,
+    "env": {
+        "browser": true,
+        "es6": true,
+        "node": true
+    },
+    "extends": [],
+    "globals": {
+        "Atomics": "readonly",
+        "SharedArrayBuffer": "readonly"
+    },
+    "overrides": [
+        {
+            "files": [
+                "**/*.ts",
+                "**/*.tsx"
+            ]
+        }
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaVersion": 2018,
+        "sourceType": "module",
+        "project": "./tsconfig.json",
+        "tsconfigRootDir": __dirname,
+    },
+    "plugins": [
+        "@typescript-eslint"
+    ],
+    "rules": {
+        "@typescript-eslint/unbound-method": [
+            "error",
+            {
+                "ignoreStatic": true
+            }
+        ],
+        "@typescript-eslint/no-floating-promises": [
+            "error"
+        ],
+        "quotes": [
+            "error",
+            "double",
+            {
+                "allowTemplateLiterals": true
+            }
+        ],
+        "eqeqeq": "error",
+        "semi": "error"
+    }
+};

package/.vscode/settings.json

@@ -0,0 +1,26 @@
+{
+    // Quentin: DON'T commit { debug.node.autoAttach: "on" }. This setting (while useful), has serious performance ramifications.
+    // "debug.node.autoAttach": "on",
+    "files.watcherExclude": {
+        "**/.git/objects/**": true,
+        "**/.git/subtree-cache/**": true,
+        "**/node_modules/*/**": true,
+        "**/.hg/store/**": true
+    },
+    "[typescript]": {
+        "editor.formatOnSave": true,
+    },
+    "[typescriptreact]": {
+        "editor.formatOnSave": true,
+    },
+    "[javascript]": {
+        "editor.formatOnSave": true,
+    },
+    "typescript.tsdk": "node_modules/typescript/lib",
+    "search.exclude": {
+        "**/node_modules": true,
+        "**/bower_components": true,
+        "**/*.code-search": true,
+        "**.cache": true
+    }
+}

package/CallInstance.ts

@@ -0,0 +1,323 @@
+import { CallerContext, CallType, NetworkLocation } from "./SocketFunctionTypes";
+import type * as ws from "ws";
+import type * as net from "net";
+import { performLocalCall } from "./callManager";
+import { convertErrorStackToError } from "./misc";
+import { createWebsocket, getNodeId, getTLSSocket } from "./nodeAuthentication";
+import debugbreak from "debugbreak";
+
+const retryInterval = 2000;
+
+type InternalCallType = CallType & {
+    seqNum: number;
+    isReturn: false;
+}
+
+type InternalReturnType = {
+    isReturn: true;
+    result: unknown;
+    error?: string;
+    seqNum: number;
+};
+
+
+export interface CallFactory {
+    nodeId: string;
+    location: NetworkLocation;
+    // NOTE: May or may not have reconnection or retry logic inside of performCall.
+    //  Trigger performLocalCall on the other side of the connection
+    performCall(call: CallType): Promise<unknown>;
+}
+
+
+export async function callFactoryFromLocation(
+    location: NetworkLocation
+): Promise<CallFactory> {
+    if (location.localPort !== 0) {
+        throw new Error(`Expected localPort to be 0, but it was ${location.localPort}`);
+    }
+
+    let listeningPort = location.listeningPorts[0];
+    if (typeof listeningPort !== "number") {
+        throw new Error(`Expected listeningPorts to be provided, but it was empty`);
+    }
+
+    return await createCallFactory(undefined, location);
+}
+
+export async function callFactoryFromWS(
+    webSocket: ws.WebSocket
+): Promise<CallFactory> {
+    let socket = getTLSSocket(webSocket);
+    let remoteAddress = socket.remoteAddress;
+    let remotePort = socket.remotePort;
+    if (!remoteAddress) {
+        throw new Error("No remote address?");
+    }
+    if (!remotePort) {
+        throw new Error("No remote port?");
+    }
+
+    // NOTE: We COULD reconnect to clients, but... chances are... when they go down,
+    //  their process is dead, and is going to stay dead.
+    let location: NetworkLocation = {
+        address: remoteAddress,
+        localPort: remotePort,
+        listeningPorts: [],
+    };
+
+    return await createCallFactory(webSocket, location);
+}
+
+async function createCallFactory(
+    webSocketBase: ws.WebSocket | undefined,
+    location: NetworkLocation,
+): Promise<CallFactory> {
+
+    let closedForever = false;
+
+    let niceConnectionName = `${location.address}:${location.localPort}`;
+
+    let retriesEnabled = location.listeningPorts.length === 0;
+
+    let lastReceivedSeqNum = 0;
+
+    let reconnectingPromise: Promise<void> | undefined;
+    let reconnectAttempts = 0;
+
+
+    let pendingCalls: Map<number, {
+        data: string;
+        call: InternalCallType;
+        reconnectTimeout: number | undefined;
+        callback: (result: InternalReturnType) => void;
+    }> = new Map();
+    // NOTE: It is important to make this as random as possible, to prevent
+    //  reconnections dues to a process being reset causing seqNum collisions
+    //  in return calls.
+    let nextSeqNum = Math.random();
+
+    const pendingNodeId = "PENDING";
+    let callerContext: CallerContext = { location, nodeId: pendingNodeId };
+    let webSocket!: ws.WebSocket;
+    if (!webSocketBase) {
+        await tryToReconnect();
+    } else {
+        webSocket = webSocketBase;
+        setupWebsocket(webSocketBase);
+    }
+    callerContext.nodeId = getNodeId(webSocket);
+
+    niceConnectionName = `${niceConnectionName} (${callerContext.nodeId})`;
+
+    async function sendWithRetry(reconnectTimeout: number | undefined, data: string) {
+        if (!retriesEnabled) {
+            webSocket.send(data);
+            return;
+        }
+
+        while (true) {
+            if (reconnectingPromise) {
+                if (reconnectTimeout) {
+                    await Promise.race([
+                        reconnectingPromise,
+                        new Promise<ws.WebSocket>(resolve =>
+                            setTimeout(() => {
+                                retriesEnabled = false;
+                                resolve(webSocket);
+                            }, reconnectTimeout)
+                        )
+                    ]);
+                } else {
+                    await reconnectingPromise;
+                }
+            }
+
+            if (!retriesEnabled) {
+                webSocket.send(data);
+                break;
+            }
+
+            try {
+                webSocket.send(data);
+                break;
+            } catch (e) {
+                // Ignore errors, as we will catch them synchronously in the next loop.
+                void (tryToReconnect());
+            }
+        }
+    }
+    function tryToReconnect(): Promise<void> {
+        if (reconnectingPromise) return reconnectingPromise;
+        return reconnectingPromise = (async () => {
+            while (true) {
+                let ports = location.listeningPorts;
+
+                if (ports.length === 0) {
+                    closedForever = true;
+                    console.log(`No ports to reconnect for ${niceConnectionName}, pendingCall count: ${pendingCalls.size}`);
+                    for (let call of pendingCalls.values()) {
+                        call.callback({
+                            isReturn: true,
+                            result: undefined,
+                            error: `Connection lost to ${location.address}:${location.localPort}`,
+                            seqNum: call.call.seqNum,
+                        });
+                    }
+                    return;
+                }
+
+                let port = ports[reconnectAttempts % ports.length];
+                let newWebSocket = createWebsocket(location.address, port);
+
+                setupWebsocket(newWebSocket);
+
+                let connectError = await new Promise<string | undefined>(resolve => {
+                    newWebSocket.once("open", () => {
+                        resolve(undefined);
+                    });
+                    newWebSocket.once("close", () => {
+                        resolve("Connection closed for non-error reason?");
+                    });
+                    newWebSocket.once("error", e => {
+                        resolve(String(e.stack));
+                    });
+                });
+
+                if (!connectError) {
+                    console.log(`Reconnected to ${location.address}:${port}`);
+
+                    let newNodeId = getNodeId(newWebSocket);
+
+                    let prevNodeId = callerContext.nodeId;
+                    if (prevNodeId === pendingNodeId) {
+                        callerContext.nodeId = newNodeId;
+                    } else {
+                        if (newNodeId !== prevNodeId) {
+                            throw new Error(`Connection lost to at ${niceConnectionName} ("${prevNodeId}"), but then re-established, however it is now "${newNodeId}"!`);
+                        }
+                    }
+
+                    // I'm not sure if we should clear reconnectAttempts? All the ports should be the same, and actually...
+                    //  why would there even be a bad port?
+                    //reconnectAttempts = 0;
+                    reconnectingPromise = undefined;
+
+                    webSocket = newWebSocket;
+
+                    for (let call of pendingCalls.values()) {
+                        sendWithRetry(call.reconnectTimeout, call.data).catch(e => {
+                            call.callback({
+                                isReturn: true,
+                                result: undefined,
+                                error: String(e),
+                                seqNum: call.call.seqNum,
+                            });
+                        });
+                    }
+                    return;
+                }
+
+                console.error(`Connection retry to ${location.address}:${port} failed, retrying in ${retryInterval}ms`);
+                reconnectAttempts++;
+                await new Promise(resolve => setTimeout(resolve, retryInterval));
+            }
+        })();
+    }
+
+    function setupWebsocket(webSocket: ws.WebSocket) {
+        webSocket.on("error", e => {
+            console.log(`Websocket error for ${niceConnectionName}`, e);
+        });
+
+        webSocket.on("close", async () => {
+            console.log(`Websocket closed ${niceConnectionName}`);
+            await tryToReconnect();
+        });
+
+        webSocket.on("message", onMessage);
+    }
+
+
+    async function onMessage(message: ws.RawData) {
+        try {
+            if (message instanceof Buffer) {
+                let call = JSON.parse(message.toString()) as InternalCallType | InternalReturnType;
+                if (call.isReturn) {
+                    let callbackObj = pendingCalls.get(call.seqNum);
+                    if (!callbackObj) {
+                        console.log(`Got return for unknown call ${call.seqNum}`);
+                        return;
+                    }
+                    callbackObj.callback(call);
+                } else {
+                    if (call.seqNum <= lastReceivedSeqNum) {
+                        console.log(`Received out of sequence call ${call.seqNum}`);
+                        return;
+                    }
+                    lastReceivedSeqNum = call.seqNum;
+
+                    let response: InternalReturnType;
+                    try {
+                        let result = await performLocalCall({ call, caller: callerContext });
+                        response = {
+                            isReturn: true,
+                            result,
+                            seqNum: call.seqNum,
+                        };
+                    } catch (e: any) {
+                        response = {
+                            isReturn: true,
+                            result: undefined,
+                            seqNum: call.seqNum,
+                            error: e.stack,
+                        };
+                    }
+
+                    await sendWithRetry(call.reconnectTimeout, JSON.stringify(response));
+                }
+                return;
+            }
+            debugbreak(1);
+            debugger;
+            throw new Error(`Unhandled data type ${typeof message}`);
+        } catch (e: any) {
+            console.error(e.stack);
+        }
+    }
+
+    return {
+        nodeId: callerContext.nodeId,
+        location,
+        async performCall(call: CallType) {
+            if (closedForever) {
+                throw new Error(`Connection lost to ${location.address}:${location.localPort}`);
+            }
+
+            let seqNum = nextSeqNum++;
+            let fullCall: InternalCallType = {
+                isReturn: false,
+                args: call.args,
+                classGuid: call.classGuid,
+                functionName: call.functionName,
+                seqNum,
+            };
+            let data = JSON.stringify(fullCall);
+            let resultPromise = new Promise((resolve, reject) => {
+                let callback = (result: InternalReturnType) => {
+                    pendingCalls.delete(seqNum);
+                    if (result.error) {
+                        reject(convertErrorStackToError(result.error));
+                    } else {
+                        resolve(result.result);
+                    }
+                };
+                pendingCalls.set(seqNum, { callback, data, call: fullCall, reconnectTimeout: call.reconnectTimeout });
+            });
+
+            await sendWithRetry(call.reconnectTimeout, data);
+
+            return await resultPromise;
+        }
+    };
+}

package/SocketFunction.ts

@@ -0,0 +1,131 @@
+import { SocketExposedInterface, CallContextType, SocketFunctionHook, SocketFunctionClientHook, SocketExposedShape, SocketRegistered, NetworkLocation, CallerContext, SocketExposedInterfaceClass, CallType } from "./SocketFunctionTypes";
+import { exposeClass, registerClass, registerGlobalClientHook, registerGlobalHook, runClientHooks } from "./callManager";
+import { SocketServerConfig, startSocketServer } from "./socketServer";
+import { getCallFactoryNodeId, getCreateCallFactoryLocation } from "./nodeCache";
+import { getCallProxy } from "./nodeProxy";
+
+
+type ExtractShape<ClassType, Shape> = {
+    [key in keyof Shape]: (
+        key extends keyof ClassType
+        ? ClassType[key] extends SocketExposedInterface[""]
+        ? ClassType[key]
+        : ClassType[key] extends Function ? "All exposed function must be async (or return a Promise)" : never
+        : "Function is in shape, but not in class"
+    );
+};
+// https://stackoverflow.com/a/69756175/1117119
+type PickByType<T, Value> = {
+    [P in keyof T as T[P] extends Value | undefined ? P : never]: T[P]
+};
+
+export class SocketFunction {
+    public static register<
+        ClassType extends SocketExposedInterfaceClass,
+        Shape extends SocketExposedShape<SocketExposedInterface, CallContext>,
+        CallContext extends CallContextType
+    >(
+        classGuid: string,
+        classType: ClassType,
+        shape: Shape
+    ): (
+            // Essentially just returns SocketRegistered
+            ExtractShape<ClassType["prototype"], Shape> extends SocketExposedInterface
+            ? SocketRegistered<ExtractShape<ClassType["prototype"], Shape>, CallContext>
+            : {
+                error: "invalid shape";
+            } & PickByType<ExtractShape<ClassType["prototype"], Shape>, string>
+        ) {
+        registerClass(classGuid, classType, shape as any as SocketExposedShape);
+
+        let nodeProxy = getCallProxy(classGuid, async (nodeId, functionName, args) => {
+            let callFactory = getCallFactoryNodeId(nodeId);
+            if (!callFactory) {
+                throw new Error(`Cannot reach node ${nodeId}. Either it was established via an HTTP call, or was incorrect provided to us via another node, which should have provided us a NetworkLocation instead.`);
+            }
+
+            let shapeObj = shape[functionName];
+            if (!shapeObj) {
+                throw new Error(`Function ${functionName} is not in shape`);
+            }
+
+            let call: CallType = {
+                classGuid,
+                args,
+                functionName,
+            };
+
+            let hookResult = await runClientHooks(call, shapeObj as SocketExposedShape[""]);
+
+            if ("overrideResult" in hookResult) {
+                return hookResult.overrideResult;
+            }
+
+            return await callFactory.performCall(call);
+        });
+
+        let output: SocketRegistered = {
+            context: curSocketContext,
+            nodes: nodeProxy,
+        };
+
+        return output as any;
+    }
+
+    /** Expose should be called before your mounting occurs. It mostly just exists to ensure you include the class type,
+     *      so the class type's module construction runs, which should trigger register. Otherwise you would have
+     *      to add additional imports to ensure the register call runs.
+     */
+    public static expose(classType: SocketExposedInterfaceClass) {
+        exposeClass(classType);
+    }
+
+    public static async mount(config: SocketServerConfig) {
+        await startSocketServer(config);
+    }
+
+    public static async connect(location: NetworkLocation | { address: string; port: number }): Promise<string> {
+        if (!("localPort" in location)) {
+            location = {
+                address: location.address,
+                listeningPorts: [location.port],
+                localPort: 0,
+            };
+        }
+        return await getCreateCallFactoryLocation(location);
+    }
+
+
+    public static addGlobalHook<CallContext extends CallContextType>(hook: SocketFunctionHook<SocketExposedInterface, CallContext>) {
+        registerGlobalHook(hook as SocketFunctionHook);
+    }
+    public static addGlobalClientHook<CallContext extends CallContextType>(hook: SocketFunctionClientHook<SocketExposedInterface, CallContext>) {
+        registerGlobalClientHook(hook as SocketFunctionClientHook);
+    }
+}
+
+
+const curSocketContext: SocketRegistered["context"] = {
+    curContext: undefined,
+    caller: undefined,
+};
+let socketContextSeqNum = 1;
+
+export function _setSocketContext<T>(
+    callContext: CallContextType,
+    caller: CallerContext,
+    code: () => T,
+) {
+    socketContextSeqNum++;
+    let seqNum = socketContextSeqNum;
+    curSocketContext.curContext = callContext;
+    curSocketContext.caller = caller;
+    try {
+        return code();
+    } finally {
+        if (seqNum === socketContextSeqNum) {
+            curSocketContext.curContext = undefined;
+            curSocketContext.caller = undefined;
+        }
+    }
+}

package/SocketFunctionTypes.ts

@@ -0,0 +1,80 @@
+export const socket = Symbol.for("socket");
+
+export type SocketExposedInterface = {
+    [functionName: string]: (...args: any[]) => Promise<unknown>;
+};
+export type SocketExposedInterfaceClass = {
+    //new(): SocketExposedInterface;
+    new(): unknown;
+    prototype: unknown;
+};
+export interface SocketExposedShape<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> {
+    [functionName: string]: {
+        hooks?: SocketFunctionHook<ExposedType, CallContext>[];
+        clientHooks?: SocketFunctionClientHook<ExposedType, CallContext>[];
+    };
+}
+
+export interface CallType {
+    classGuid: string;
+    functionName: string;
+    args: unknown[];
+    // NOTE: When making calls this needs to be set in the client hook.
+    //  To set a timeout on returns, you can set it in the server hook.
+    reconnectTimeout?: number;
+}
+
+export interface SocketFunctionHook<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> {
+    (config: HookContext<ExposedType, CallContext>): Promise<void>;
+}
+export type HookContext<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> = {
+    call: CallType;
+    context: SocketRegistered["context"];
+    // If the result is overriden, we continue evaluating hooks and perform the final call
+    overrideResult?: unknown;
+};
+
+export type ClientHookContext<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> = {
+    call: CallType;
+    // If the result is overriden, we continue evaluating hooks and perform the final call
+    overrideResult?: unknown;
+};
+export interface SocketFunctionClientHook<ExposedType extends SocketExposedInterface = SocketExposedInterface, CallContext extends CallContextType = CallContextType> {
+    (config: ClientHookContext<ExposedType, CallContext>): Promise<void>;
+}
+
+export type CallContextType = {
+    [key: string]: unknown;
+};
+
+export interface SocketRegistered<ExposedType extends SocketExposedInterface = SocketExposedInterface, DynamicCallContext extends CallContextType = CallContextType> {
+    nodes: {
+        // NOTE: Don't pass around nodeId to other nodes, instead pass around NetworkLocation (which they
+        //  then turn into a nodeId, which they can then check permissions on themself).
+        [nodeId: string]: ExposedType;
+    };
+    context: {
+        // If undefined we are not synchronously in a call
+        curContext: DynamicCallContext | undefined;
+        caller: CallerContext | undefined;
+    };
+}
+export type CallerContext = {
+    // IMPORTANT! Do not pass nodeId to other nodes with the intention of having
+    //  them call functions directly using nodeId. Instead pass location, and have them use connect.
+    //  - nodeId SHOULD be used to identify users though, as it cannot be impersonated
+    nodeId: string;
+    location: NetworkLocation;
+};
+
+// IMPORTANT! Nodes at the same network location may vary, so you cannot store NetworkLocation
+//  in a list of allowed users, otherwise they can be impersonated!
+export interface NetworkLocation {
+    address: string;
+    /** localPort is the port the connection was made from, which is often uninteresting.
+     *      It is useful internally, to help reduce collisions when a process restarts and
+     *      reconnects (as it may have the same source address, but the localPort will likely change).
+     */
+    localPort: number;
+    listeningPorts: number[];
+}

package/args.ts

@@ -0,0 +1,22 @@
+import { lazy } from "./caching";
+
+export const getArgs = lazy(() => {
+    let args = process.argv.slice(2);
+    let argObj: { [key: string]: string | undefined } = {};
+    for (let arg of args) {
+        if (arg.startsWith("-")) {
+            arg = arg.slice(1);
+        }
+        if (arg.startsWith("-")) {
+            arg = arg.slice(1);
+        }
+        if (arg.includes("=")) {
+            let key = arg.split("=")[0];
+            let value = arg.split("=").slice(1).join("=");
+            argObj[key] = value;
+        } else {
+            argObj[arg] = "true";
+        }
+    }
+    return argObj;
+});