socket-function 0.5.0

package/caching.ts

@@ -0,0 +1,301 @@
+import { arrayEqual } from "./misc";
+import { AnyFunction, Args, canHaveChildren } from "./types";
+
+export function lazy<T>(factory: () => T): () => T {
+    let value: { value: T }|undefined = undefined;
+
+    return () => {
+        if(!value) {
+            value = { value: factory() };
+        }
+        return value.value;
+    };
+}
+
+// NOTE: The reason we need to periodically clear, is because sometimes a very small
+//      part of a large payload (ex, persisted overrides) is cached, which then results
+//      in the whole payload being cached, which results in a lot of memory being used.
+
+// IMPORTANT! The cleanup functions CANNOT close upon anything, or else they will cause leaks!
+//  All data they use should be in data.
+interface CleanupFnc<T extends object> {
+    (data: T): void;
+}
+
+
+// NOTE: Empty arrays are so common, that it is useful to represent them as the same
+//  emtpy array, to increase cache hit rates.
+const emptyArray: any[] = [];
+export function cacheEmptyArray<T>(array: T[]): T[] {
+    if (array.length === 0) return emptyArray;
+    return array;
+}
+
+export function cache<Output, Key>(getValue: (key: Key) => Output): {
+    (key: Key): Output;
+} {
+    let startingCalculating = new Set<Key>();
+    let values = new Map<Key, Output>();
+    function cache(input: Key) {
+        let key = input;
+        if (values.has(key)) {
+            return values.get(key) as any;
+        }
+        if (startingCalculating.has(key)) {
+            // TODO: Fix the types here, by throwing, and then for the cases
+            //  that don't throw, make our output type include undefined
+            return undefined;
+        }
+        startingCalculating.add(key);
+        let value = getValue(input);
+        values.set(key, value);
+        return value;
+    }
+    return cache;
+}
+
+
+/** Makes a cache that limits the number of entries, allowing you to put arbitrary data in it
+ *      without worrying about leaking memory
+  */
+export function cacheLimited<Output, Key>(
+    // NOTE: We can't calculate what limit should be based on comparing the evaluation time
+    //  and the time to compare against the values. Because, even if finding a match takes far longer than
+    //  calculating, keeping a consistent output can save (a considerable amount of) time in downstream caches.
+    maxCount: number,
+    getValue: (key: Key) => Output
+): (key: Key) => Output {
+    let startingCalculating = new Set<Key>();
+    let values = new Map<Key, Output>();
+    return (input) => {
+        let key = input;
+        if (values.has(key)) {
+            return values.get(key) as any;
+        }
+        if (startingCalculating.has(key)) {
+            throw new Error(`Cyclic access in cache`);
+        }
+        startingCalculating.add(key);
+
+        // Clear when it gets too big. This is kind of like a worse
+        //  least recently used cache, because entries that are accessed
+        //  often will quickly get put back in. This is effective as long
+        //  as accesses take similar amounts of time. If there is a very slow
+        //  and very commonly accessed value, it could be evicted by many very
+        //  fast accesses, which would be unfortunate.
+        if (values.size >= maxCount) {
+            values.clear();
+            startingCalculating.clear();
+        }
+
+        let value = getValue(input);
+        values.set(key, value);
+        return value;
+    };
+}
+
+export function cacheWeak<Output, Key extends object>(getValue: (key: Key) => Output): (key: Key) => Output {
+    let state = {
+        startingCalculating: new WeakSet<Key>(),
+        values: new WeakMap<Key, Output>(),
+    };
+
+    return (input) => {
+        let key = input;
+        if (state.values.has(key)) {
+            return state.values.get(key) as any;
+        }
+        if (state.startingCalculating.has(key)) {
+            throw new Error(`Cyclic access in cacheWeak`);
+        }
+        state.startingCalculating.add(key);
+        let value = getValue(input);
+        state.values.set(key, value);
+        return value;
+    };
+}
+
+// A list cache, which... maybe faster than a Map?
+export function cacheList<Value>(
+    getLength: () => number,
+    getValue: (index: number) => Value,
+): { (index: number): Value; } {
+    let state = {
+        cache: [] as Value[],
+        length: undefined as undefined | number,
+        getLength,
+    };
+    function get(i: number) {
+        let cache = state.cache;
+        let length = state.length;
+        if (length === undefined) {
+            length = state.length = state.getLength();
+        }
+        if (i < 0 || i >= length) {
+            throw new Error(`Index out of bounds`);
+        }
+        if (!(i in cache)) {
+            cache[i] = getValue(i);
+        }
+        return cache[i];
+    };
+    return get;
+}
+
+function cacheArrayEqualCleanup(state: any) {
+    state.cache = [];
+}
+
+/** A cache half way between caching based on === and caching based on hash. Caches
+ *      based on arrayEqual, which does === on all values in an array. Requires localized
+ *      caching (as the comparisons don't scale with many candidates, unlike hashing),
+ *      however works with non trival transformations (ex, resolving many persisted overrides
+ *      to get a value), unlike cache().
+ *  Also, limits itself, more of a performance optimization than memory optimization, as it scales
+ *      very poorly with the number of candidates.
+ *  
+ *  TIMING: About 6us with limit = 100, array size = 294, and the cache being full.
+ */
+export function cacheArrayEqual<Input extends unknown[] | undefined, Output>(
+    map: (arrays: Input) => Output,
+    limit = 10
+): {
+    (array: Input): Output;
+    clear(array: Input): void;
+} {
+    let state: {
+        cache: {
+            input: Input;
+            output: Output;
+        }[]
+    } = { cache: [] };
+    function isMatch(lhs: Input, rhs: Input) {
+        if (lhs === rhs) {
+            return true;
+        }
+        if (lhs === undefined || rhs === undefined) {
+            return false;
+        }
+        if (arrayEqual(lhs, rhs)) {
+            return true;
+        }
+        return false;
+    }
+    return Object.assign(
+        (input: Input) => {
+            let cache = state.cache;
+            for (let obj of cache) {
+                if (isMatch(obj.input, input)) {
+                    return obj.output;
+                }
+            }
+            let output = map(input);
+            cache.unshift({ input, output });
+            while (cache.length > limit) {
+                cache.pop();
+            }
+            return output;
+        },
+        {
+            clear(array: Input) {
+                for (let i = state.cache.length - 1; i >= 0; i--) {
+                    if (isMatch(state.cache[i].input, array)) {
+                        state.cache.splice(i, 1);
+                    }
+                }
+            }
+        }
+    );
+}
+
+/** Caches when arguments are ===. See cacheArrayEqual */
+export function cacheArgsEqual<Fnc extends AnyFunction>(
+    fnc: Fnc,
+    limit = 10
+): Fnc & { clear(...args: Args<Fnc>): void } {
+    let cache = cacheArrayEqual((args: unknown[]) => {
+        return fnc(...args);
+    }, limit);
+    return Object.assign(
+        ((...args: unknown[]) => {
+            return cache(args);
+        }) as Fnc,
+        {
+            clear(...args: unknown[]) {
+                cache.clear(args);
+            }
+        }
+    );
+}
+
+export function cacheJSONArgsEqual<Fnc extends AnyFunction>(
+    fnc: Fnc,
+    limit = 10
+): Fnc {
+    let cache = cacheLimited(limit, (argsJSON: string) => {
+        return fnc(...JSON.parse(argsJSON));
+    });
+    return ((...args: unknown[]) => {
+        return cache(JSON.stringify(args));
+    }) as Fnc;
+}
+
+export function cacheShallowConfigArgEqual<Fnc extends AnyFunction>(
+    fnc: Fnc,
+    limit = 10
+): Fnc & {
+    clear(...args: Args<Fnc>): void;
+} {
+    let cache = cacheArrayEqual((kvpsFlat: unknown[]) => {
+        output.missCount++;
+        let arg: any;
+        if (kvpsFlat.length === 1) {
+            arg = kvpsFlat[0];
+        } else {
+            let kvps: [unknown, unknown][] = [];
+            for (let i = 0; i < kvpsFlat.length; i += 2) {
+                kvps.push([kvpsFlat[i], kvpsFlat[i + 1]]);
+            }
+            arg = Object.fromEntries(kvps);
+        }
+        return fnc(arg);
+    }, limit);
+    function getKVPs(configArg: object) {
+        if (!canHaveChildren(configArg) || Array.isArray(configArg)) {
+            return [configArg];
+        }
+        let keys = Object.keys(configArg);
+        keys.sort();
+        return keys.flatMap(key => [key, (configArg as any)[key]]);
+    }
+    let output = Object.assign(
+        ((configArg: object) => {
+            output.callCount++;
+            return cache(getKVPs(configArg));
+        }) as Fnc,
+        {
+            clear(configArg: object) {
+                cache.clear(getKVPs(configArg));
+            },
+            callCount: 0,
+            missCount: 0,
+        }
+    );
+    return output;
+}
+
+
+export function externalCache<Key, Value>(): {
+    get: (key: Key) => Value | undefined;
+    set: (key: Key, value: Value) => void;
+} {
+    let values = new Map<Key, Value>();
+    return {
+        get: (key) => {
+            return values.get(key);
+        },
+        set: (key, value) => {
+            values.set(key, value);
+        },
+    };
+}

package/callManager.ts

@@ -0,0 +1,108 @@
+import { CallContextType, CallerContext, CallType, ClientHookContext, HookContext, NetworkLocation, SocketExposedInterface, SocketExposedInterfaceClass, SocketExposedShape, SocketFunctionClientHook, SocketFunctionHook, SocketRegistered } from "./SocketFunctionTypes";
+import { _setSocketContext } from "./SocketFunction";
+
+let classes: {
+    [classGuid: string]: {
+        classType: SocketExposedInterfaceClass;
+        controller: SocketExposedInterface;
+        shape: SocketExposedShape;
+    }
+} = {};
+let exposedClasses = new Set<SocketExposedInterfaceClass>();
+
+let globalHooks: SocketFunctionHook[] = [];
+let globalClientHooks: SocketFunctionClientHook[] = [];
+
+export async function performLocalCall(
+    config: {
+        call: CallType;
+        caller: CallerContext;
+    }
+): Promise<unknown> {
+    const { call, caller } = config;
+    let classDef = classes[call.classGuid];
+
+    if (!classDef) {
+        throw new Error(`Class ${call.classGuid} not found`);
+    }
+
+    if (!exposedClasses.has(classDef.classType)) {
+        throw new Error(`Class ${call.classGuid} not exposed`);
+    }
+
+    let controller = classDef.controller;
+    let shape = classDef.shape;
+    let functionShape = shape[call.functionName];
+    if (!functionShape) {
+        throw new Error(`Function ${call.functionName} not exposed`);
+    }
+
+    if (!controller[call.functionName]) {
+        throw new Error(`Function ${call.functionName} does not exist`);
+    }
+
+    let curContext: CallContextType = {};
+    let serverContext = await runServerHooks(call, { caller, curContext }, shape);
+    if ("overrideResult" in serverContext) {
+        return serverContext.overrideResult;
+    }
+
+    // NOTE: We purposely don't await inside _setSocketContext, so the context is reset synchronously
+    let result = _setSocketContext(curContext, caller, () => {
+        return controller[call.functionName](...call.args);
+    });
+
+    return await result;
+}
+
+export function registerClass(classGuid: string, exposedClass: SocketExposedInterfaceClass, shape: SocketExposedShape) {
+    if (classes[classGuid]) {
+        throw new Error(`Class ${classGuid} already registered`);
+    }
+
+    classes[classGuid] = {
+        classType: exposedClass,
+        controller: new exposedClass() as SocketExposedInterface,
+        shape,
+    };
+}
+
+export function exposeClass(exposedClass: SocketExposedInterfaceClass) {
+    exposedClasses.add(exposedClass);
+}
+
+export function registerGlobalHook(hook: SocketFunctionHook) {
+    globalHooks.push(hook);
+}
+export function registerGlobalClientHook(hook: SocketFunctionClientHook) {
+    globalClientHooks.push(hook);
+}
+
+export async function runClientHooks(
+    callType: CallType,
+    hooks: SocketExposedShape[""],
+): Promise<ClientHookContext> {
+    let context: ClientHookContext = { call: callType };
+    for (let hook of globalClientHooks.concat(hooks.clientHooks || [])) {
+        await hook(context);
+        if ("overrideResult" in context) {
+            break;
+        }
+    }
+    return context;
+}
+
+async function runServerHooks(
+    callType: CallType,
+    context: SocketRegistered["context"],
+    hooks: SocketExposedShape[""],
+): Promise<HookContext> {
+    let hookContext: HookContext = { call: callType, context };
+    for (let hook of globalHooks.concat(hooks.hooks || [])) {
+        await hook(hookContext);
+        if ("overrideResult" in hookContext) {
+            break;
+        }
+    }
+    return hookContext;
+}

package/misc.ts

@@ -0,0 +1,27 @@
+import * as crypto from "crypto";
+
+export function convertErrorStackToError(error: string): Error {
+    let errorObj = new Error();
+    errorObj.stack = String(error);
+    errorObj.message = String(error).split("\n")[0].slice("Error: ".length);
+    return errorObj;
+}
+
+export function sha256Hash(buffer: Buffer) {
+    return crypto.createHash("sha256").update(buffer).digest("hex");
+}
+
+
+export function arrayEqual(a: unknown[], b: unknown[]) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) {
+        if (a[i] !== b[i]) return false;
+    }
+    return true;
+}
+
+
+// TODO: Find a better place for this...
+process.on("unhandledRejection", async (reason: any, promise) => {
+    console.error(`Uncaught promise rejection: ${String(reason.stack || reason)}`);
+});

package/nodeAuthentication.ts

@@ -0,0 +1,110 @@
+import ws from "ws";
+import tls from "tls";
+import { getAppFolder } from "./storagePath";
+import fs from "fs";
+import child_process from "child_process";
+import { cacheWeak, lazy } from "./caching";
+import https from "https";
+import debugbreak from "debugbreak";
+import crypto from "crypto";
+import { sha256Hash } from "./misc";
+import { getArgs } from "./args";
+
+export const getCertKeyPair = lazy((): { key: Buffer; cert: Buffer } => {
+    // TODO: Also get this working clientside...
+    //  - Probably using node-forge, maybe using this as an example: https://github.com/jfromaniello/selfsigned/blob/master/index.js
+
+    // https://nodejs.org/en/knowledge/HTTP/servers/how-to-create-a-HTTPS-server/
+
+    let folder = getAppFolder();
+    let identityPrefix = getArgs().identity || "";
+    let keyPath = folder + identityPrefix + "key.pem";
+    let certPath = folder + identityPrefix + "cert.pem";
+    if (!fs.existsSync(keyPath) || !fs.existsSync(certPath)) {
+        child_process.execSync(`openssl genrsa -out "${keyPath}"`);
+        child_process.execSync(`openssl req -new -key "${keyPath}" -out csr.pem -subj "/CN=notused"`);
+        child_process.execSync(`openssl x509 -req -days 9999 -in csr.pem -signkey "${keyPath}" -out "${certPath}"`);
+        fs.rmSync("csr.pem");
+    }
+
+    let key = fs.readFileSync(keyPath);
+    let cert = fs.readFileSync(certPath);
+    return { key, cert };
+});
+
+export function getTLSSocket(webSocket: ws.WebSocket) {
+    return (webSocket as any)._socket as tls.TLSSocket;
+}
+
+export const getNodeId = cacheWeak(function (webSocket: ws.WebSocket): string {
+    let socket = getTLSSocket(webSocket);
+    let peerCert = socket.getPeerCertificate();
+    if (!peerCert) {
+        throw new Error("WebSocket connections must provided a peer certificate");
+    }
+    let pubkey = (peerCert as any).pubkey as Buffer | undefined;
+    if (!pubkey) {
+        throw new Error(`Peer certificate must use an RSA key or EC key (which should have a .pubkey property)`);
+    }
+    return sha256Hash(pubkey);
+});
+
+export function createWebsocket(address: string, port: number): ws.WebSocket {
+    let { key, cert } = getCertKeyPair();
+    console.log(`Connecting to ${address}:${port}`);
+    return new ws.WebSocket(`wss://${address}:${port}`, {
+        cert,
+        key,
+        rejectUnauthorized: false,
+    });
+}
+
+
+
+/*
+const port = 2422;
+let { key, cert } = getCertKeyPair();
+console.log(process.argv);
+if (process.argv.includes("--server")) {
+    
+    let server = https.createServer({
+        key,
+        cert,
+        rejectUnauthorized: false,
+        requestCert: true
+    });
+    let listenPromise = new Promise<void>((resolve, error) => {
+        server.on("listening", () => {
+            resolve();
+        });
+        server.on("error", e => {
+            error(e);
+        });
+    });
+    
+    server.on("request", (request, response) => {
+        // TODO: Handle HTTP requests
+        //  - HTTP CAN have a nodeId, simply through setting cookies
+        //      - Cookies could always be set via a request before we open
+        //          the websocket connection?
+    });
+
+    const webSocketServer = new ws.Server({
+        noServer: true,
+    });
+    server.on("upgrade", (request, socket, upgradeHead) => {
+        webSocketServer.handleUpgrade(request, socket, upgradeHead, (ws) => {
+            console.log("peer", getTLSSocket(ws).getPeerCertificate()?.pubkey.toString("hex").slice(100));
+            console.log("cert", getTLSSocket(ws).getCertificate()?.pubkey.toString("hex").slice(100));
+        });
+    });
+
+    server.listen(2422, "127.0.0.1");
+} else {
+    let socket = new ws.WebSocket(`wss://127.0.0.1:${port}`, { rejectUnauthorized: false, cert, key });
+    socket.on("open", () => {
+        console.log("peer", getTLSSocket(socket).getPeerCertificate()?.pubkey.toString("hex").slice(100));
+        console.log("cert", getTLSSocket(socket).getCertificate()?.pubkey.toString("hex").slice(100));
+    });
+}
+*/

package/nodeCache.ts

@@ -0,0 +1,79 @@
+import { callFactoryFromLocation, CallFactory } from "./CallInstance";
+import { NetworkLocation } from "./SocketFunctionTypes";
+import { MaybePromise } from "./types";
+
+// TODO: Add CallInstanceFactory.isClosed, so nodeCache can clean up old entries.
+//  This is only needed for memory management, and not for correctness. Entries never
+//  need to be refreshed, because NetworkLocation.listeningPorts shouldn't really change.
+//  Either we will have listeningPorts and re-establish the connection, or we won't, and
+//  then it is a client, in which case we cannot re-establish the connection (and we just
+//  have to wait for the client to re-establish it). AND, if the listeningPorts change from
+//  a value to a new value... then they should be obtained using connect() anyway,
+//  and so whatever way the user got the NetworkLocation to begin with, they should use again.
+
+
+// nodeId => 
+const nodeCache = new Map<string, {
+    callFactory: CallFactory;
+}>();
+const locationLookup = new Map<string, MaybePromise<string>>();
+
+function getNetworkLocationHash(location: NetworkLocation): string {
+    return location.address + ":" + location.localPort + "=" + location.listeningPorts.join("|");
+}
+
+// NOTE: For client connections, at which point we have the nodeId, location and callFactory.
+export function registerNodeClient(callFactory: CallFactory) {
+    let { nodeId } = callFactory;
+    // NOTE: We can always clobber the entry, AS, during client connection we give NetworkLocation information,
+    //  so even if we already have this node with NetworkLocation.listeningPorts, this new values should
+    //  be even newer, or the same.
+    //  - AND, clobbering shouldn't happen often, if the other end connected to us they should have given us their
+    //      nodeId. So they'll use the existing websocket when using that nodeId, instead of establishing a new connection,
+    //      except for race conditions cases, in which case we just have an extra connection, which isn't so bad...
+    //      - And of course, we have to use the newer connection, as it might be the case that the NetworkLocation has actually
+    //          updated, and the old connection is now forever closed.
+
+    // Never go from listening ports to no listening ports. Worst case the listening ports are old
+    //  and won't work, in which case... we won't be able to reconnect, which basically what
+    //  we would do if there were no listening ports.
+    let prevListeningPorts = nodeCache.get(nodeId)?.callFactory.location.listeningPorts;
+    if (prevListeningPorts && !callFactory.location.listeningPorts.length) {
+        callFactory.location.listeningPorts = prevListeningPorts;
+    }
+    // TODO: Maybe even preserve the address in some cases, such as if it was a domain, and is now an ip?
+    nodeCache.set(nodeId, {
+        callFactory,
+    });
+}
+
+export function getCreateCallFactoryLocation(location: NetworkLocation): MaybePromise<string> {
+    let locationHash = getNetworkLocationHash(location);
+    let nodeId = locationLookup.get(locationHash);
+    if (nodeId !== undefined) {
+        return nodeId;
+    }
+
+    let callFactoryPromise = callFactoryFromLocation(location);
+    let nodeIdPromise = callFactoryPromise.then(x => x.nodeId);
+    locationLookup.set(locationHash, nodeIdPromise);
+
+    return callFactoryPromise.then(callFactory => {
+        let nodeId = callFactory.nodeId;
+        // TODO: Maybe warn if we just clobbered a nodeId?
+        let prevEntry = nodeCache.get(nodeId);
+        if (prevEntry) {
+            console.warn(`Clobbering nodeId ${nodeId}, with a new location ${locationHash}, was ${getNetworkLocationHash(prevEntry.callFactory.location)}. (This might indiciate multiple locations with the same nodeId, which could cause an issue. If this happens repeatedly it will cause stability issues).`);
+        }
+        nodeCache.set(nodeId, {
+            callFactory,
+        });
+        return nodeId;
+    });
+}
+
+
+// TODO: Give a special error if the nodeId has been seen, but is only one-way (from HTTP requests).
+export function getCallFactoryNodeId(nodeId: string): CallFactory|undefined {
+    return nodeCache.get(nodeId)?.callFactory;
+}

package/nodeProxy.ts

@@ -0,0 +1,36 @@
+import { lazy } from "./caching";
+import { SocketExposedInterface } from "./SocketFunctionTypes";
+
+//todonext
+//  We need some sort of cache, but... maybe not here?
+//  Proxy => callback?
+
+type CallProxyType = {
+    [controllerName: string]: SocketExposedInterface;
+};
+
+let proxyCache = new Map<string, CallProxyType>();
+export function getCallProxy(id: string, callback: (controllerName: string, functionName: string, args: unknown[]) => Promise<unknown>): CallProxyType {
+    let value = proxyCache.get(id);
+    if(!value) {
+        let controllerCache = new Map<string, CallProxyType[""]>();
+        value = new Proxy(Object.create(null), {
+            get(target, controllerName) {
+                if(typeof controllerName !== "string") return undefined;
+                let controller = controllerCache.get(controllerName);
+                if(!controller) {
+                    controller = new Proxy(Object.create(null), {
+                        get(target, functionName) {
+                            if (typeof functionName !== "string") return undefined;
+                            return (...args: unknown[]) => callback(controllerName, functionName, args);
+                        }
+                    }) as CallProxyType[""];
+                    controllerCache.set(controllerName, controller);
+                }
+                return controller;
+            },
+        }) as CallProxyType;
+        proxyCache.set(id, value);
+    }
+    return value;
+}

package/package.json

@@ -0,0 +1,17 @@
+{
+	"name": "socket-function",
+	"version": "0.5.0",
+	"main": "index.js",
+	"license": "MIT",
+	"dependencies": {
+		"@types/node": "^18.0.0",
+		"@types/ws": "^8.5.3",
+		"debugbreak": "^0.6.3",
+		"typenode": "0.3.0",
+		"ws": "^8.8.0"
+	},
+	"scripts": {
+		"test": "yarn typenode ./nodeAuthentication.ts --server",
+		"type": "yarn tsc --noEmit"
+	}
+}