socket-function 0.9.5 → 0.9.7

package/SetProcessVariables.ts

@@ -0,0 +1,6 @@
+module.allowclient = true;
+import { isNode } from "./src/misc";
+
+if (!isNode()) {
+    process.env.CBOR_NATIVE_ACCELERATION_DISABLED = "true";
+}

package/SocketFunction.ts

@@ -10,6 +10,13 @@ import { setDefaultHTTPCall } from "./src/callHTTPHandler";
 import debugbreak from "debugbreak";
 import { lazy } from "./src/caching";
 import { delay } from "./src/batching";
+import { blue, magenta } from "./src/formatting/logColors";
+import { JSONLACKS } from "./src/JSONLACKS/JSONLACKS";
+import "./SetProcessVariables";
+import cborx from "cbor-x";
+import { setFlag } from "./require/compileFlags";
+setFlag(require, "cbor-x", "allowclient", true);
+let cborxInstance = new cborx.Encoder({ structuredClone: true });
 
 module.allowclient = true;
 
@@ -22,22 +29,23 @@ type ExtractShape<ClassType, Shape> = {
         : "Function is in shape, but not in class"
     );
 };
-// https://stackoverflow.com/a/69756175/1117119
-type PickByType<T, Value> = {
-    [P in keyof T as T[P] extends Value | undefined ? P : never]: T[P]
-};
 
 export class SocketFunction {
     public static logMessages = false;
 
     public static MAX_MESSAGE_SIZE = 1024 * 1024 * 32;
-    public static compression: undefined | {
-        type: "gzip";
-    };
 
     public static httpETagCache = false;
     public static silent = true;
 
+    // In retrospect... dynamically changing the wire serializer is a BAD idea. If any calls happen
+    //  before it is changed, things just break. Also, it needs to be changed on both sides,
+    //  or else things break. Also, it is very hard to detect when the issue is different serializers
+    public static readonly WIRE_SERIALIZER = {
+        serialize: (obj: unknown): MaybePromise<Buffer[]> => [cborxInstance.encode(obj)],
+        deserialize: (buffers: Buffer[]): MaybePromise<unknown> => cborxInstance.decode(buffers[0]),
+    };
+
     public static WIRE_WARN_TIME = 100;
 
     private static onMountCallbacks = new Map<string, (() => MaybePromise<void>)[]>();
@@ -54,18 +62,24 @@ export class SocketFunction {
     //  (ex, using a hook in a controller where the hook also calls the controller).
     public static register<
         ClassInstance extends object,
-        Shape extends SocketExposedShape<SocketExposedInterface>,
+        Shape extends SocketExposedShape<{
+            [key in keyof ClassInstance]: (...args: any[]) => Promise<unknown>;
+        }>,
     >(
         classGuid: string,
         instance: ClassInstance,
         shapeFnc: () => Shape,
         defaultHooksFnc?: () => SocketExposedShape[""] & {
             onMount?: () => MaybePromise<void>;
+        },
+        config?: {
+            /** @noAutoExpose If true SocketFunction.expose(Controller) must be called explicitly. */
+            noAutoExpose?: boolean;
         }
     ): SocketRegistered<ExtractShape<ClassInstance, Shape>> {
         let getDefaultHooks = defaultHooksFnc && lazy(defaultHooksFnc);
         const getShape = lazy(() => {
-            let shape = shapeFnc();
+            let shape = shapeFnc() as SocketExposedShape;
             let defaultHooks = getDefaultHooks?.();
 
             for (let value of Object.values(shape)) {
@@ -96,7 +110,7 @@ export class SocketFunction {
                     throw new Error(`Function ${functionName} is not in shape`);
                 }
 
-                let hookResult = await runClientHooks(call, shapeObj as SocketExposedShape[""], callFactory.connectionId);
+                let hookResult = await runClientHooks(call, shapeObj as Exclude<SocketExposedShape[""], undefined>, callFactory.connectionId);
 
                 if ("overrideResult" in hookResult) {
                     return hookResult.overrideResult;
@@ -128,7 +142,11 @@ export class SocketFunction {
             }
         });
 
-        return output as any;
+        let result = output as any as SocketRegistered;
+        if (!config?.noAutoExpose) {
+            this.expose(result);
+        }
+        return result;
     }
 
     public static onNextDisconnect(nodeId: string, callback: () => void) {
@@ -185,8 +203,9 @@ export class SocketFunction {
      *      to add additional imports to ensure the register call runs.
      */
     public static expose(socketRegistered: SocketRegistered) {
+        console.log(blue(`Exposing Controller ${socketRegistered._classGuid}`));
         exposeClass(socketRegistered);
-        SocketFunction.exposedClasses.add(socketRegistered._classGuid);
+        this.exposedClasses.add(socketRegistered._classGuid);
 
         if (this.hasMounted) {
             let mountCallbacks = SocketFunction.onMountCallbacks.get(socketRegistered._classGuid);
@@ -199,6 +218,7 @@ export class SocketFunction {
     }
 
     public static mountedNodeId: string = "";
+    public static isMounted() { return !!this.mountedNodeId; }
     public static mountedIP: string = "";
     private static hasMounted = false;
     private static onMountCallback: () => void = () => { };

package/SocketFunctionTypes.ts

@@ -22,8 +22,8 @@ export type SocketExposedInterfaceClass = {
     new(): unknown;
     prototype: unknown;
 };
-export interface SocketExposedShape<ExposedType extends SocketExposedInterface = SocketExposedInterface> {
-    [functionName: string]: {
+export type SocketExposedShape<ExposedType extends SocketExposedInterface = SocketExposedInterface> = {
+    [functionName in keyof ExposedType]?: {
         /** Indicates with the same input, we give the same output, forever,
          *      independent of code changes. This only works for data storage.
          */
@@ -31,7 +31,7 @@ export interface SocketExposedShape<ExposedType extends SocketExposedInterface =
         hooks?: SocketFunctionHook<ExposedType>[];
         clientHooks?: SocketFunctionClientHook<ExposedType>[];
     };
-}
+};
 
 export interface CallType {
     classGuid: string;

package/package.json

@@ -1,15 +1,16 @@
 {
 	"name": "socket-function",
-	"version": "0.9.5",
+	"version": "0.9.7",
 	"main": "index.js",
 	"license": "MIT",
 	"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
 	"dependencies": {
+		"cbor-x": "^1.5.6",
 		"cookie": "^0.5.0",
 		"mobx": "^6.6.2",
 		"node-forge": "https://github.com/sliftist/forge#name",
 		"preact": "^10.10.6",
-		"typenode": "^4.9.4-g",
+		"typenode": "5.3.3",
 		"ws": "^8.8.0"
 	},
 	"optionalDependencies": {

package/require/RequireController.ts

@@ -223,7 +223,7 @@ class RequireControllerBase {
             //require(rootPath);
             let clientModule = require.cache[resolvedPath];
             if (!clientModule) {
-                clientModule = createNotFoundModule(`Module ${pathRequest} (resolved to ${JSON.stringify(resolvedPath)}) was not included serverside. Resolve root ${JSON.stringify(this.rootResolvePath)} (set by call to setRequireBootRequire), resolve search paths: ${JSON.stringify(searchPaths)})}`);
+                clientModule = createNotFoundModule(`Module ${pathRequest} (resolved to ${JSON.stringify(resolvedPath)}) was not included serverside. Resolved from root dir ${JSON.stringify(this.rootResolvePath)} (set by call to setRequireBootRequire), resolve search paths: ${JSON.stringify(searchPaths)})}`);
             }
             if (!clientModule.allowclient) {
                 clientModule = createNotFoundModule(`Module ${pathRequest} (resolved to ${resolvedPath}) is not allowed clientside (set module.allowclient in it, or call setFlag when it is imported).`);
@@ -249,5 +249,9 @@ export const RequireController = SocketFunction.register(
         requireHTML: {},
         bufferJS: {},
         requireJS: {},
-    })
+    }),
+    undefined,
+    {
+        noAutoExpose: true
+    }
 );

package/require/require.js

@@ -9,6 +9,9 @@
                 // Mirror the tnode.js setting
                 NODE_ENV: "production"
             },
+            versions: {
+
+            },
         },
         setImmediate(callback) {
             setTimeout(callback, 0);
@@ -35,6 +38,7 @@
         stream: {
             // HACK: Needed to get SAX JS to work correctly.
             Stream: function () { },
+            Transform: function () { },
         },
         timers: {
             // TODO: Add all members of timers

package/src/CallFactory.ts

@@ -9,17 +9,16 @@ import * as tls from "tls";
 import { getClientNodeId, getNodeIdLocation, registerNodeClient } from "./nodeCache";
 import debugbreak from "debugbreak";
 import { lazy } from "./caching";
-import { JSONLACKS } from "./JSONLACKS/JSONLACKS";
 import { red, yellow } from "./formatting/logColors";
 import { isSplitableArray, markArrayAsSplitable } from "./fixLargeNetworkCalls";
-import { delay } from "./batching";
+import { delay, runInSerial } from "./batching";
+import { formatNumber, formatTime } from "./formatting/format";
 
 const MIN_RETRY_DELAY = 1000;
 
 type InternalCallType = FullCallType & {
     seqNum: number;
     isReturn: false;
-    compress: boolean;
 }
 
 type InternalReturnType = {
@@ -76,7 +75,7 @@ export async function createCallFactory(
     const canReconnect = !!getNodeIdLocation(nodeId);
 
     let pendingCalls: Map<number, {
-        data: Buffer;
+        data: Buffer[];
         call: InternalCallType;
         callback: (resultJSON: InternalReturnType) => void;
     }> = new Map();
@@ -123,16 +122,22 @@ export async function createCallFactory(
                 classGuid: call.classGuid,
                 functionName: call.functionName,
                 seqNum,
-                compress: !!SocketFunction.compression,
             };
             let time = Date.now();
-            let data = Buffer.from(JSONLACKS.stringify(fullCall));
+            let data: Buffer[];
+            let dataMaybePromise = SocketFunction.WIRE_SERIALIZER.serialize(fullCall);
+            if (dataMaybePromise instanceof Promise) {
+                data = await dataMaybePromise;
+            } else {
+                data = dataMaybePromise;
+            }
             time = Date.now() - time;
+            let size = data.map(x => x.length).reduce((a, b) => a + b, 0);
             if (time > SocketFunction.WIRE_WARN_TIME) {
-                console.log(red(`Slow serialize, took ${time}ms to serialize ${data.byteLength} bytes. For ${call.classGuid}.${call.functionName}`));
+                console.log(red(`Slow serialize, took ${formatTime(time)} to serialize ${formatNumber(size)} bytes. For ${call.classGuid}.${call.functionName}`));
             }
 
-            if (data.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
+            if (size > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
                 let splitArgIndex = call.args.findIndex(isSplitableArray);
                 if (splitArgIndex >= 0) {
                     console.log(yellow(`Splitting large call due to large args: ${call.classGuid}.${call.functionName}`));
@@ -157,7 +162,7 @@ export async function createCallFactory(
                     return "CALLS_SPLIT_DUE_TO_LARGE_ARGS";
                 }
 
-                throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${data.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
+                throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${formatNumber(size)} > ${formatNumber(SocketFunction.MAX_MESSAGE_SIZE)}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
             }
 
             let resultPromise = new Promise((resolve, reject) => {
@@ -251,7 +256,8 @@ export async function createCallFactory(
         }
     }
 
-    async function send(data: Buffer) {
+    const BASE_LENGTH_OFFSET = 324_432_461_592_612;
+    async function send(data: Buffer[]) {
         if (!webSocketPromise) {
             if (canReconnect) {
                 webSocketPromise = tryToReconnect();
@@ -260,7 +266,10 @@ export async function createCallFactory(
             }
         }
         let webSocket = await webSocketPromise;
-        webSocket.send(data);
+        webSocket.send((data.length + BASE_LENGTH_OFFSET).toString());
+        for (let d of data) {
+            webSocket.send(d);
+        }
     }
     async function tryToReconnect(): Promise<SenderInterface> {
         // Don't try to reconnect too often!
@@ -276,7 +285,8 @@ export async function createCallFactory(
         return newWebSocket;
     }
 
-
+    let pendingCall: { buffers: Buffer[]; targetCount: number; } | undefined;
+    let clientsideSerial = runInSerial(async <T>(val: Promise<T>) => val);
     async function onMessage(message: ws.RawData | ws.MessageEvent | string) {
         try {
             if (typeof message === "object" && "data" in message) {
@@ -284,35 +294,54 @@ export async function createCallFactory(
             }
             if (!isNode()) {
                 if (message instanceof Blob) {
-                    message = Buffer.from(await message.arrayBuffer());
+                    // We need to force the results to be in serial, otherwise strings leapfrog
+                    //  ahead of buffers, which breaks things.
+                    message = Buffer.from(await clientsideSerial(message.arrayBuffer()));
+                } else {
+                    await clientsideSerial(Promise.resolve());
+                }
+            }
+            if (typeof message === "string") {
+                let size = parseInt(message);
+                if (isNaN(size)) {
+                    throw new Error(`Invalid message size ${message}`);
+                }
+                if (size < BASE_LENGTH_OFFSET) {
+                    throw new Error(`Invalid message size ${message}`);
                 }
+                size -= BASE_LENGTH_OFFSET;
+                if (size > 1000 * 1000) {
+                    throw new Error(`Invalid message size ${size}`);
+                }
+                pendingCall = {
+                    buffers: [],
+                    targetCount: size,
+                };
+                return;
             }
-            if (message instanceof Buffer || typeof message === "string") {
-
-                let resultSize = message.length;
-
-                if (message instanceof Buffer && message[0] === 0) {
-                    // First byte of 0 means it is decompressed (as JSON can't have a first byte of 0).
-                    (message as any) = message.slice(1);
-
-                    // TODO: Add typings for DecompressionStream
-                    let DecompressionStream = (window as any).DecompressionStream;
-                    // https://stackoverflow.com/a/68829631/1117119
-                    let stream = new DecompressionStream("gzip");
-                    let blob = new Blob([message]);
-                    let decompressedStream = (await (blob.stream() as any).pipeThrough(stream));
-                    let arrayBuffer = await new Response(decompressedStream).arrayBuffer();
-                    (message as any) = Buffer.from(arrayBuffer);
+            if (message instanceof Buffer) {
+                if (!pendingCall) {
+                    throw new Error(`Received data without size`);
+                }
+                pendingCall.buffers.push(message);
+                let currentBuffers: Buffer[];
+                if (pendingCall.buffers.length !== pendingCall.targetCount) {
+                    return;
                 }
 
+                currentBuffers = pendingCall.buffers;
+                pendingCall = undefined;
+
+                let resultSize = currentBuffers.map(x => x.length).reduce((a, b) => a + b, 0);
+
                 let time = Date.now();
-                let call = JSONLACKS.parse(message.toString(), { extended: false }) as InternalCallType | InternalReturnType;
+                let call = await SocketFunction.WIRE_SERIALIZER.deserialize(currentBuffers) as InternalCallType | InternalReturnType;
                 time = Date.now() - time;
 
                 if (call.isReturn) {
                     let callbackObj = pendingCalls.get(call.seqNum);
                     if (time > SocketFunction.WIRE_WARN_TIME) {
-                        console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for receieving result of call to ${callbackObj?.call.classGuid}.${callbackObj?.call.functionName}`));
+                        console.log(red(`Slow parse, took ${time}ms to parse ${resultSize} bytes, for receieving result of call to ${callbackObj?.call.classGuid}.${callbackObj?.call.functionName}`));
                     }
                     if (!callbackObj) {
                         console.log(`Got return for unknown call ${call.seqNum}`);
@@ -322,7 +351,7 @@ export async function createCallFactory(
                     callbackObj.callback(call);
                 } else {
                     if (time > SocketFunction.WIRE_WARN_TIME) {
-                        console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for call to ${call.classGuid}.${call.functionName}`));
+                        console.log(red(`Slow parse, took ${time}ms to parse ${resultSize} bytes, for call to ${call.classGuid}.${call.functionName}`));
                     }
 
                     let response: InternalReturnType;
@@ -346,27 +375,18 @@ export async function createCallFactory(
                         };
                     }
 
-                    let result: Buffer;
-                    if (isNode() && call.compress && SocketFunction.compression?.type === "gzip") {
-                        response.compressed = true;
-                        result = Buffer.from(JSONLACKS.stringify(response));
-                        result = await new Promise<Buffer>((resolve, reject) =>
-                            gzip(result, (err, result) => err ? reject(err) : resolve(result))
-                        );
-                        result = Buffer.concat([new Uint8Array([0]), result]);
-                    } else {
-                        result = Buffer.from(JSONLACKS.stringify(response));
-                    }
-                    if (result.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
+                    let result: Buffer[] = await SocketFunction.WIRE_SERIALIZER.serialize(response);
+                    let totalResultSize = result.map(x => x.length).reduce((a, b) => a + b, 0);
+                    if (totalResultSize > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
                         response = {
                             isReturn: true,
                             result: undefined,
                             seqNum: call.seqNum,
-                            error: new Error(`Response too large to send (${call.classGuid}.${call.functionName}, size: ${result.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`).stack,
+                            error: new Error(`Response too large to send (${call.classGuid}.${call.functionName}, size: ${formatNumber(totalResultSize)} > ${formatNumber(SocketFunction.MAX_MESSAGE_SIZE)}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`).stack,
                             resultSize: resultSize,
                             compressed: false,
                         };
-                        result = Buffer.from(JSONLACKS.stringify(response));
+                        result = await SocketFunction.WIRE_SERIALIZER.serialize(response);
                     }
                     await send(result);
                 }
@@ -379,8 +399,6 @@ export async function createCallFactory(
             if (e.stack.startsWith("Error: Cannot send data to") && e.stack.includes("as the connection has closed")) {
                 // This is fine, just ignore it
             } else {
-                debugbreak(2);
-                debugger;
                 console.error(e.stack);
             }
         }

package/src/JSONLACKS/JSONLACKS.ts

@@ -13,23 +13,13 @@ import { delay } from "../batching";
 const SERIALIZE_OBJECT_BATCH_COUNT = 1000;
 const PARSE_BYTE_CHUNK_SIZE = 1024 * 1024 * 10;
 
-// Supports json and also:
-//  - Non quoted field names "{ x: 1 }"?
-//  - Trailing commas
-//      NOTE: Comma only syntax is not supported, ex, "[,,]", which is an array of length 2 in javascript
-//  - Comments on input, but not on output
-//  - References
-//  - Buffers (just Buffers, not typed arrays)
-// The stringify function always creates valid json, as the syntax for references and buffers
-//  will just be special property names and values.
-// NOTE: We don't support Date serialization. Never store Dates, store "number".
-
 export interface JSONLACKS_ParseConfig {
     // Defaults to true. Enables parsing of:
     //  - Trailing commas
     //  - Non-quoted field names (ex, "{ x: 1 }")
     //  - Comments (strips them, but doesn't throw)
     extended?: boolean;
+    discardMissingReferences?: boolean;
 }
 export interface JSONLACKS_StringifyConfig {
     // If specified, we are allowed to mutate the provided object. Speeds up serialization.
@@ -41,11 +31,21 @@ interface HydrateState {
     visited: Set<unknown>,
 }
 
+// Supports json and also:
+//  - Non quoted field names "{ x: 1 }"?
+//  - Trailing commas
+//      NOTE: Comma only syntax is not supported, ex, "[,,]", which is an array of length 2 in javascript
+//  - Comments on input, but not on output
+//  - References
+//  - Buffers (just Buffers, not typed arrays)
+// The stringify function always creates valid json, as the syntax for references and buffers
+//  will just be special property names and values.
+// NOTE: We don't support Date serialization. Never store Dates, store "number".
 export class JSONLACKS {
     public static readonly LACKS_KEY = "__JSONLACKS__98cfb4a05fa34d828661cae15b8779ce__";
 
     /** If set to true parses non-quoted field names, comments, trailing commas, etc */
-    public static EXTENDED_PARSER = true;
+    public static EXTENDED_PARSER = false;
     public static IGNORE_MISSING_REFERENCES = false;
 
     @measureFnc
@@ -94,7 +94,7 @@ export class JSONLACKS {
             obj = measureBlock(function JSONparse() { return JSON.parse(text); });
         }
 
-        return JSONLACKS.hydrateSpecialObjects(obj, hydrateState) as T;
+        return JSONLACKS.hydrateSpecialObjects(obj, hydrateState, config) as T;
     }
     @measureFnc
     public static async parseLines<T>(buffer: Buffer, config?: JSONLACKS_ParseConfig): Promise<T[]> {
@@ -131,9 +131,32 @@ export class JSONLACKS {
                 linesJSON += lines[i];
             }
             linesJSON += "]";
-            let parts = JSONLACKS.parse(linesJSON, config, hydrateState) as T[];
-            for (let part of parts) {
-                output.push(part);
+            if (config?.discardMissingReferences) {
+                try {
+                    let parts = JSONLACKS.parse(linesJSON, config, hydrateState) as T[];
+                    for (let part of parts) {
+                        output.push(part);
+                    }
+                } catch (e: any) {
+                    if (!e.message.includes("Reference to undefined id")) {
+                        throw e;
+                    }
+                    for (let line of lines) {
+                        try {
+                            let part = JSONLACKS.parse(line, config, hydrateState) as T;
+                            output.push(part);
+                        } catch (e: any) {
+                            if (!e.message.includes("Reference to undefined id")) {
+                                throw e;
+                            }
+                        }
+                    }
+                }
+            } else {
+                let parts = JSONLACKS.parse(linesJSON, config, hydrateState) as T[];
+                for (let part of parts) {
+                    output.push(part);
+                }
             }
         }
         while (pos < buffer.length) {
@@ -237,7 +260,7 @@ export class JSONLACKS {
     }
 
     @measureFnc
-    private static hydrateSpecialObjects(obj: unknown, hydrateState?: HydrateState): unknown {
+    private static hydrateSpecialObjects(obj: unknown, hydrateState?: HydrateState, config?: JSONLACKS_ParseConfig): unknown {
         let references = hydrateState?.references || new Map<string, unknown>();
         let visited = hydrateState?.visited || new Set<unknown>();
         return iterate(obj);
@@ -271,8 +294,10 @@ export class JSONLACKS {
             if (type === "ref") {
                 let id = obj.id as string;
                 if (!JSONLACKS.IGNORE_MISSING_REFERENCES && !references.has(id)) {
-                    debugbreak(2);
-                    debugger;
+                    if (!config?.discardMissingReferences) {
+                        debugbreak(2);
+                        debugger;
+                    }
                     throw new Error(`Reference to undefined id "${id}"`);
                 }
                 return references.get(id);

package/src/batching.ts

@@ -1,5 +1,6 @@
 import { isNode } from "./misc";
 import { measureWrap } from "./profiling/measure";
+import { MaybePromise } from "./types";
 
 /*
     "numbers" use setTimeout
@@ -35,6 +36,14 @@ export function delay(delayTime: DelayType): Promise<void> {
     }
 }
 
+// NOTE: This is an easy way to turn off batching, without having to strip the extra batch handling code
+export function batchFunctionNone<Arg, Result = void>(
+    config: unknown,
+    fnc: (arg: Arg[]) => (Promise<Result> | Result)
+): (arg: Arg) => Promise<Result> {
+    return async arg => fnc([arg]);
+}
+
 export function batchFunction<Arg, Result = void>(
     config: {
         delay: DelayType;

package/src/caching.ts

@@ -12,6 +12,9 @@ export function lazy<T>(factory: () => T) {
     get.reset = () => {
         value = undefined;
     };
+    get.set = (newValue: T) => {
+        value = { value: newValue };
+    };
     return get;
 }
 

package/src/callHTTPHandler.ts

@@ -36,7 +36,7 @@ export function getNodeIdsFromRequest(request: http.IncomingMessage) {
     //  THAT WAY HTTP can have consistent nodeIds, instead of making them randomly every time!
     //  (This isn't needed or possible for websockets, but they stay open, so calling functions
     //      after they open to set the nodeId is possible, and preferred).
-    let remoteAddress = request.socket.remoteAddress;
+    let remoteAddress = request.socket.remoteAddress?.split(":").pop();
     if (!remoteAddress) {
         throw new Error(`Missing remoteAddress`);
     }
@@ -50,6 +50,8 @@ export function getNodeIdsFromRequest(request: http.IncomingMessage) {
 
 export async function httpCallHandler(request: http.IncomingMessage, response: http.ServerResponse) {
     try {
+        // Always set x-frame-options, to prevent iframe embedding click hijacking
+        response.setHeader("X-Frame-Options", "SAMEORIGIN");
 
         let urlBase = request.url;
         if (!urlBase) {
@@ -147,15 +149,6 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
         }
 
         let headers = (resultBuffer as HTTPResultType)[resultHeaders];
-        if (SocketFunction.compression?.type === "gzip" && !headers?.["Content-Encoding"]) {
-            if (request.headers["accept-encoding"]?.includes("gzip")) {
-                resultBuffer = await new Promise<Buffer>((resolve, reject) =>
-                    gzip(resultBuffer, (err, result) => err ? reject(err) : resolve(result))
-                );
-                response.setHeader("Content-Encoding", "gzip");
-            }
-        }
-
 
         // NOTE: Our ETag caching is only to reduce data sent on the wire, we evaluate the calls
         //  every time (so it is strictly a wire cache for HTTP, not a computation cache)

package/src/callManager.ts

@@ -29,7 +29,7 @@ export async function performLocalCall(
     }
 
     if (!exposedClasses.has(call.classGuid)) {
-        throw new Error(`Class ${call.classGuid} not exposed`);
+        throw new Error(`Class ${call.classGuid} not exposed, exposed classes: ${Array.from(exposedClasses).join(", ")}`);
     }
 
     let controller = classDef.controller;
@@ -95,7 +95,7 @@ export function unregisterGlobalClientHook(hook: SocketFunctionClientHook) {
 
 export const runClientHooks = measureWrap(async function runClientHooks(
     callType: FullCallType,
-    hooks: SocketExposedShape[""],
+    hooks: Exclude<SocketExposedShape[""], undefined>,
     connectionId: { nodeId: string },
 ): Promise<ClientHookContext> {
     let context: ClientHookContext = { call: callType, connectionId };
@@ -122,7 +122,7 @@ export const runClientHooks = measureWrap(async function runClientHooks(
 export const runServerHooks = measureWrap(async function runServerHooks(
     callType: FullCallType,
     caller: CallerContext,
-    hooks: SocketExposedShape[""],
+    hooks: Exclude<SocketExposedShape[""], undefined>,
 ): Promise<HookContext> {
     let hookContext: HookContext = { call: callType };
     for (let hook of globalHooks.concat(hooks.hooks || [])) {

package/src/formatting/format.ts

@@ -134,7 +134,12 @@ export function formatNumber(count: number | undefined, maxAbsoluteValue?: numbe
     // NOTE: We don't switch units as soon as we possible can, because...
     //  3.594 vs 3.584 is harder to quickly distinguish compared to 3594 and 3584,
     //  the decimal simply makes it harder to read, and larger.
-    const extraFactor = 10;
+    // NOTE: This number should prevent us from ever using 5 digits, so that we never need commas
+    //  For example, if the factor is 10 then, 9999.5 is kept with a divisor of 1, and then rounds up to
+    //  "10,000". So we want any value which rounds up at 5 digits to be put in the next group (and having
+    //  extra values put in the next group is fine, as we won't show the decimal value anyways, so it only
+    //  means 9999 wraps around to 10K a bit faster).
+    const extraFactor = 9.99949999;
     let divisor = 1;
     let suffix = "";
     let currencyDecimalsNeeded = false;

package/src/misc.ts

@@ -313,4 +313,53 @@ export function last<T>(arr: T[]): T | undefined {
 export type ObjectValues<T> = T[keyof T];
 export function entries<Obj extends { [key: string]: unknown }>(obj: Obj): [keyof Obj, ObjectValues<Obj>][] {
     return Object.entries(obj) as any;
+}
+
+export function sort<T>(arr: T[], sortKey: (obj: T) => unknown) {
+    if (arr.length <= 1) return arr;
+    arr.sort((a, b) => compare(sortKey(a), sortKey(b)));
+    return arr;
+}
+
+// NOTE: If there are duplicates, returns the first match.
+export function binarySearchIndex(listCount: number, compare: (lhsIndex: number) => number): number {
+    if (listCount === 0) {
+        return ~0;
+    }
+    let min = 0;
+    let max = listCount - 1;
+    while (min < max) {
+        let fingerIndex = Math.floor((max + min) / 2);
+        let comparisonValue = compare(fingerIndex);
+        if (comparisonValue < 0) {
+            min = fingerIndex + 1;
+        } else {
+            max = fingerIndex;
+        }
+    }
+    let comparison = compare(min);
+    if (comparison === 0) return min;
+    if (comparison > 0) return ~min;
+    return ~(min + 1);
+}
+
+export function compare(lhs: unknown, rhs: unknown): number {
+    if (typeof lhs !== typeof rhs) {
+        return compare(typeof lhs, typeof rhs);
+    }
+    if (lhs === rhs) return 0;
+    if (lhs as any < (rhs as any)) return -1;
+    return 1;
+}
+
+export function insertIntoSortedList<T>(list: T[], map: (val: T) => string | number, element: T) {
+    let searchValue = map(element);
+    let index = binarySearchIndex(list.length, i => compare(map(list[i]), searchValue));
+    if (index < 0) index = ~index;
+    list.splice(index, 0, element);
+}
+export function removeFromSortedList<T>(list: T[], map: (val: T) => string | number, searchValue: string | number) {
+    let index = binarySearchIndex(list.length, i => compare(map(list[i]), searchValue));
+    if (index < 0) return;
+    list.splice(index, 1);
 }

package/src/nodeCache.ts

@@ -2,6 +2,7 @@ import { CallFactory, createCallFactory } from "./CallFactory";
 import { MaybePromise } from "./types";
 import { lazy } from "./caching";
 import { SocketFunction } from "../SocketFunction";
+import { isNode } from "./misc";
 
 // TODO: Add CallInstanceFactory.isClosed, so nodeCache can clean up old entries.
 //  This is only needed for memory management, and not for correctness. Entries never
@@ -17,6 +18,11 @@ export function getNodeId(domain: string, port: number): string {
     return `${domain}:${port}`;
 }
 
+export function getNodeIdFromLocation() {
+    if (isNode()) throw new Error(`Cannot get nodeId from location, as we are running in NodeJS`);
+    return getNodeId(location.hostname, location.port ? parseInt(location.port) : 443);
+}
+
 /** A nodeId not available for reconnecting. */
 export function getClientNodeId(address: string): string {
     return `client:${address}:${Date.now()}:${Math.random()}`;
@@ -44,9 +50,16 @@ export function getNodeIdLocation(nodeId: string): { address: string, port: numb
 }
 
 export function getNodeIdDomain(nodeId: string): string {
+    let result = getNodeIdDomainMaybeUndefined(nodeId);
+    if (result === undefined) {
+        throw new Error(`Cannot get domain from nodeId, which is only usable as a client. NodeId: ${JSON.stringify(nodeId)}`);
+    }
+    return result;
+}
+export function getNodeIdDomainMaybeUndefined(nodeId: string): string | undefined {
     let location = getNodeIdLocation(nodeId);
     if (!location) {
-        throw new Error(`Cannot get domain from nodeId, which is only usable as a client. NodeId: ${JSON.stringify(nodeId)}`);
+        return undefined;
     }
     return new URL(location.address).hostname.split(".").slice(-2).join(".");
 }

package/src/profiling/getOwnTime.ts

@@ -52,6 +52,7 @@ let addMeasureOverheadTime = 0;
 
 // TIMING: About 60ns, of which 40ns is just now() calls.
 //  If async is closer to 300ns.
+// NOTE: Handles promises correctly
 export function getOwnTime<T>(
     name: string,
     code: () => T,

package/src/profiling/measure.ts

@@ -6,7 +6,9 @@ import { getOpenTimesBase, getOwnTime, OwnTimeObj } from "./getOwnTime";
 import { addToStats, addToStatsValue, createStatsValue, getStatsTop, StatsValue } from "./stats";
 import { white } from "../formatting/logColors";
 import { isNode } from "../misc";
+import { formatStats } from "./statsFormat";
 
+let measurementsDisabled = false;
 /** NOTE: Must be called BEFORE anything else is imported!
  *      NOTE: Measurements on on by default now, so this doesn't really need to be called...
 */
@@ -19,6 +21,7 @@ export function enableMeasurements() {
 /** NOTE: Must be called BEFORE anything else is imported! */
 export function disableMeasurements() {
     measurementsEnabled = false;
+    measurementsDisabled = true;
 }
 
 let functionsSkipped = 0;
@@ -30,6 +33,7 @@ const AsyncFunction = (async () => { }).constructor;
 // TIMING: 1-5us. I have seen timing values greatly vary, but it does seem to be quite high, despite
 //  microbenchmarks saying it is slow. Perhaps it is because getOwnTime breaks the cpu pipeline,
 //  which causes slowness for code around us, but not if we are running in isolation?
+// NOTE: Handles promises correctly
 export function measureFnc(target: any, propertyKey: string, descriptor: PropertyDescriptor) {
     let name = propertyKey;
     if (target.name) {
@@ -51,6 +55,7 @@ export function nameFunction<T extends Function>(name: string, fnc: T) {
     Object.defineProperty(fnc, "name", { value: name });
     return fnc;
 }
+// NOTE: Handles promises correctly
 export function measureWrap<T extends (...args: any[]) => any>(fnc: T, name?: string): T {
     if (!measurementsEnabled) {
         functionsSkipped++;
@@ -71,7 +76,7 @@ export function measureBlock<T extends (...args: any[]) => any>(fnc: T, name?: s
 export function startMeasure(): {
     finish: () => MeasureProfile;
 } {
-    if (!measurementsEnabled) {
+    if (!measurementsEnabled && !measurementsDisabled) {
         console.warn(red(`To capture measurements enableMeasurements() must be called before any other imports in your entry point`));
     }
     let profile: MeasureProfile = {
@@ -154,27 +159,8 @@ export function logMeasureTable(
             return String(text).padStart(count, " ");
         }
         let fractionText = percent(getTime(entry).sum / totalTime);
-        let perText = formatTime(getTime(entry).sum / getTime(entry).count);
-        let countText = formatNumber(getTime(entry).count);
-        let sumText = formatTime(getTime(entry).sum);
-
-        let equation = `${p(6, sumText)} =  ${p(6, countText)} * ${p(6, perText)}`;
-
-        let ownTimeTop = getStatsTop(getTime(entry));
-        if (ownTimeTop.topHeavy) {
-            let topText = formatTime(ownTimeTop.value / ownTimeTop.count);
-            let topCountText = formatNumber(ownTimeTop.count);
-            let bottomText = formatTime((getTime(entry).sum - ownTimeTop.value) / getTime(entry).count, ownTimeTop.value);
-            let bottomCountText = formatNumber(getTime(entry).count - ownTimeTop.count);
-            let topPart = `${p(6, topText)} per * ${topCountText}`;
-            let bottomPart = `${bottomText} * ${bottomCountText}`;
-            if (isNode()) {
-                topPart = red(topPart);
-            } else {
-                bottomPart = white(bottomPart);
-            }
-            equation = `${p(6, sumText)} = ${p(6, topPart)}   +  ${bottomPart}`;
-        }
+
+        let equation = formatStats(getTime(entry));
 
         let text = `${p(6, fractionText)} ( ${equation} )`;
         let overhead = measureOverhead * getTime(entry).count;
@@ -204,7 +190,7 @@ export async function measureCode<T>(code: () => Promise<T>, config?: LogMeasure
     try {
         return await measureBlock(code, code.name || "untracked");
     } finally {
-        finishProfile(measure, config);
+        finishProfile(measure, config || { name: code.name, minTimeToLog: 0 });
     }
 }
 export function measureCodeSync<T>(code: () => T, config?: LogMeasureTableConfig): T {

package/src/profiling/statsFormat.ts

@@ -0,0 +1,41 @@
+import { formatTime, formatNumber } from "../formatting/format";
+import { red, white } from "../formatting/logColors";
+import { isNode } from "../misc";
+import { StatsValue, getStatsTop } from "./stats";
+
+export function percent(value: number) {
+    return `${(value * 100).toFixed(2)}%`;
+}
+
+export function formatStats(stats: StatsValue, config?: {
+    noColor?: boolean;
+    noSum?: boolean;
+}) {
+    function p(count: number, text: string | number) {
+        return String(text).padStart(count, " ");
+    }
+
+    let perText = formatTime(stats.sum / stats.count);
+    let countText = formatNumber(stats.count);
+    let sumText = formatTime(stats.sum);
+    let equation = (!config?.noSum && `${p(6, sumText)} =  ` || "") + `${p(6, countText)} * ${p(6, perText)}`;
+
+    let top = getStatsTop(stats);
+    if (top.topHeavy) {
+        let topText = formatTime(top.value / top.count);
+        let topCountText = formatNumber(top.count);
+        let bottomText = formatTime((stats.sum - top.value) / (stats.count - top.count) || 0);
+        let bottomCountText = formatNumber(stats.count - top.count);
+        let topPart = `${topCountText} * ${p(6, topText)}`;
+        let bottomPart = `${bottomCountText} * ${bottomText}`;
+        if (!config?.noColor) {
+            if (isNode()) {
+                topPart = red(topPart);
+            } else {
+                bottomPart = white(bottomPart);
+            }
+        }
+        equation = (!config?.noSum && `${p(6, sumText)} = ` || "") + `${p(6, topPart)}   +  ${bottomPart}`;
+    }
+    return equation;
+}

package/src/webSocketServer.ts

@@ -239,9 +239,7 @@ export async function startSocketServer(
 
     port = (realServer.address() as net.AddressInfo).port;
     let nodeId = getNodeId(getCommonName(config.cert), port);
-    if (!SocketFunction.silent) {
-        console.log(green(`Started Listening on ${nodeId}`));
-    }
+    console.log(green(`Started Listening on ${nodeId}`));
 
     return nodeId;
 }