socket-function 0.9.4 → 0.9.6

package/SocketFunction.ts

@@ -10,6 +10,12 @@ import { setDefaultHTTPCall } from "./src/callHTTPHandler";
 import debugbreak from "debugbreak";
 import { lazy } from "./src/caching";
 import { delay } from "./src/batching";
+import { blue, magenta } from "./src/formatting/logColors";
+import { JSONLACKS } from "./src/JSONLACKS/JSONLACKS";
+import cborx from "cbor-x";
+import { setFlag } from "./require/compileFlags";
+setFlag(require, "cbor-x", "allowclient", true);
+let cborxInstance = new cborx.Encoder({ structuredClone: true });
 
 module.allowclient = true;
 
@@ -22,22 +28,23 @@ type ExtractShape<ClassType, Shape> = {
         : "Function is in shape, but not in class"
     );
 };
-// https://stackoverflow.com/a/69756175/1117119
-type PickByType<T, Value> = {
-    [P in keyof T as T[P] extends Value | undefined ? P : never]: T[P]
-};
 
 export class SocketFunction {
     public static logMessages = false;
 
     public static MAX_MESSAGE_SIZE = 1024 * 1024 * 32;
-    public static compression: undefined | {
-        type: "gzip";
-    };
 
     public static httpETagCache = false;
     public static silent = true;
 
+    // In retrospect... dynamically changing the wire serializer is a BAD idea. If any calls happen
+    //  before it is changed, things just break. Also, it needs to be changed on both sides,
+    //  or else things break. Also, it is very hard to detect when the issue is different serializers
+    public static readonly WIRE_SERIALIZER = {
+        serialize: (obj: unknown): MaybePromise<Buffer[]> => [cborxInstance.encode(obj)],
+        deserialize: (buffers: Buffer[]): MaybePromise<unknown> => cborxInstance.decode(buffers[0]),
+    };
+
     public static WIRE_WARN_TIME = 100;
 
     private static onMountCallbacks = new Map<string, (() => MaybePromise<void>)[]>();
@@ -54,18 +61,24 @@ export class SocketFunction {
     //  (ex, using a hook in a controller where the hook also calls the controller).
     public static register<
         ClassInstance extends object,
-        Shape extends SocketExposedShape<SocketExposedInterface>,
+        Shape extends SocketExposedShape<{
+            [key in keyof ClassInstance]: (...args: any[]) => Promise<unknown>;
+        }>,
     >(
         classGuid: string,
         instance: ClassInstance,
         shapeFnc: () => Shape,
         defaultHooksFnc?: () => SocketExposedShape[""] & {
             onMount?: () => MaybePromise<void>;
+        },
+        config?: {
+            /** @noAutoExpose If true SocketFunction.expose(Controller) must be called explicitly. */
+            noAutoExpose?: boolean;
         }
     ): SocketRegistered<ExtractShape<ClassInstance, Shape>> {
         let getDefaultHooks = defaultHooksFnc && lazy(defaultHooksFnc);
         const getShape = lazy(() => {
-            let shape = shapeFnc();
+            let shape = shapeFnc() as SocketExposedShape;
             let defaultHooks = getDefaultHooks?.();
 
             for (let value of Object.values(shape)) {
@@ -96,7 +109,7 @@ export class SocketFunction {
                     throw new Error(`Function ${functionName} is not in shape`);
                 }
 
-                let hookResult = await runClientHooks(call, shapeObj as SocketExposedShape[""], callFactory.connectionId);
+                let hookResult = await runClientHooks(call, shapeObj as Exclude<SocketExposedShape[""], undefined>, callFactory.connectionId);
 
                 if ("overrideResult" in hookResult) {
                     return hookResult.overrideResult;
@@ -128,7 +141,11 @@ export class SocketFunction {
             }
         });
 
-        return output as any;
+        let result = output as any as SocketRegistered;
+        if (!config?.noAutoExpose) {
+            this.expose(result);
+        }
+        return result;
     }
 
     public static onNextDisconnect(nodeId: string, callback: () => void) {
@@ -185,8 +202,9 @@ export class SocketFunction {
      *      to add additional imports to ensure the register call runs.
      */
     public static expose(socketRegistered: SocketRegistered) {
+        console.log(blue(`Exposing Controller ${socketRegistered._classGuid}`));
         exposeClass(socketRegistered);
-        SocketFunction.exposedClasses.add(socketRegistered._classGuid);
+        this.exposedClasses.add(socketRegistered._classGuid);
 
         if (this.hasMounted) {
             let mountCallbacks = SocketFunction.onMountCallbacks.get(socketRegistered._classGuid);
@@ -199,6 +217,7 @@ export class SocketFunction {
     }
 
     public static mountedNodeId: string = "";
+    public static isMounted() { return !!this.mountedNodeId; }
     public static mountedIP: string = "";
     private static hasMounted = false;
     private static onMountCallback: () => void = () => { };

package/SocketFunctionTypes.ts

@@ -22,8 +22,8 @@ export type SocketExposedInterfaceClass = {
     new(): unknown;
     prototype: unknown;
 };
-export interface SocketExposedShape<ExposedType extends SocketExposedInterface = SocketExposedInterface> {
-    [functionName: string]: {
+export type SocketExposedShape<ExposedType extends SocketExposedInterface = SocketExposedInterface> = {
+    [functionName in keyof ExposedType]?: {
         /** Indicates with the same input, we give the same output, forever,
          *      independent of code changes. This only works for data storage.
          */
@@ -31,7 +31,7 @@ export interface SocketExposedShape<ExposedType extends SocketExposedInterface =
         hooks?: SocketFunctionHook<ExposedType>[];
         clientHooks?: SocketFunctionClientHook<ExposedType>[];
     };
-}
+};
 
 export interface CallType {
     classGuid: string;

package/package.json

@@ -1,15 +1,16 @@
 {
 	"name": "socket-function",
-	"version": "0.9.4",
+	"version": "0.9.6",
 	"main": "index.js",
 	"license": "MIT",
 	"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
 	"dependencies": {
+		"cbor-x": "^1.5.6",
 		"cookie": "^0.5.0",
 		"mobx": "^6.6.2",
 		"node-forge": "https://github.com/sliftist/forge#name",
 		"preact": "^10.10.6",
-		"typenode": "^4.9.4-g",
+		"typenode": "^4.9.4-j",
 		"ws": "^8.8.0"
 	},
 	"optionalDependencies": {

package/require/RequireController.ts

@@ -223,7 +223,7 @@ class RequireControllerBase {
             //require(rootPath);
             let clientModule = require.cache[resolvedPath];
             if (!clientModule) {
-                clientModule = createNotFoundModule(`Module ${pathRequest} (resolved to ${JSON.stringify(resolvedPath)}) was not included serverside. Resolve root ${JSON.stringify(this.rootResolvePath)} (set by call to setRequireBootRequire), resolve search paths: ${JSON.stringify(searchPaths)})}`);
+                clientModule = createNotFoundModule(`Module ${pathRequest} (resolved to ${JSON.stringify(resolvedPath)}) was not included serverside. Resolved from root dir ${JSON.stringify(this.rootResolvePath)} (set by call to setRequireBootRequire), resolve search paths: ${JSON.stringify(searchPaths)})}`);
             }
             if (!clientModule.allowclient) {
                 clientModule = createNotFoundModule(`Module ${pathRequest} (resolved to ${resolvedPath}) is not allowed clientside (set module.allowclient in it, or call setFlag when it is imported).`);
@@ -249,5 +249,9 @@ export const RequireController = SocketFunction.register(
         requireHTML: {},
         bufferJS: {},
         requireJS: {},
-    })
+    }),
+    undefined,
+    {
+        noAutoExpose: true
+    }
 );

package/require/require.js

@@ -27,12 +27,15 @@
             // https://nodejs.org/api/util.html#util_util_inherits_constructor_superconstructor
             inherits(constructor, superConstructor) {
                 Object.setPrototypeOf(constructor.prototype, superConstructor.prototype);
-            }
+            },
+            TextDecoder: TextDecoder,
+            TextEncoder: TextEncoder,
         },
         buffer: { Buffer },
         stream: {
             // HACK: Needed to get SAX JS to work correctly.
             Stream: function () { },
+            Transform: function () { },
         },
         timers: {
             // TODO: Add all members of timers

package/src/CallFactory.ts

@@ -9,17 +9,16 @@ import * as tls from "tls";
 import { getClientNodeId, getNodeIdLocation, registerNodeClient } from "./nodeCache";
 import debugbreak from "debugbreak";
 import { lazy } from "./caching";
-import { JSONLACKS } from "./JSONLACKS/JSONLACKS";
 import { red, yellow } from "./formatting/logColors";
 import { isSplitableArray, markArrayAsSplitable } from "./fixLargeNetworkCalls";
-import { delay } from "./batching";
+import { delay, runInSerial } from "./batching";
+import { formatNumber, formatTime } from "./formatting/format";
 
 const MIN_RETRY_DELAY = 1000;
 
 type InternalCallType = FullCallType & {
     seqNum: number;
     isReturn: false;
-    compress: boolean;
 }
 
 type InternalReturnType = {
@@ -76,7 +75,7 @@ export async function createCallFactory(
     const canReconnect = !!getNodeIdLocation(nodeId);
 
     let pendingCalls: Map<number, {
-        data: Buffer;
+        data: Buffer[];
         call: InternalCallType;
         callback: (resultJSON: InternalReturnType) => void;
     }> = new Map();
@@ -123,16 +122,22 @@ export async function createCallFactory(
                 classGuid: call.classGuid,
                 functionName: call.functionName,
                 seqNum,
-                compress: !!SocketFunction.compression,
             };
             let time = Date.now();
-            let data = Buffer.from(JSONLACKS.stringify(fullCall));
+            let data: Buffer[];
+            let dataMaybePromise = SocketFunction.WIRE_SERIALIZER.serialize(fullCall);
+            if (dataMaybePromise instanceof Promise) {
+                data = await dataMaybePromise;
+            } else {
+                data = dataMaybePromise;
+            }
             time = Date.now() - time;
+            let size = data.map(x => x.length).reduce((a, b) => a + b, 0);
             if (time > SocketFunction.WIRE_WARN_TIME) {
-                console.log(red(`Slow serialize, took ${time}ms to serialize ${data.byteLength} bytes. For ${call.classGuid}.${call.functionName}`));
+                console.log(red(`Slow serialize, took ${formatTime(time)} to serialize ${formatNumber(size)} bytes. For ${call.classGuid}.${call.functionName}`));
             }
 
-            if (data.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
+            if (size > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
                 let splitArgIndex = call.args.findIndex(isSplitableArray);
                 if (splitArgIndex >= 0) {
                     console.log(yellow(`Splitting large call due to large args: ${call.classGuid}.${call.functionName}`));
@@ -157,7 +162,7 @@ export async function createCallFactory(
                     return "CALLS_SPLIT_DUE_TO_LARGE_ARGS";
                 }
 
-                throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${data.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
+                throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${formatNumber(size)} > ${formatNumber(SocketFunction.MAX_MESSAGE_SIZE)}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
             }
 
             let resultPromise = new Promise((resolve, reject) => {
@@ -251,7 +256,8 @@ export async function createCallFactory(
         }
     }
 
-    async function send(data: Buffer) {
+    const BASE_LENGTH_OFFSET = 324_432_461_592_612;
+    async function send(data: Buffer[]) {
         if (!webSocketPromise) {
             if (canReconnect) {
                 webSocketPromise = tryToReconnect();
@@ -260,7 +266,10 @@ export async function createCallFactory(
             }
         }
         let webSocket = await webSocketPromise;
-        webSocket.send(data);
+        webSocket.send((data.length + BASE_LENGTH_OFFSET).toString());
+        for (let d of data) {
+            webSocket.send(d);
+        }
     }
     async function tryToReconnect(): Promise<SenderInterface> {
         // Don't try to reconnect too often!
@@ -276,7 +285,8 @@ export async function createCallFactory(
         return newWebSocket;
     }
 
-
+    let pendingCall: { buffers: Buffer[]; targetCount: number; } | undefined;
+    let clientsideSerial = runInSerial(async <T>(val: Promise<T>) => val);
     async function onMessage(message: ws.RawData | ws.MessageEvent | string) {
         try {
             if (typeof message === "object" && "data" in message) {
@@ -284,35 +294,54 @@ export async function createCallFactory(
             }
             if (!isNode()) {
                 if (message instanceof Blob) {
-                    message = Buffer.from(await message.arrayBuffer());
+                    // We need to force the results to be in serial, otherwise strings leapfrog
+                    //  ahead of buffers, which breaks things.
+                    message = Buffer.from(await clientsideSerial(message.arrayBuffer()));
+                } else {
+                    await clientsideSerial(Promise.resolve());
+                }
+            }
+            if (typeof message === "string") {
+                let size = parseInt(message);
+                if (isNaN(size)) {
+                    throw new Error(`Invalid message size ${message}`);
+                }
+                if (size < BASE_LENGTH_OFFSET) {
+                    throw new Error(`Invalid message size ${message}`);
                 }
+                size -= BASE_LENGTH_OFFSET;
+                if (size > 1000 * 1000) {
+                    throw new Error(`Invalid message size ${size}`);
+                }
+                pendingCall = {
+                    buffers: [],
+                    targetCount: size,
+                };
+                return;
             }
-            if (message instanceof Buffer || typeof message === "string") {
-
-                let resultSize = message.length;
-
-                if (message instanceof Buffer && message[0] === 0) {
-                    // First byte of 0 means it is decompressed (as JSON can't have a first byte of 0).
-                    (message as any) = message.slice(1);
-
-                    // TODO: Add typings for DecompressionStream
-                    let DecompressionStream = (window as any).DecompressionStream;
-                    // https://stackoverflow.com/a/68829631/1117119
-                    let stream = new DecompressionStream("gzip");
-                    let blob = new Blob([message]);
-                    let decompressedStream = (await (blob.stream() as any).pipeThrough(stream));
-                    let arrayBuffer = await new Response(decompressedStream).arrayBuffer();
-                    (message as any) = Buffer.from(arrayBuffer);
+            if (message instanceof Buffer) {
+                if (!pendingCall) {
+                    throw new Error(`Received data without size`);
+                }
+                pendingCall.buffers.push(message);
+                let currentBuffers: Buffer[];
+                if (pendingCall.buffers.length !== pendingCall.targetCount) {
+                    return;
                 }
 
+                currentBuffers = pendingCall.buffers;
+                pendingCall = undefined;
+
+                let resultSize = currentBuffers.map(x => x.length).reduce((a, b) => a + b, 0);
+
                 let time = Date.now();
-                let call = JSONLACKS.parse(message.toString(), { extended: false }) as InternalCallType | InternalReturnType;
+                let call = await SocketFunction.WIRE_SERIALIZER.deserialize(currentBuffers) as InternalCallType | InternalReturnType;
                 time = Date.now() - time;
 
                 if (call.isReturn) {
                     let callbackObj = pendingCalls.get(call.seqNum);
                     if (time > SocketFunction.WIRE_WARN_TIME) {
-                        console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for receieving result of call to ${callbackObj?.call.classGuid}.${callbackObj?.call.functionName}`));
+                        console.log(red(`Slow parse, took ${time}ms to parse ${resultSize} bytes, for receieving result of call to ${callbackObj?.call.classGuid}.${callbackObj?.call.functionName}`));
                     }
                     if (!callbackObj) {
                         console.log(`Got return for unknown call ${call.seqNum}`);
@@ -322,7 +351,7 @@ export async function createCallFactory(
                     callbackObj.callback(call);
                 } else {
                     if (time > SocketFunction.WIRE_WARN_TIME) {
-                        console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for call to ${call.classGuid}.${call.functionName}`));
+                        console.log(red(`Slow parse, took ${time}ms to parse ${resultSize} bytes, for call to ${call.classGuid}.${call.functionName}`));
                     }
 
                     let response: InternalReturnType;
@@ -346,27 +375,18 @@ export async function createCallFactory(
                         };
                     }
 
-                    let result: Buffer;
-                    if (isNode() && call.compress && SocketFunction.compression?.type === "gzip") {
-                        response.compressed = true;
-                        result = Buffer.from(JSONLACKS.stringify(response));
-                        result = await new Promise<Buffer>((resolve, reject) =>
-                            gzip(result, (err, result) => err ? reject(err) : resolve(result))
-                        );
-                        result = Buffer.concat([new Uint8Array([0]), result]);
-                    } else {
-                        result = Buffer.from(JSONLACKS.stringify(response));
-                    }
-                    if (result.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
+                    let result: Buffer[] = await SocketFunction.WIRE_SERIALIZER.serialize(response);
+                    let totalResultSize = result.map(x => x.length).reduce((a, b) => a + b, 0);
+                    if (totalResultSize > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
                         response = {
                             isReturn: true,
                             result: undefined,
                             seqNum: call.seqNum,
-                            error: new Error(`Response too large to send (${call.classGuid}.${call.functionName}, size: ${result.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`).stack,
+                            error: new Error(`Response too large to send (${call.classGuid}.${call.functionName}, size: ${formatNumber(totalResultSize)} > ${formatNumber(SocketFunction.MAX_MESSAGE_SIZE)}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`).stack,
                             resultSize: resultSize,
                             compressed: false,
                         };
-                        result = Buffer.from(JSONLACKS.stringify(response));
+                        result = await SocketFunction.WIRE_SERIALIZER.serialize(response);
                     }
                     await send(result);
                 }
@@ -379,8 +399,6 @@ export async function createCallFactory(
             if (e.stack.startsWith("Error: Cannot send data to") && e.stack.includes("as the connection has closed")) {
                 // This is fine, just ignore it
             } else {
-                debugbreak(2);
-                debugger;
                 console.error(e.stack);
             }
         }

package/src/JSONLACKS/JSONLACKS.ts

@@ -13,23 +13,13 @@ import { delay } from "../batching";
 const SERIALIZE_OBJECT_BATCH_COUNT = 1000;
 const PARSE_BYTE_CHUNK_SIZE = 1024 * 1024 * 10;
 
-// Supports json and also:
-//  - Non quoted field names "{ x: 1 }"?
-//  - Trailing commas
-//      NOTE: Comma only syntax is not supported, ex, "[,,]", which is an array of length 2 in javascript
-//  - Comments on input, but not on output
-//  - References
-//  - Buffers (just Buffers, not typed arrays)
-// The stringify function always creates valid json, as the syntax for references and buffers
-//  will just be special property names and values.
-// NOTE: We don't support Date serialization. Never store Dates, store "number".
-
 export interface JSONLACKS_ParseConfig {
     // Defaults to true. Enables parsing of:
     //  - Trailing commas
     //  - Non-quoted field names (ex, "{ x: 1 }")
     //  - Comments (strips them, but doesn't throw)
     extended?: boolean;
+    discardMissingReferences?: boolean;
 }
 export interface JSONLACKS_StringifyConfig {
     // If specified, we are allowed to mutate the provided object. Speeds up serialization.
@@ -41,11 +31,21 @@ interface HydrateState {
     visited: Set<unknown>,
 }
 
+// Supports json and also:
+//  - Non quoted field names "{ x: 1 }"?
+//  - Trailing commas
+//      NOTE: Comma only syntax is not supported, ex, "[,,]", which is an array of length 2 in javascript
+//  - Comments on input, but not on output
+//  - References
+//  - Buffers (just Buffers, not typed arrays)
+// The stringify function always creates valid json, as the syntax for references and buffers
+//  will just be special property names and values.
+// NOTE: We don't support Date serialization. Never store Dates, store "number".
 export class JSONLACKS {
     public static readonly LACKS_KEY = "__JSONLACKS__98cfb4a05fa34d828661cae15b8779ce__";
 
     /** If set to true parses non-quoted field names, comments, trailing commas, etc */
-    public static EXTENDED_PARSER = true;
+    public static EXTENDED_PARSER = false;
     public static IGNORE_MISSING_REFERENCES = false;
 
     @measureFnc
@@ -68,6 +68,18 @@ export class JSONLACKS {
             return Buffer.concat(buffers);
         });
     }
+    public static stringifyFileSync(obj: unknown[], config?: JSONLACKS_StringifyConfig): Buffer {
+        let serialized = JSONLACKS.escapeSpecialObjects(obj, config) as unknown[];
+        return measureBlock(function JSONstringifyAndJoin() {
+            let buffers: Buffer[] = [];
+            for (let i = 0; i < serialized.length; i += SERIALIZE_OBJECT_BATCH_COUNT) {
+                let str = serialized.slice(i, i + SERIALIZE_OBJECT_BATCH_COUNT).map(x => JSON.stringify(x) + "\n").join("");
+                buffers.push(Buffer.from(str));
+            }
+            // Break up into chunks, as string => Buffer i
+            return Buffer.concat(buffers);
+        });
+    }
     // TIMING: Seems to be about 40X slower than JSON.parse unless extended is set to false,
     //  then it is about 2X slower (although it depends on the size and complexity of the objects!)
     @measureFnc
@@ -82,7 +94,7 @@ export class JSONLACKS {
             obj = measureBlock(function JSONparse() { return JSON.parse(text); });
         }
 
-        return JSONLACKS.hydrateSpecialObjects(obj, hydrateState) as T;
+        return JSONLACKS.hydrateSpecialObjects(obj, hydrateState, config) as T;
     }
     @measureFnc
     public static async parseLines<T>(buffer: Buffer, config?: JSONLACKS_ParseConfig): Promise<T[]> {
@@ -119,9 +131,32 @@ export class JSONLACKS {
                 linesJSON += lines[i];
             }
             linesJSON += "]";
-            let parts = JSONLACKS.parse(linesJSON, config, hydrateState) as T[];
-            for (let part of parts) {
-                output.push(part);
+            if (config?.discardMissingReferences) {
+                try {
+                    let parts = JSONLACKS.parse(linesJSON, config, hydrateState) as T[];
+                    for (let part of parts) {
+                        output.push(part);
+                    }
+                } catch (e: any) {
+                    if (!e.message.includes("Reference to undefined id")) {
+                        throw e;
+                    }
+                    for (let line of lines) {
+                        try {
+                            let part = JSONLACKS.parse(line, config, hydrateState) as T;
+                            output.push(part);
+                        } catch (e: any) {
+                            if (!e.message.includes("Reference to undefined id")) {
+                                throw e;
+                            }
+                        }
+                    }
+                }
+            } else {
+                let parts = JSONLACKS.parse(linesJSON, config, hydrateState) as T[];
+                for (let part of parts) {
+                    output.push(part);
+                }
             }
         }
         while (pos < buffer.length) {
@@ -225,7 +260,7 @@ export class JSONLACKS {
     }
 
     @measureFnc
-    private static hydrateSpecialObjects(obj: unknown, hydrateState?: HydrateState): unknown {
+    private static hydrateSpecialObjects(obj: unknown, hydrateState?: HydrateState, config?: JSONLACKS_ParseConfig): unknown {
         let references = hydrateState?.references || new Map<string, unknown>();
         let visited = hydrateState?.visited || new Set<unknown>();
         return iterate(obj);
@@ -259,8 +294,10 @@ export class JSONLACKS {
             if (type === "ref") {
                 let id = obj.id as string;
                 if (!JSONLACKS.IGNORE_MISSING_REFERENCES && !references.has(id)) {
-                    debugbreak(2);
-                    debugger;
+                    if (!config?.discardMissingReferences) {
+                        debugbreak(2);
+                        debugger;
+                    }
                     throw new Error(`Reference to undefined id "${id}"`);
                 }
                 return references.get(id);

package/src/batching.ts

@@ -113,15 +113,22 @@ export function batchFunction<Arg, Result = void>(
 }
 
 export function runInSerial<T extends (...args: any[]) => Promise<any>>(fnc: T): T {
-    let updateQueue: (() => void)[] = [];
+    let updateQueue: { promise: Promise<void>; resolve: () => void; }[] = [];
 
     return (async (...args: any[]) => {
+        let promise = {
+            promise: null as any as Promise<void>,
+            resolve: () => { },
+        };
+        promise.promise = new Promise<void>(resolve => {
+            promise.resolve = resolve;
+        });
         const queueWasEmpty = updateQueue.length === 0;
-        if (!queueWasEmpty) {
-            // Wait for the previous promise to resolve
-            await new Promise<void>(resolve => updateQueue.push(resolve));
+        if (queueWasEmpty) {
+            promise.resolve();
         }
-        updateQueue.push(() => { });
+        updateQueue.push(promise);
+        await promise.promise;
 
         try {
             return await fnc(...args);
@@ -129,7 +136,7 @@ export function runInSerial<T extends (...args: any[]) => Promise<any>>(fnc: T):
             // Pop ourself off
             updateQueue.shift();
             // Resolve the next promise
-            updateQueue[0]?.();
+            updateQueue[0]?.resolve();
         }
     }) as T;
 }

package/src/caching.ts

@@ -12,6 +12,9 @@ export function lazy<T>(factory: () => T) {
     get.reset = () => {
         value = undefined;
     };
+    get.set = (newValue: T) => {
+        value = { value: newValue };
+    };
     return get;
 }
 
@@ -38,8 +41,10 @@ export function cache<Output, Key>(getValue: (key: Key) => Output): {
     (key: Key): Output;
     // NOTE: If you want to clear all, just make a new cache!
     clear(key: Key): void;
+    clearAll(): void;
     forceSet(key: Key, value: Output): void;
     getAllKeys(): Key[];
+    get(key: Key): Output | undefined;
 } {
     let startingCalculating = new Set<Key>();
     let values = new Map<Key, Output>();
@@ -69,6 +74,13 @@ export function cache<Output, Key>(getValue: (key: Key) => Output): {
     cache.getAllKeys = () => {
         return [...values.keys()];
     };
+    cache.get = (key: Key) => {
+        return values.get(key);
+    };
+    cache.clearAll = () => {
+        values.clear();
+        startingCalculating.clear();
+    };
     return cache;
 }
 
@@ -114,6 +126,14 @@ export function cacheLimited<Output, Key>(
         values.set(key, value);
         startingCalculating.add(key);
     };
+    get["clearKey"] = (key: Key) => {
+        values.delete(key);
+        startingCalculating.delete(key);
+    };
+    get["clear"] = () => {
+        values.clear();
+        startingCalculating.clear();
+    };
 
     return get;
 }
@@ -186,6 +206,7 @@ export function cacheArrayEqual<Input extends unknown[] | undefined, Output>(
 ): {
     (array: Input): Output;
     clear(array: Input): void;
+    clearAll(): void;
 } {
     let state: {
         cache: {
@@ -227,7 +248,10 @@ export function cacheArrayEqual<Input extends unknown[] | undefined, Output>(
                         state.cache.splice(i, 1);
                     }
                 }
-            }
+            },
+            clearAll() {
+                state.cache = [];
+            },
         }
     );
 }
@@ -247,7 +271,7 @@ export function cacheArgsEqual<Fnc extends AnyFunction>(
         {
             clear(...args: unknown[]) {
                 cache.clear(args);
-            }
+            },
         }
     );
 }
@@ -255,13 +279,23 @@ export function cacheArgsEqual<Fnc extends AnyFunction>(
 export function cacheJSONArgsEqual<Fnc extends AnyFunction>(
     fnc: Fnc,
     limit = 10
-): Fnc {
+) {
     let cache = cacheLimited(limit, (argsJSON: string) => {
         return fnc(...JSON.parse(argsJSON));
     });
-    return ((...args: unknown[]) => {
-        return cache(JSON.stringify(args));
-    }) as Fnc;
+    return Object.assign(
+        ((...args: unknown[]) => {
+            return cache(JSON.stringify(args));
+        }) as Fnc,
+        {
+            clear(...args: unknown[]) {
+                cache.clearKey(JSON.stringify(args));
+            },
+            clearAll() {
+                cache.clear();
+            }
+        }
+    );
 }
 
 export function cacheShallowConfigArgEqual<Fnc extends AnyFunction>(
@@ -269,6 +303,7 @@ export function cacheShallowConfigArgEqual<Fnc extends AnyFunction>(
     limit = 10
 ): Fnc & {
     clear(...args: Args<Fnc>): void;
+    clearAll(): void;
 } {
     let cache = cacheArrayEqual((kvpsFlat: unknown[]) => {
         output.missCount++;
@@ -301,6 +336,9 @@ export function cacheShallowConfigArgEqual<Fnc extends AnyFunction>(
             clear(configArg: object) {
                 cache.clear(getKVPs(configArg));
             },
+            clearAll() {
+                cache.clearAll();
+            },
             callCount: 0,
             missCount: 0,
         }

package/src/callHTTPHandler.ts

@@ -36,7 +36,7 @@ export function getNodeIdsFromRequest(request: http.IncomingMessage) {
     //  THAT WAY HTTP can have consistent nodeIds, instead of making them randomly every time!
     //  (This isn't needed or possible for websockets, but they stay open, so calling functions
     //      after they open to set the nodeId is possible, and preferred).
-    let remoteAddress = request.socket.remoteAddress;
+    let remoteAddress = request.socket.remoteAddress?.split(":").pop();
     if (!remoteAddress) {
         throw new Error(`Missing remoteAddress`);
     }
@@ -147,15 +147,6 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
         }
 
         let headers = (resultBuffer as HTTPResultType)[resultHeaders];
-        if (SocketFunction.compression?.type === "gzip" && !headers?.["Content-Encoding"]) {
-            if (request.headers["accept-encoding"]?.includes("gzip")) {
-                resultBuffer = await new Promise<Buffer>((resolve, reject) =>
-                    gzip(resultBuffer, (err, result) => err ? reject(err) : resolve(result))
-                );
-                response.setHeader("Content-Encoding", "gzip");
-            }
-        }
-
 
         // NOTE: Our ETag caching is only to reduce data sent on the wire, we evaluate the calls
         //  every time (so it is strictly a wire cache for HTTP, not a computation cache)

package/src/callManager.ts

@@ -29,7 +29,7 @@ export async function performLocalCall(
     }
 
     if (!exposedClasses.has(call.classGuid)) {
-        throw new Error(`Class ${call.classGuid} not exposed`);
+        throw new Error(`Class ${call.classGuid} not exposed, exposed classes: ${Array.from(exposedClasses).join(", ")}`);
     }
 
     let controller = classDef.controller;
@@ -95,7 +95,7 @@ export function unregisterGlobalClientHook(hook: SocketFunctionClientHook) {
 
 export const runClientHooks = measureWrap(async function runClientHooks(
     callType: FullCallType,
-    hooks: SocketExposedShape[""],
+    hooks: Exclude<SocketExposedShape[""], undefined>,
     connectionId: { nodeId: string },
 ): Promise<ClientHookContext> {
     let context: ClientHookContext = { call: callType, connectionId };
@@ -122,7 +122,7 @@ export const runClientHooks = measureWrap(async function runClientHooks(
 export const runServerHooks = measureWrap(async function runServerHooks(
     callType: FullCallType,
     caller: CallerContext,
-    hooks: SocketExposedShape[""],
+    hooks: Exclude<SocketExposedShape[""], undefined>,
 ): Promise<HookContext> {
     let hookContext: HookContext = { call: callType };
     for (let hook of globalHooks.concat(hooks.hooks || [])) {

package/src/formatting/format.ts

@@ -134,7 +134,12 @@ export function formatNumber(count: number | undefined, maxAbsoluteValue?: numbe
     // NOTE: We don't switch units as soon as we possible can, because...
     //  3.594 vs 3.584 is harder to quickly distinguish compared to 3594 and 3584,
     //  the decimal simply makes it harder to read, and larger.
-    const extraFactor = 10;
+    // NOTE: This number should prevent us from ever using 5 digits, so that we never need commas
+    //  For example, if the factor is 10 then, 9999.5 is kept with a divisor of 1, and then rounds up to
+    //  "10,000". So we want any value which rounds up at 5 digits to be put in the next group (and having
+    //  extra values put in the next group is fine, as we won't show the decimal value anyways, so it only
+    //  means 9999 wraps around to 10K a bit faster).
+    const extraFactor = 9.99949999;
     let divisor = 1;
     let suffix = "";
     let currencyDecimalsNeeded = false;

package/src/misc.ts

@@ -1,6 +1,13 @@
 import * as crypto from "crypto";
 import { canHaveChildren, MaybePromise } from "./types";
 
+export const timeInSecond = 1000;
+export const timeInMinute = timeInSecond * 60;
+export const timeInHour = timeInMinute * 60;
+export const timeInDay = timeInHour * 24;
+export const timeInWeek = timeInDay * 7;
+export const timeInYear = timeInDay * 365;
+
 export type Watchable<T> = (callback: (value: T) => void) => MaybePromise<void>;
 
 export function convertErrorStackToError(error: string): Error {
@@ -301,4 +308,9 @@ export function arrayFromOrderObject<T>(obj: { [order: number]: T }): T[] {
 
 export function last<T>(arr: T[]): T | undefined {
     return arr[arr.length - 1];
+}
+
+export type ObjectValues<T> = T[keyof T];
+export function entries<Obj extends { [key: string]: unknown }>(obj: Obj): [keyof Obj, ObjectValues<Obj>][] {
+    return Object.entries(obj) as any;
 }

package/src/nodeCache.ts

@@ -2,6 +2,7 @@ import { CallFactory, createCallFactory } from "./CallFactory";
 import { MaybePromise } from "./types";
 import { lazy } from "./caching";
 import { SocketFunction } from "../SocketFunction";
+import { isNode } from "./misc";
 
 // TODO: Add CallInstanceFactory.isClosed, so nodeCache can clean up old entries.
 //  This is only needed for memory management, and not for correctness. Entries never
@@ -17,6 +18,11 @@ export function getNodeId(domain: string, port: number): string {
     return `${domain}:${port}`;
 }
 
+export function getNodeIdFromLocation() {
+    if (isNode()) throw new Error(`Cannot get nodeId from location, as we are running in NodeJS`);
+    return getNodeId(location.hostname, location.port ? parseInt(location.port) : 443);
+}
+
 /** A nodeId not available for reconnecting. */
 export function getClientNodeId(address: string): string {
     return `client:${address}:${Date.now()}:${Math.random()}`;

package/src/profiling/getOwnTime.ts

@@ -8,39 +8,18 @@ export type OwnTimeObj = {
     time: number;
     ownTime: number;
 };
-type OwnTimeObjInternal = OwnTimeObj & {
+export type OwnTimeObjInternal = OwnTimeObj & {
     lastStartTime: number;
     firstStartTime: number;
-    parent: OwnTimeObjInternal | undefined;
-    child: OwnTimeObjInternal | undefined;
 };
 
-let pendingCallTime: OwnTimeObjInternal | undefined;
-export function getPendingOwnTimeObjs(): (OwnTimeObj & { source: OwnTimeObjInternal })[] | undefined {
-    let time = now();
-    let instances = getPendingOwnTimeInstances();
-    if (!instances) return undefined;
-    if (!pendingCallTime) return undefined;
-    let results = instances.map((instance) => ({
-        name: instance.name,
-        ownTime: instance.ownTime,
-        time: time - instance.firstStartTime,
-        source: instance
-    }));
-    results[0].ownTime += time - pendingCallTime.lastStartTime;
-    return results;
-}
-export function getPendingOwnTimeInstances(): OwnTimeObjInternal[] | undefined {
-    if (!pendingCallTime) return undefined;
-    let results: OwnTimeObjInternal[] = [];
-    let current: OwnTimeObjInternal | undefined = pendingCallTime;
-    while (current) {
-        results.push(current);
-        current = current.parent;
-    }
-    return results;
+let openTimes: OwnTimeObjInternal[] = [];
+
+export function getOpenTimesBase(): OwnTimeObjInternal[] {
+    return openTimes;
 }
-(global as any).pendingOwnCallTime = pendingCallTime;
+
+(global as any).pendingOwnCallTime = openTimes;
 
 // NOTE: This overhead time is actually mostly for aggregate time, but it is needed,
 //  otherwise we consistently underestimate the time spent.
@@ -73,6 +52,7 @@ let addMeasureOverheadTime = 0;
 
 // TIMING: About 60ns, of which 40ns is just now() calls.
 //  If async is closer to 300ns.
+// NOTE: Handles promises correctly
 export function getOwnTime<T>(
     name: string,
     code: () => T,
@@ -85,39 +65,25 @@ export function getOwnTime<T>(
         ownTime: 0,
         firstStartTime: time,
         lastStartTime: time,
-        parent: pendingCallTime,
-        child: undefined,
     };
-    if (pendingCallTime) {
-        pendingCallTime.child = obj;
-    }
-    pendingCallTime = obj;
-    if (obj.parent) {
-        obj.parent.ownTime += obj.lastStartTime - obj.parent.lastStartTime;
+    let prevOwnTime = openTimes[openTimes.length - 1];
+    if (prevOwnTime) {
+        prevOwnTime.ownTime += time - prevOwnTime.lastStartTime;
     }
+    openTimes.push(obj);
 
     function finish() {
         let time = now();
         obj.time = time - obj.firstStartTime;
-        if (pendingCallTime === obj) {
-            // Good case, all of our children call ended before us.
-
-            // End our own time calculation
+        if (obj === openTimes[openTimes.length - 1]) {
             obj.ownTime += time - obj.lastStartTime;
-
-            // Our parent is now the last open call
-            pendingCallTime = obj.parent;
-            if (pendingCallTime) {
-                // Resume our parent ownTime counting
-                pendingCallTime.lastStartTime = time;
+            let newOwnTime = openTimes[openTimes.length - 2];
+            if (newOwnTime) {
+                newOwnTime.lastStartTime = time;
             }
         }
-        if (obj.child && obj.parent) {
-            obj.child.parent = obj.parent;
-            obj.parent.child = obj.child;
-        }
-        obj.parent = undefined;
-        obj.child = undefined;
+        let index = openTimes.indexOf(obj);
+        openTimes.splice(index, 1);
 
         obj.time += addMeasureOverheadTime;
         obj.ownTime += addMeasureOverheadTime;

package/src/profiling/measure.ts

@@ -2,12 +2,16 @@ import debugbreak from "debugbreak";
 import { formatTime, formatNumber } from "../formatting/format";
 import { red, yellow, blue, magenta } from "../formatting/logColors";
 
-import { getOwnTime, getPendingOwnTimeInstances, getPendingOwnTimeObjs, OwnTimeObj } from "./getOwnTime";
+import { getOpenTimesBase, getOwnTime, OwnTimeObj } from "./getOwnTime";
 import { addToStats, addToStatsValue, createStatsValue, getStatsTop, StatsValue } from "./stats";
 import { white } from "../formatting/logColors";
 import { isNode } from "../misc";
+import { formatStats } from "./statsFormat";
 
-/** NOTE: Must be called BEFORE anything else is imported! */
+let measurementsDisabled = false;
+/** NOTE: Must be called BEFORE anything else is imported!
+ *      NOTE: Measurements on on by default now, so this doesn't really need to be called...
+*/
 export function enableMeasurements() {
     if (functionsSkipped) {
         console.warn(red(`Skipped measure shimming ${functionsSkipped} functions. Fix this by calling enableMeasurements before any other imports.`));
@@ -17,6 +21,7 @@ export function enableMeasurements() {
 /** NOTE: Must be called BEFORE anything else is imported! */
 export function disableMeasurements() {
     measurementsEnabled = false;
+    measurementsDisabled = true;
 }
 
 let functionsSkipped = 0;
@@ -28,6 +33,7 @@ const AsyncFunction = (async () => { }).constructor;
 // TIMING: 1-5us. I have seen timing values greatly vary, but it does seem to be quite high, despite
 //  microbenchmarks saying it is slow. Perhaps it is because getOwnTime breaks the cpu pipeline,
 //  which causes slowness for code around us, but not if we are running in isolation?
+// NOTE: Handles promises correctly
 export function measureFnc(target: any, propertyKey: string, descriptor: PropertyDescriptor) {
     let name = propertyKey;
     if (target.name) {
@@ -49,6 +55,7 @@ export function nameFunction<T extends Function>(name: string, fnc: T) {
     Object.defineProperty(fnc, "name", { value: name });
     return fnc;
 }
+// NOTE: Handles promises correctly
 export function measureWrap<T extends (...args: any[]) => any>(fnc: T, name?: string): T {
     if (!measurementsEnabled) {
         functionsSkipped++;
@@ -69,21 +76,31 @@ export function measureBlock<T extends (...args: any[]) => any>(fnc: T, name?: s
 export function startMeasure(): {
     finish: () => MeasureProfile;
 } {
-    if (!measurementsEnabled) {
+    if (!measurementsEnabled && !measurementsDisabled) {
         console.warn(red(`To capture measurements enableMeasurements() must be called before any other imports in your entry point`));
     }
     let profile: MeasureProfile = {
         entries: Object.create(null),
     };
-    let openAtStart = new Set(getPendingOwnTimeInstances());
+    let openAtStart = new Set(getOpenTimesBase());
 
     outstandingProfiles.push(profile);
     return {
         finish() {
-            let pending = getPendingOwnTimeObjs() || [];
+            let pending = getOpenTimesBase();
+            let last = pending[pending.length - 1];
+            let time = Date.now();
             for (let timeObj of pending) {
-                if (openAtStart.has(timeObj.source)) continue;
-                addToProfile(profile, timeObj, true);
+                // Ignore any values that were already open, as they are clearly not
+                //  caused by our code.
+                if (openAtStart.has(timeObj)) continue;
+                timeObj = { ...timeObj };
+
+                if (timeObj === last) {
+                    timeObj.ownTime += time - timeObj.lastStartTime;
+                }
+                timeObj.time = time - timeObj.firstStartTime;
+                addToProfile(profile, timeObj);
             }
             outstandingProfiles.splice(outstandingProfiles.indexOf(profile), 1);
             return profile;
@@ -96,6 +113,8 @@ export interface LogMeasureTableConfig {
     name?: string;
     // Defaults to 0.05
     thresholdInTable?: number;
+    // Details to 50
+    minTimeToLog?: number;
 }
 
 export function logMeasureTable(
@@ -106,6 +125,7 @@ export function logMeasureTable(
     if (thresholdInTable === undefined) {
         thresholdInTable = 0.05;
     }
+    let minTimeToLog = config?.minTimeToLog ?? 50;
 
     function getTime(entry: ProfileEntry) {
         return useTotalTime ? entry.totalTime : entry.ownTime;
@@ -114,6 +134,7 @@ export function logMeasureTable(
     entries.sort((a, b) => getTime(b).sum - getTime(a).sum);
 
     let totalTime = entries.map(x => getTime(x).sum).reduce((a, b) => a + b, 0);
+    if (totalTime < minTimeToLog) return;
 
     console.log();
     let title = yellow(`Profiled ${formatTime(totalTime)} (logged at ${new Date().toISOString()})`);
@@ -138,27 +159,8 @@ export function logMeasureTable(
             return String(text).padStart(count, " ");
         }
         let fractionText = percent(getTime(entry).sum / totalTime);
-        let perText = formatTime(getTime(entry).sum / getTime(entry).count);
-        let countText = formatNumber(getTime(entry).count);
-        let sumText = formatTime(getTime(entry).sum);
-
-        let equation = `${p(6, sumText)} =  ${p(6, countText)} * ${p(6, perText)}`;
-
-        let ownTimeTop = getStatsTop(getTime(entry));
-        if (ownTimeTop.topHeavy) {
-            let topText = formatTime(ownTimeTop.value / ownTimeTop.count);
-            let topCountText = formatNumber(ownTimeTop.count);
-            let bottomText = formatTime((getTime(entry).sum - ownTimeTop.value) / getTime(entry).count, ownTimeTop.value);
-            let bottomCountText = formatNumber(getTime(entry).count - ownTimeTop.count);
-            let topPart = `${p(6, topText)} per * ${topCountText}`;
-            let bottomPart = `${bottomText} * ${bottomCountText}`;
-            if (isNode()) {
-                topPart = red(topPart);
-            } else {
-                bottomPart = white(bottomPart);
-            }
-            equation = `${p(6, sumText)} = ${p(6, topPart)}   +  ${bottomPart}`;
-        }
+
+        let equation = formatStats(getTime(entry));
 
         let text = `${p(6, fractionText)} ( ${equation} )`;
         let overhead = measureOverhead * getTime(entry).count;
@@ -188,7 +190,7 @@ export async function measureCode<T>(code: () => Promise<T>, config?: LogMeasure
     try {
         return await measureBlock(code, code.name || "untracked");
     } finally {
-        finishProfile(measure, config);
+        finishProfile(measure, config || { name: code.name, minTimeToLog: 0 });
     }
 }
 export function measureCodeSync<T>(code: () => T, config?: LogMeasureTableConfig): T {

package/src/profiling/statsFormat.ts

@@ -0,0 +1,41 @@
+import { formatTime, formatNumber } from "../formatting/format";
+import { red, white } from "../formatting/logColors";
+import { isNode } from "../misc";
+import { StatsValue, getStatsTop } from "./stats";
+
+export function percent(value: number) {
+    return `${(value * 100).toFixed(2)}%`;
+}
+
+export function formatStats(stats: StatsValue, config?: {
+    noColor?: boolean;
+    noSum?: boolean;
+}) {
+    function p(count: number, text: string | number) {
+        return String(text).padStart(count, " ");
+    }
+
+    let perText = formatTime(stats.sum / stats.count);
+    let countText = formatNumber(stats.count);
+    let sumText = formatTime(stats.sum);
+    let equation = (!config?.noSum && `${p(6, sumText)} =  ` || "") + `${p(6, countText)} * ${p(6, perText)}`;
+
+    let top = getStatsTop(stats);
+    if (top.topHeavy) {
+        let topText = formatTime(top.value / top.count);
+        let topCountText = formatNumber(top.count);
+        let bottomText = formatTime((stats.sum - top.value) / (stats.count - top.count) || 0);
+        let bottomCountText = formatNumber(stats.count - top.count);
+        let topPart = `${topCountText} * ${p(6, topText)}`;
+        let bottomPart = `${bottomCountText} * ${bottomText}`;
+        if (!config?.noColor) {
+            if (isNode()) {
+                topPart = red(topPart);
+            } else {
+                bottomPart = white(bottomPart);
+            }
+        }
+        equation = (!config?.noSum && `${p(6, sumText)} = ` || "") + `${p(6, topPart)}   +  ${bottomPart}`;
+    }
+    return equation;
+}

package/src/webSocketServer.ts

@@ -96,7 +96,11 @@ export async function startSocketServer(
             console.error(`Connection attempt error ${e.message}`);
         });
         httpsServer.on("tlsClientError", e => {
-            console.error(`TLS client error ${e.message}`);
+            // NOTE: This happens a lot when we have tabs open that connected to an old
+            //  server (with old certs, that the browser will reject?)
+            if (!SocketFunction.silent) {
+                console.error(`TLS client error ${e.message}`);
+            }
         });
 
         httpsServer.on("request", httpCallHandler);
@@ -235,9 +239,7 @@ export async function startSocketServer(
 
     port = (realServer.address() as net.AddressInfo).port;
     let nodeId = getNodeId(getCommonName(config.cert), port);
-    if (!SocketFunction.silent) {
-        console.log(green(`Started Listening on ${nodeId}`));
-    }
+    console.log(green(`Started Listening on ${nodeId}`));
 
     return nodeId;
 }