socket-function 0.10.4 → 0.10.6

package/SocketFunction.ts

@@ -1,6 +1,6 @@
 /// <reference path="./require/RequireController.ts" />
 
-import { SocketExposedInterface, SocketFunctionHook, SocketFunctionClientHook, SocketExposedShape, SocketRegistered, CallerContext, FullCallType } from "./SocketFunctionTypes";
+import { SocketExposedInterface, SocketFunctionHook, SocketFunctionClientHook, SocketExposedShape, SocketRegistered, CallerContext, FullCallType, CallType } from "./SocketFunctionTypes";
 import { exposeClass, registerClass, registerGlobalClientHook, registerGlobalHook, runClientHooks } from "./src/callManager";
 import { SocketServerConfig, startSocketServer } from "./src/webSocketServer";
 import { getCallFactory, getCreateCallFactory, getNodeId, getNodeIdLocation } from "./src/nodeCache";
@@ -32,12 +32,18 @@ type ExtractShape<ClassType, Shape> = {
 
 export class SocketFunction {
     public static logMessages = false;
+    public static trackMessageSizes = {
+        upload: [] as ((size: number) => void)[],
+        download: [] as ((size: number) => void)[],
+    };
 
     public static MAX_MESSAGE_SIZE = 1024 * 1024 * 32;
 
-    public static httpETagCache = false;
+    public static HTTP_ETAG_CACHE = false;
     public static silent = true;
 
+    public static HTTP_COMPRESS = false;
+
     // In retrospect... dynamically changing the wire serializer is a BAD idea. If any calls happen
     //  before it is changed, things just break. Also, it needs to be changed on both sides,
     //  or else things break. Also, it is very hard to detect when the issue is different serializers

package/SocketFunctionTypes.ts

@@ -24,6 +24,7 @@ export type SocketExposedInterfaceClass = {
 };
 export type SocketExposedShape<ExposedType extends SocketExposedInterface = SocketExposedInterface> = {
     [functionName in keyof ExposedType]?: {
+        compress?: boolean;
         /** Indicates with the same input, we give the same output, forever,
          *      independent of code changes. This only works for data storage.
          */

package/package.json

@@ -1,17 +1,19 @@
 {
 	"name": "socket-function",
-	"version": "0.10.4",
+	"version": "0.10.6",
 	"main": "index.js",
 	"license": "MIT",
 	"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
 	"dependencies": {
+		"@types/pako": "^2.0.3",
 		"@types/ws": "^8.5.3",
 		"cbor-x": "^1.5.6",
 		"cookie": "^0.5.0",
 		"mobx": "^6.6.2",
 		"node-forge": "https://github.com/sliftist/forge#name",
+		"pako": "^2.1.0",
 		"preact": "^10.10.6",
-		"typenode": "^5.3.8",
+		"typenode": "^5.3.10",
 		"ws": "^8.8.0"
 	},
 	"optionalDependencies": {

package/require/RequireController.ts

@@ -2,8 +2,13 @@
 import debugbreak from "debugbreak";
 import fs from "fs";
 import { SocketFunction } from "../SocketFunction";
-import { setHTTPResultHeaders } from "../src/callHTTPHandler";
-import { isNodeTrue } from "../src/misc";
+import { getCurrentHTTPRequest, setHTTPResultHeaders } from "../src/callHTTPHandler";
+import { formatNumberSuffixed, isNodeTrue, sha256Hash, sha256HashPromise } from "../src/misc";
+import zlib from "zlib";
+import { cacheLimited } from "../src/caching";
+import { formatNumber } from "../src/formatting/format";
+
+const COMPRESS_CACHE_SIZE = 1024 * 1024 * 128;
 
 module.allowclient = true;
 
@@ -55,6 +60,9 @@ export interface SerializedModule {
     source?: string;
 
     seqNum: number;
+
+    size?: number;
+    version?: number;
 }
 
 let nextModuleSeqNum = 1;
@@ -128,6 +136,8 @@ class RequireControllerBase {
         };
         requireSeqNumProcessId: string;
     }> {
+        let httpRequest = getCurrentHTTPRequest();
+
         let seqNums: { [seqNum: number]: 1 } = {};
         if (alreadyHave?.requireSeqNumProcessId === requireSeqNumProcessId) {
             for (let { s, e } of alreadyHave.seqNumRanges) {
@@ -168,6 +178,8 @@ class RequireControllerBase {
                 serveronly: module.serveronly,
                 requests: Object.create(null),
                 seqNum: module.requireControllerSeqNum,
+                size: module.size,
+                version: module.version,
             };
             let moduleObj = modules[module.filename];
             if (moduleObj.allowclient) {
@@ -256,10 +268,53 @@ class RequireControllerBase {
         for (let remap of mapGetModules) {
             result = await remap.remap(result, [pathRequests, alreadyHave, config]);
         }
+
+        // NOTE: Handling compression ourself allows us to efficiently cache (otherwise caching would require
+        //      hashing the output, which takes almost as long as compression!)
+        if (httpRequest && SocketFunction.HTTP_COMPRESS && httpRequest.headers["accept-encoding"]?.includes("gzip")) {
+            let simplifiedResult = {
+                ...result,
+                modules: Object.entries(result.modules).map(x => [x[0], {
+                    filename: x[1].filename,
+                    version: x[1].version,
+                    sourceLength: x[1].source?.length,
+                }]),
+            };
+            let key = sha256Hash(JSON.stringify(simplifiedResult));
+            let buffer = await compressCached(key, () => Buffer.from(JSON.stringify(result)));
+            setHTTPResultHeaders(buffer, { "Content-Type": "application/json", "Content-Encoding": "gzip" });
+            return buffer as any;
+        }
+
         return result;
     }
 }
 
+let compressCacheSize = 0;
+let compressCache = new Map<string, Buffer>();
+async function compressCached(bufferKey: string, buffer: () => Buffer): Promise<Buffer> {
+    let cached = compressCache.get(bufferKey);
+    if (!cached) {
+        cached = await new Promise<Buffer>((resolve, reject) => {
+            zlib.gzip(buffer(), {}, (err, result) => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve(result);
+                }
+            });
+        });
+        compressCacheSize += cached.length;
+        // TODO: Make the cache LRU eviction, instead of just resetting it
+        if (compressCacheSize > COMPRESS_CACHE_SIZE) {
+            compressCache.clear();
+            compressCacheSize = cached.length;
+        }
+        compressCache.set(bufferKey, cached);
+    }
+    return cached;
+}
+
 type ClientRemapCallback = (args: GetModulesArgs) => Promise<GetModulesArgs>;
 declare global {
     /** Must be set clientside BEFORE requests are made (so you likely want to use RequireController.addMapGetModules

package/require/require.js

@@ -416,6 +416,7 @@
             let time = Date.now();
             currentModuleEvaluationStack.push(module.filename);
             try {
+                module.isPreloading = true;
                 moduleFnc.call(
                     {
                         // NOTE: Adding __importStar to the module causes typescript to use our implementation,
@@ -440,6 +441,7 @@
                 //  As in, adding about 500ms to our load time, which is annoying when debugging.
                 //console.debug(`Evaluated module ${module.filename} ${Math.ceil(source.length / 1024)}KB`);
             } finally {
+                module.isPreloading = false;
                 currentModuleEvaluationStack.pop();
             }
 

package/src/CallFactory.ts

@@ -1,10 +1,9 @@
 import { CallerContext, CallerContextBase, CallType, FullCallType } from "../SocketFunctionTypes";
 import * as ws from "ws";
-import { performLocalCall } from "./callManager";
+import { performLocalCall, shouldCompressCall } from "./callManager";
 import { convertErrorStackToError, formatNumberSuffixed, isNode, list } from "./misc";
 import { createWebsocketFactory, getTLSSocket } from "./websocketFactory";
 import { SocketFunction } from "../SocketFunction";
-import { gzip } from "zlib";
 import * as tls from "tls";
 import { getClientNodeId, getNodeIdLocation, registerNodeClient } from "./nodeCache";
 import debugbreak from "debugbreak";
@@ -13,12 +12,16 @@ import { red, yellow } from "./formatting/logColors";
 import { isSplitableArray, markArrayAsSplitable } from "./fixLargeNetworkCalls";
 import { delay, runInSerial } from "./batching";
 import { formatNumber, formatTime } from "./formatting/format";
+import pako from "pako";
+import { setFlag } from "../require/compileFlags";
+setFlag(require, "pako", "allowclient", true);
 
 const MIN_RETRY_DELAY = 1000;
 
 type InternalCallType = FullCallType & {
     seqNum: number;
     isReturn: false;
+    isArgsCompressed?: boolean;
 }
 
 type InternalReturnType = {
@@ -27,7 +30,7 @@ type InternalReturnType = {
     error?: string;
     seqNum: number;
     resultSize: number;
-    compressed: boolean;
+    isResultCompressed?: boolean;
 };
 
 
@@ -123,6 +126,11 @@ export async function createCallFactory(
                 functionName: call.functionName,
                 seqNum,
             };
+            let originalArgs = call.args;
+            if (shouldCompressCall(fullCall)) {
+                fullCall.args = await compressObj(fullCall.args) as any;
+                fullCall.isArgsCompressed = true;
+            }
             let time = Date.now();
             let data: Buffer[];
             let dataMaybePromise = SocketFunction.WIRE_SERIALIZER.serialize(fullCall);
@@ -138,11 +146,11 @@ export async function createCallFactory(
             }
 
             if (size > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
-                let splitArgIndex = call.args.findIndex(isSplitableArray);
+                let splitArgIndex = originalArgs.findIndex(isSplitableArray);
                 if (splitArgIndex >= 0) {
                     console.log(yellow(`Splitting large call due to large args: ${call.classGuid}.${call.functionName}`));
                     let SPLIT_GROUPS = 10;
-                    let splitArg = call.args[splitArgIndex] as unknown[];
+                    let splitArg = originalArgs[splitArgIndex] as unknown[];
                     let subCalls = list(SPLIT_GROUPS).map(index => {
                         let start = Math.floor(index / SPLIT_GROUPS * splitArg.length);
                         let end = Math.floor((index + 1) / SPLIT_GROUPS * splitArg.length);
@@ -167,9 +175,6 @@ export async function createCallFactory(
 
             let resultPromise = new Promise((resolve, reject) => {
                 let callback = (result: InternalReturnType) => {
-                    if (SocketFunction.logMessages) {
-                        console.log(`SIZE\t${(formatNumberSuffixed(result.resultSize) + "B").padEnd(4, " ")}\t${call.classGuid}.${call.functionName} at ${Date.now()}`);
-                    }
                     pendingCalls.delete(seqNum);
                     if (result.error) {
                         reject(convertErrorStackToError(result.error));
@@ -180,6 +185,20 @@ export async function createCallFactory(
                 pendingCalls.set(seqNum, { callback, data, call: fullCall });
             });
 
+            {
+                let resultSize = data.map(x => x.length).reduce((a, b) => a + b, 0);
+                for (let callback of SocketFunction.trackMessageSizes.upload) {
+                    callback(resultSize);
+                }
+                if (SocketFunction.logMessages) {
+                    let fncHack = "";
+                    if (call.functionName === "addCall") {
+                        let arg = originalArgs[0] as any;
+                        fncHack = `.${arg.DomainName}.${arg.ModuleId}.${arg.FunctionId}`;
+                    }
+                    console.log(`SIZE\t${(formatNumberSuffixed(resultSize) + "B").padEnd(4, " ")}\t${call.classGuid}.${call.functionName}${fncHack} at ${Date.now()}`);
+                }
+            }
             await send(data);
 
             return await resultPromise;
@@ -210,7 +229,6 @@ export async function createCallFactory(
                     error: error,
                     seqNum: call.call.seqNum,
                     resultSize: 0,
-                    compressed: false,
                 });
             }
 
@@ -337,6 +355,9 @@ export async function createCallFactory(
                 let time = Date.now();
                 let call = await SocketFunction.WIRE_SERIALIZER.deserialize(currentBuffers) as InternalCallType | InternalReturnType;
                 time = Date.now() - time;
+                for (let callback of SocketFunction.trackMessageSizes.download) {
+                    callback(resultSize);
+                }
 
                 if (call.isReturn) {
                     let callbackObj = pendingCalls.get(call.seqNum);
@@ -347,9 +368,24 @@ export async function createCallFactory(
                         console.log(`Got return for unknown call ${call.seqNum}`);
                         return;
                     }
+                    if (SocketFunction.logMessages) {
+                        let call = callbackObj.call;
+                        console.log(`SIZE\t${(formatNumberSuffixed(resultSize) + "B").padEnd(4, " ")}\t${call.classGuid}.${call.functionName} at ${Date.now()}`);
+                    }
+                    if (call.isResultCompressed) {
+                        call.result = await decompressObj(call.result as Buffer);
+                        call.isResultCompressed = false;
+                    }
                     call.resultSize = resultSize;
                     callbackObj.callback(call);
                 } else {
+                    if (call.isArgsCompressed) {
+                        call.args = await decompressObj(call.args as any as Buffer) as any;
+                        call.isArgsCompressed = false;
+                    }
+                    if (SocketFunction.logMessages) {
+                        console.log(`SIZE\t${(formatNumberSuffixed(resultSize) + "B").padEnd(4, " ")}\t${call.classGuid}.${call.functionName} at ${Date.now()}`);
+                    }
                     if (time > SocketFunction.WIRE_WARN_TIME) {
                         console.log(red(`Slow parse, took ${time}ms to parse ${resultSize} bytes, for call to ${call.classGuid}.${call.functionName}`));
                     }
@@ -362,8 +398,11 @@ export async function createCallFactory(
                             result,
                             seqNum: call.seqNum,
                             resultSize: resultSize,
-                            compressed: false,
                         };
+                        if (shouldCompressCall(call)) {
+                            response.result = await compressObj(response.result) as any;
+                            response.isResultCompressed = true;
+                        }
                     } catch (e: any) {
                         response = {
                             isReturn: true,
@@ -371,7 +410,6 @@ export async function createCallFactory(
                             seqNum: call.seqNum,
                             error: e.stack,
                             resultSize: resultSize,
-                            compressed: false,
                         };
                     }
 
@@ -384,7 +422,6 @@ export async function createCallFactory(
                             seqNum: call.seqNum,
                             error: new Error(`Response too large to send (${call.classGuid}.${call.functionName}, size: ${formatNumber(totalResultSize)} > ${formatNumber(SocketFunction.MAX_MESSAGE_SIZE)}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`).stack,
                             resultSize: resultSize,
-                            compressed: false,
                         };
                         result = await SocketFunction.WIRE_SERIALIZER.serialize(response);
                     }
@@ -394,15 +431,47 @@ export async function createCallFactory(
             }
             throw new Error(`Unhandled data type ${typeof message}`);
         } catch (e: any) {
+            let message = e.stack || e.message || e;
             // NOTE: I'm looking for all types of errors here (specifically, .send errors), in case
             //  there are errors I should be handling.
-            if (e.stack.startsWith("Error: Cannot send data to") && e.stack.includes("as the connection has closed")) {
+            if (message.startsWith("Error: Cannot send data to") && message.includes("as the connection has closed")) {
                 // This is fine, just ignore it
             } else {
+                debugbreak(2);
+                debugger;
                 console.error(e.stack);
             }
         }
     }
 
     return callFactory;
+}
+
+
+async function compressObj(obj: unknown): Promise<Buffer> {
+    let buffers = await SocketFunction.WIRE_SERIALIZER.serialize(obj);
+    let lengthBuffer = Buffer.from((new Float64Array(buffers.map(x => x.length))).buffer);
+    let buffer = Buffer.concat([lengthBuffer, ...buffers]);
+    return Buffer.from(pako.gzip(buffer));
+}
+async function decompressObj(obj: Buffer): Promise<unknown> {
+    try {
+        let buffer = Buffer.from(pako.ungzip(obj));
+        let lengthBuffer = buffer.slice(0, 8);
+        let lengths = new Float64Array(lengthBuffer.buffer, lengthBuffer.byteOffset, lengthBuffer.byteLength / 8);
+        let buffers: Buffer[] = [];
+        let offset = 8;
+        for (let length of lengths) {
+            buffers.push(buffer.slice(offset, offset + length));
+            offset += length;
+        }
+
+        return await SocketFunction.WIRE_SERIALIZER.deserialize(buffers);
+    } catch (e) {
+        // We were encountering issues with the checksum failing when unzipping. Presumably if the data
+        //      is bad deserialize will also fail. I can't repro it anymore though...
+        debugbreak(2);
+        debugger;
+        throw e;
+    }
 }

package/src/callHTTPHandler.ts

@@ -4,6 +4,7 @@ import { CallerContext, CallType, FullCallType } from "../SocketFunctionTypes";
 import { isDataImmutable, performLocalCall } from "./callManager";
 import { SocketFunction } from "../SocketFunction";
 import { gzip } from "zlib";
+import zlib from "zlib";
 import { formatNumberSuffixed, sha256Hash } from "./misc";
 import { getClientNodeId, getNodeId } from "./nodeCache";
 
@@ -48,6 +49,11 @@ export function getNodeIdsFromRequest(request: http.IncomingMessage) {
     return { nodeId, localNodeId };
 }
 
+let requests = new Map<CallerContext, http.IncomingMessage>();
+export function getCurrentHTTPRequest(): http.IncomingMessage | undefined {
+    return requests.get(SocketFunction.getCaller());
+}
+
 export async function httpCallHandler(request: http.IncomingMessage, response: http.ServerResponse) {
     try {
         // Always set x-frame-options, to prevent iframe embedding click hijacking
@@ -65,7 +71,9 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
         let protocol = "https";
         let url = protocol + "://" + request.headers.host + request.url;
 
-        console.log(`HTTP request (${request.method}) ${url}`);
+        if (SocketFunction.logMessages) {
+            console.log(`HTTP request (${request.method}) ${url}`);
+        }
         let urlObj = new URL(url);
 
         let payload = await new Promise<Buffer>((resolve, reject) => {
@@ -136,10 +144,16 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
             }
         }
 
-        let result = await performLocalCall({
-            caller,
-            call
-        });
+        requests.set(caller, request);
+        let result: unknown;
+        try {
+            result = await performLocalCall({
+                caller,
+                call
+            });
+        } finally {
+            requests.delete(caller);
+        }
 
         let resultBuffer: Buffer;
         if (typeof result === "object" && result && result instanceof Buffer) {
@@ -152,7 +166,7 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
 
         // NOTE: Our ETag caching is only to reduce data sent on the wire, we evaluate the calls
         //  every time (so it is strictly a wire cache for HTTP, not a computation cache)
-        if (SocketFunction.httpETagCache) {
+        if (SocketFunction.HTTP_ETAG_CACHE) {
             response.setHeader("cache-control", "private, max-age=0, must-revalidate");
             let hash = sha256Hash(resultBuffer);
             response.setHeader("ETag", hash);
@@ -173,8 +187,26 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
                 return;
             }
         }
+        if (SocketFunction.HTTP_COMPRESS && request.headers["accept-encoding"]?.includes("gzip") && !headers?.["Content-Encoding"]) {
+            // NOTE: This is a BIT slow. To speed it up, functions can use an internal cache, according to their function,
+            //  and return a Buffer (which they can as any cast to make the returned type allowed, as returned Buffers will
+            //  just be treated like a buffer of JSON data).
+            //  - The caller should use getCurrentHTTPRequest first though, to check if gzip is allowed
+            response.setHeader("Content-Encoding", "gzip");
+            resultBuffer = await new Promise<Buffer>((resolve, reject) => {
+                zlib.gzip(resultBuffer, {}, (err, result) => {
+                    if (err) {
+                        reject(err);
+                    } else {
+                        resolve(result);
+                    }
+                });
+            });
+        }
         response.write(resultBuffer);
-        console.log(`HTTP response  ${formatNumberSuffixed(resultBuffer.length)}B  (${request.method}) ${url}`);
+        if (SocketFunction.logMessages) {
+            console.log(`HTTP response  ${formatNumberSuffixed(resultBuffer.length)}B  (${request.method}) ${url}`);
+        }
 
     } catch (e: any) {
         console.log(`HTTP error  (${request.method}) ${e.stack}`);

package/src/callManager.ts

@@ -15,6 +15,10 @@ let exposedClasses = new Set<string>();
 let globalHooks: SocketFunctionHook[] = [];
 let globalClientHooks: SocketFunctionClientHook[] = [];
 
+export function shouldCompressCall(call: CallType) {
+    return !!classes[call.classGuid]?.shape[call.functionName]?.compress;
+}
+
 export async function performLocalCall(
     config: {
         call: FullCallType;

package/src/formatting/logColors.ts

@@ -5,14 +5,16 @@ function ansiHSL(h: number, s: number, l: number, text: string): string {
     return ansiRGB(r, g, b, text);
 }
 function ansiRGB(r: number, g: number, b: number, text: string): string {
-    return `\x1b[38;2;${r};${g};${b}m${text}\x1b[39m`;
+    return `\x1b[38;2;${r};${g};${b}m${text}\x1b[0m`;
 }
 
 const lightness = 68;
-export const blue = ansiHSL.bind(null, 235, 100, lightness);
+export const blue = (text: string) => `\x1b[34m${text}\x1b[0m`;
 export const red = ansiHSL.bind(null, 0, 100, lightness);
-export const green = (text: string) => `\x1b[32m${text}\x1b[39m`; // chalk.hsl(120, 100, lightness).bind(chalk);
-export const yellow = (text: string) => `\x1b[33m${text}\x1b[39m`;
+export const green = (text: string) => `\x1b[32m${text}\x1b[0m`;
+export const yellow = (text: string) => `\x1b[33m${text}\x1b[0m`;
 export const white = ansiHSL.bind(null, 0, 0, 80);
 
-export const magenta = (text: string) => `\x1b[35m${text}\x1b[39m`;
+export const magenta = (text: string) => `\x1b[35m${text}\x1b[0m`;
+
+console.log(blue('blue'));

package/src/misc.ts

@@ -1,5 +1,6 @@
 import * as crypto from "crypto";
 import { canHaveChildren, MaybePromise } from "./types";
+import { formatNumber } from "./formatting/format";
 
 export const timeInSecond = 1000;
 export const timeInMinute = timeInSecond * 60;
@@ -58,32 +59,7 @@ export function isNodeTrue() {
 }
 
 export function formatNumberSuffixed(count: number): string {
-    if (typeof count !== "number") return "0";
-    if (count < 0) {
-        return "-" + formatNumberSuffixed(-count);
-    }
-
-    let absValue = Math.abs(count);
-
-    const extraFactor = 10;
-    let divisor = 1;
-    let suffix = "";
-    if (absValue < 1000 * extraFactor) {
-
-    } else if (absValue < 1000 * 1000 * extraFactor) {
-        suffix = "K";
-        divisor = 1000;
-    } else if (absValue < 1000 * 1000 * 1000 * extraFactor) {
-        suffix = "M";
-        divisor = 1000 * 1000;
-    } else {
-        suffix = "B";
-        divisor = 1000 * 1000 * 1000;
-    }
-    count /= divisor;
-    absValue /= divisor;
-
-    return Math.round(count).toString() + suffix;
+    return formatNumber(count);
 }
 
 export function list(count: number) {