socket-function 0.8.38 → 0.8.39

package/SocketFunction.ts

@@ -9,6 +9,7 @@ import { Args, MaybePromise } from "./src/types";
 import { setDefaultHTTPCall } from "./src/callHTTPHandler";
 import debugbreak from "debugbreak";
 import { lazy } from "./src/caching";
+import { delay } from "./src/batching";
 
 module.allowclient = true;
 
@@ -76,7 +77,7 @@ export class SocketFunction {
             return shape as any as SocketExposedShape;
         });
 
-        setImmediate(() => {
+        void Promise.resolve().then(() => {
             registerClass(classGuid, instance as SocketExposedInterface, getShape());
         });
 
@@ -115,7 +116,7 @@ export class SocketFunction {
             _classGuid: classGuid,
         };
 
-        setImmediate(() => {
+        void Promise.resolve().then(() => {
             let onMount = getDefaultHooks?.().onMount;
             if (onMount) {
                 let callbacks = SocketFunction.onMountCallbacks.get(classGuid);
@@ -209,6 +210,10 @@ export class SocketFunction {
         if (config.ip) {
             this.mountedIP = config.ip;
         }
+
+        // Wait for any additionals functions to expose themselves
+        await delay("immediate");
+
         this.mountedNodeId = await startSocketServer(config);
         this.hasMounted = true;
         for (let classGuid of SocketFunction.exposedClasses) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "socket-function",
-	"version": "0.8.38",
+	"version": "0.8.39",
 	"main": "index.js",
 	"license": "MIT",
 	"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -10,7 +10,7 @@
 		"node-forge": "https://github.com/sliftist/forge#name",
 		"preact": "^10.10.6",
 		"rdtsc-now": "^0.3.0",
-		"typenode": "^4.9.4-b",
+		"typenode": "^4.9.4-c",
 		"ws": "^8.8.0"
 	},
 	"scripts": {

package/require/RequireController.ts

@@ -13,6 +13,11 @@ declare global {
             /** Indicates the module is allowed clientside. */
             allowclient?: boolean;
 
+            /** Causes the module to not preload, requiring `await import()` for it to load correctly
+             *      - Shouldn't be set recursively, otherwise nested packages will break.
+             */
+            lazyload?: boolean;
+
             /** Indicates the module is definitely not allowed clientside */
             serveronly?: boolean;
 
@@ -118,7 +123,8 @@ class RequireControllerBase {
         let modules: {
             [resolvedPath: string]: SerializedModule;
         } = Object.create(null);
-        function addModule(module: NodeJS.Module) {
+        function addModule(module: NodeJS.Module, rootImport = false) {
+            if (!rootImport && module.lazyload) return;
             if (!module.requireControllerSeqNum) {
                 module.requireControllerSeqNum = nextModuleSeqNum++;
             }
@@ -223,7 +229,7 @@ class RequireControllerBase {
                 clientModule = createNotFoundModule(`Module ${pathRequest} (resolved to ${resolvedPath}) is not allowed clientside (set module.allowclient in it, or call setFlag when it is imported).`);
             }
 
-            addModule(clientModule);
+            addModule(clientModule, true);
         }
 
         return { requestsResolvedPaths, modules, requireSeqNumProcessId };

package/require/require.js

@@ -242,6 +242,15 @@
             }
 
             let resolvedPath = serializedModule.requests[request];
+            if (resolvedPath !== "NOTALLOWEDCLIENTSIDE" && !serializedModules[resolvedPath]) {
+                if (!asyncIsFine) {
+                    console.warn(`Accessed unexpected module %c${request}%c in %c${module.id}%c\n\tTreating it as an async require.\n\tAll modules require synchronously clientside must be required serverside at a module level.`,
+                        "color: red", "color: unset",
+                        "color: red", "color: unset",
+                    );
+                }
+                return rootRequire(resolvedPath);
+            }
 
             let exportsOverride = undefined;
             if (resolvedPath === "NOTALLOWEDCLIENTSIDE" || !serializedModules[resolvedPath].allowclient) {
@@ -337,6 +346,7 @@
         module.id = resolvedId;
         module.filename = serializedModule?.filename;
         module.exports = {};
+        module.exports.default = module.exports;
         module.children = [];
 
         module.load = load;
@@ -383,19 +393,6 @@
 
             let dirname = module.filename.replace(/\\/g, "/").split("/").slice(0, -1).join("/");
 
-            var __createBinding = (Object.create ? (function (o, m, k, k2) {
-                if (k2 === undefined) k2 = k;
-                Object.defineProperty(o, k2, { enumerable: true, get: function () { return m[k]; } });
-            }) : (function (o, m, k, k2) {
-                if (k2 === undefined) k2 = k;
-                o[k2] = m[k];
-            }));
-            var __setModuleDefault = (Object.create ? (function (o, v) {
-                Object.defineProperty(o, "default", { enumerable: true, value: v });
-            }) : function (o, v) {
-                o["default"] = v;
-            });
-
             let time = Date.now();
             currentModuleEvaluationStack.push(module.filename);
             try {
@@ -405,21 +402,9 @@
                         //  which checks for unloadedModule and returns undefined in that case.
                         __importStar(mod) {
                             if (mod[unloadedModule]) return undefined;
-                            if (mod && mod.__esModule) return mod;
-                            var result = {};
-                            if (mod !== null && mod !== undefined) {
-                                for (var k in mod) {
-                                    if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) {
-                                        __createBinding(result, mod, k);
-                                    }
-                                }
-                            }
-                            __setModuleDefault(result, mod);
-                            return result;
+                            return mod;
                         },
                         __importDefault(mod) {
-                            // If typescript isn't going to complain about importing from a module with no default export,
-                            //  then we'll just change our implementation to work the same way as typescript types...
                             return mod.default ? mod : { default: mod };
                         },
                     },

package/src/CallFactory.ts

@@ -1,7 +1,7 @@
 import { CallerContext, CallerContextBase, CallType, FullCallType } from "../SocketFunctionTypes";
 import * as ws from "ws";
 import { performLocalCall } from "./callManager";
-import { convertErrorStackToError, formatNumberSuffixed, isNode } from "./misc";
+import { convertErrorStackToError, formatNumberSuffixed, isNode, list } from "./misc";
 import { createWebsocketFactory, getTLSSocket } from "./websocketFactory";
 import { SocketFunction } from "../SocketFunction";
 import { gzip } from "zlib";
@@ -11,6 +11,7 @@ import debugbreak from "debugbreak";
 import { lazy } from "./caching";
 import { JSONLACKS } from "./JSONLACKS/JSONLACKS";
 import { red } from "./formatting/logColors";
+import { isSplitableArray } from "./fixLargeNetworkCalls";
 
 const MIN_RETRY_DELAY = 1000;
 
@@ -115,6 +116,31 @@ export async function createCallFactory(
             if (time > SocketFunction.WIRE_WARN_TIME) {
                 console.log(red(`Slow serialize, took ${time}ms to serialize ${data.byteLength} bytes. For ${call.classGuid}.${call.functionName}`));
             }
+
+            if (data.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
+                let splitArgIndex = call.args.findIndex(isSplitableArray);
+                if (splitArgIndex >= 0) {
+                    let SPLIT_GROUPS = 10;
+                    let splitArg = call.args[splitArgIndex] as unknown[];
+                    let subCalls = list(SPLIT_GROUPS).map(index => {
+                        let start = Math.floor(index / SPLIT_GROUPS * splitArg.length);
+                        let end = Math.floor((index + 1) / SPLIT_GROUPS * splitArg.length);
+                        return splitArg.slice(start, end);
+                    }).filter(x => x.length > 0);
+                    for (let splitList of subCalls) {
+                        let subCall = { ...call };
+                        subCall.args = subCall.args.slice();
+                        subCall.args[splitArgIndex] = splitList;
+                        await callFactory.performCall(subCall);
+                    }
+                    // Eh... we COULD return the array of results, but... then the result would sometimes be an array,
+                    //  some times not, so, it is better to return a string which will make it more clear why it sometimes varies.
+                    return "CALLS_SPLIT_DUE_TO_LARGE_ARGS";
+                }
+
+                throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${data.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
+            }
+
             let resultPromise = new Promise((resolve, reject) => {
                 let callback = (result: InternalReturnType) => {
                     if (SocketFunction.logMessages) {
@@ -130,10 +156,6 @@ export async function createCallFactory(
                 pendingCalls.set(seqNum, { callback, data, call: fullCall });
             });
 
-            if (data.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
-                throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${data.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
-            }
-
             await send(data);
 
             return await resultPromise;

package/src/batching.ts

@@ -91,4 +91,37 @@ export function runInSerial<T extends (...args: any[]) => Promise<any>>(fnc: T):
             updateQueue[0]?.();
         }
     }) as T;
+}
+
+export function runInfinitePoll(
+    delayTime: number,
+    fnc: () => Promise<void> | void
+) {
+    void (async () => {
+        while (true) {
+            await delay(delayTime);
+            try {
+                await fnc();
+            } catch (e: any) {
+                console.error(`Error in infinite poll ${fnc.name || fnc.toString().slice(0, 100).split("\n").slice(0, 2).join("\n")} (continuing poll loop)\n${e.stack}`);
+            }
+        }
+    })();
+}
+
+export async function runInfinitePollCallAtStart(
+    delayTime: number,
+    fnc: () => Promise<void> | void
+) {
+    void (async () => {
+        while (true) {
+            await delay(delayTime);
+            try {
+                await fnc();
+            } catch (e: any) {
+                console.error(`Error in infinite poll ${fnc.name} (continuing poll loop)\n${e.stack}`);
+            }
+        }
+    })();
+    return await fnc();
 }

package/src/fixLargeNetworkCalls.ts

@@ -0,0 +1,9 @@
+const arrayIsSplitable = Symbol.for("arrayIsSplitable");
+export function markArrayAsSplitable<T>(data: T[]): T[] {
+    (data as any)[arrayIsSplitable] = true;
+    return data;
+}
+export function isSplitableArray<T>(data: T): data is T & (unknown[]) {
+    if (!Array.isArray(data)) return false;
+    return (data as any)[arrayIsSplitable] === true;
+}

package/src/profiling/measure.ts

@@ -12,6 +12,10 @@ export function enableMeasurements() {
     }
     measurementsEnabled = true;
 }
+/** NOTE: Must be called BEFORE anything else is imported! */
+export function disableMeasurements() {
+    measurementsEnabled = false;
+}
 
 let functionsSkipped = 0;
 
@@ -209,7 +213,7 @@ interface ProfileEntry {
     stillOpenCount: number;
 }
 
-let measurementsEnabled = false;
+let measurementsEnabled = true;
 
 let outstandingProfiles: MeasureProfile[] = [];
 function recordOwnTime(ownTimeObj: OwnTimeObj) {