socket-function 0.13.0 → 0.15.0

package/SocketFunction.ts

@@ -25,6 +25,18 @@ setImmediate(async () => {
 
 setFlag(require, "cbor-x", "allowclient", true);
 let cborxInstance = new cborx.Encoder({ structuredClone: true });
+if (isNode()) {
+    // Do not crash on unhandled errors. SocketFunction is made to run a webserver,
+    //  which will run perfectly after 99.9% of errors. Crashing the process is
+    //  not a good alternative to proper error log and notifications. Do you guys
+    //  not get automated emails when unexpected errors are logged? I do.
+    process.on("unhandledRejection", (e) => {
+        console.error("Unhandled rejection", e);
+    });
+    process.on("uncaughtException", (e) => {
+        console.error("Uncaught exception", e);
+    });
+}
 
 module.allowclient = true;
 
@@ -72,6 +84,7 @@ export class SocketFunction {
         return caller;
     }
 
+    private static getShapeHotReloadable = new Map<string, () => SocketExposedShape<SocketExposedInterface>>();
     // NOTE: We use callbacks we don't run into issues with cyclic dependencies
     //  (ex, using a hook in a controller where the hook also calls the controller).
     public static register<
@@ -107,7 +120,9 @@ export class SocketFunction {
 
             for (let value of Object.values(shape)) {
                 if (!value) continue;
-                value.clientHooks = [...(defaultHooks?.clientHooks || []), ...(value.clientHooks || [])];
+                if (!value.noClientHooks) {
+                    value.clientHooks = [...(defaultHooks?.clientHooks || []), ...(value.clientHooks || [])];
+                }
                 if (value.noDefaultHooks) {
                     value.hooks = [...(value.hooks || [])];
                 } else {
@@ -118,6 +133,9 @@ export class SocketFunction {
             return shape as any as SocketExposedShape;
         });
 
+        // Wait, so any constants referenced by the base shapeFnc will be fully resolved
+        //  by now. This is IMPORTANT, as it allows permissions functions to be moved
+        //  to a common module, instead of all being inline.
         void Promise.resolve().then(() => {
             registerClass(classGuid, instance as SocketExposedInterface, getShape(), {
                 noFunctionMeasure: config?.noFunctionMeasure,

package/SocketFunctionTypes.ts

@@ -38,6 +38,7 @@ export type SocketExposedShape<ExposedType extends SocketExposedInterface = Sock
         hooks?: SocketFunctionHook<ExposedType>[];
         clientHooks?: SocketFunctionClientHook<ExposedType>[];
         noDefaultHooks?: boolean;
+        noClientHooks?: boolean;
     };
 };
 

package/hot/HotReloadController.ts

@@ -44,12 +44,17 @@ declare global {
             noserverhotreload?: boolean;
         }
     }
+    var isHotReloading: (() => boolean) | undefined;
 }
 
 let isHotReloadingValue = false;
 export function isHotReloading() {
     return isHotReloadingValue;
 }
+globalThis.isHotReloading = isHotReloading;
+export function hotReloadingGuard(): true {
+    return !isHotReloading() as any;
+}
 export function setExternalHotReloading(value: boolean) {
     isHotReloadingValue = value;
 }
@@ -86,9 +91,14 @@ const hotReloadModule = cache((module: NodeJS.Module) => {
                     console.error(red(`Error hot reloading ${module.id}`));
                     console.error(e);
                 } finally {
-                    isHotReloadingValue = false;
+                    setTimeout(() => {
+                        isHotReloadingValue = false;
+                    }, 1000);
                 }
             }
+            for (let callback of hotReloadCallbacks) {
+                callback([module]);
+            }
         }
         triggerClientSideReload({
             files: [module.filename],
@@ -149,7 +159,14 @@ class HotReloadControllerBase {
         for (let module of modules) {
             module.loaded = false;
         }
-        await Promise.all(modules.map(module => module.load(module.filename)));
+        isHotReloadingValue = true;
+        try {
+            await Promise.all(modules.map(module => module.load(module.filename)));
+        } finally {
+            setTimeout(() => {
+                isHotReloadingValue = false;
+            }, 1000);
+        }
 
         for (let callback of hotReloadCallbacks) {
             callback(modules);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "socket-function",
-	"version": "0.13.0",
+	"version": "0.15.0",
 	"main": "index.js",
 	"license": "MIT",
 	"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -8,7 +8,6 @@
 		"@types/pako": "^2.0.3",
 		"@types/ws": "^8.5.3",
 		"cbor-x": "^1.5.6",
-		"cookie": "^0.5.0",
 		"mobx": "^6.6.2",
 		"node-forge": "https://github.com/sliftist/forge#name",
 		"pako": "^2.1.0",
@@ -21,7 +20,8 @@
 	},
 	"scripts": {
 		"test": "yarn typenode ./test/server.ts",
-		"type": "yarn tsc --noEmit"
+		"type": "yarn tsc --noEmit",
+		"testsni": "yarn typenode ./src/sniTest.ts"
 	},
 	"devDependencies": {
 		"@types/cookie": "^0.5.1",

package/require/RequireController.ts

@@ -69,6 +69,10 @@ export interface SerializedModule {
 
     size?: number;
     version?: number;
+
+    flags?: {
+        [flag: string]: true;
+    };
 }
 
 let nextModuleSeqNum = 1;
@@ -203,7 +207,13 @@ class RequireControllerBase {
                 size: module.size,
                 version: module.version,
                 asyncRequests: module.asyncRequires,
+                flags: {},
             };
+            for (let [flag, value] of Object.entries(module)) {
+                if (value === true) {
+                    modules[module.filename].flags![flag] = value;
+                }
+            }
             let moduleObj = modules[module.filename];
             if (moduleObj.allowclient) {
                 moduleObj.source = module.moduleContents;

package/require/require.js

@@ -3,6 +3,9 @@
 
     let startTime = Date.now();
 
+    Symbol.dispose = Symbol.dispose || Symbol("dispose");
+    Symbol.asyncDispose = Symbol.asyncDispose || Symbol("asyncDispose");
+
     // Globals
     Object.assign(window, {
         process: {
@@ -397,6 +400,9 @@
         module.exports = {};
         module.exports.default = module.exports;
         module.children = [];
+        for (let key in serializedModule.flags || {}) {
+            module[key] = true;
+        }
 
         module.load = load;
 

package/src/callManager.ts

@@ -69,7 +69,7 @@ export function isDataImmutable(call: CallType) {
 export function registerClass(classGuid: string, controller: SocketExposedInterface, shape: SocketExposedShape, config?: {
     noFunctionMeasure?: boolean;
 }) {
-    if (classes[classGuid]) {
+    if (!isHotReloading?.() && classes[classGuid]) {
         throw new Error(`Class ${classGuid} already registered`);
     }
 

package/src/formatting/format.ts

@@ -154,9 +154,15 @@ export function formatNumber(count: number | undefined, maxAbsoluteValue?: numbe
     } else if (maxAbsoluteValue < 1000 * 1000 * 1000 * extraFactor) {
         suffix = "M";
         divisor = 1000 * 1000;
-    } else {
+    } else if (maxAbsoluteValue < 1000 * 1000 * 1000 * 1000 * extraFactor) {
         suffix = "B";
         divisor = 1000 * 1000 * 1000;
+    } else if (maxAbsoluteValue < 1000 * 1000 * 1000 * 1000 * 1000 * extraFactor) {
+        suffix = "T";
+        divisor = 1000 * 1000 * 1000 * 1000;
+    } else {
+        suffix = "Q";
+        divisor = 1000 * 1000 * 1000 * 1000 * 1000;
     }
     count /= divisor;
     maxAbsoluteValue /= divisor;

package/src/formatting/logColors.ts

@@ -14,5 +14,6 @@ export const red = ansiHSL.bind(null, 0, 100, lightness);
 export const green = (text: string) => `\x1b[32m${text}\x1b[0m`;
 export const yellow = (text: string) => `\x1b[33m${text}\x1b[0m`;
 export const white = ansiHSL.bind(null, 0, 0, 80);
+export const gray = ansiHSL.bind(null, 0, 0, 50);
 
 export const magenta = (text: string) => `\x1b[35m${text}\x1b[0m`;

package/src/forwardPort.ts

@@ -0,0 +1,157 @@
+import * as dgram from "dgram";
+import os from "os";
+
+const SSDP_DISCOVER_MX = 2;
+const SSDP_DISCOVER_MSG = `M-SEARCH * HTTP/1.1\r\nHOST: 239.255.255.250:1900\r\nMAN: "ssdp:discover"\r\nMX: ${SSDP_DISCOVER_MX}\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\n\r\n`;
+
+export async function forwardPort(config: {
+    externalPort: number;
+    internalPort: number;
+}) {
+    const { externalPort, internalPort } = config;
+
+    const localObj = getLocalInterfaceAddress();
+    if (!localObj) throw new Error("Could not find the local address / gateway");
+
+    const { internalIP, gatewayIP } = localObj;
+    let gateway = await discoverGateway(internalIP);
+    let controlURLs = await getControlPaths(gateway);
+    let controlPort = Number(new URL(gateway).port);
+
+    for (let controlURL of controlURLs) {
+        try {
+            await createPortMapping({
+                externalPort, internalPort,
+                gatewayIP,
+                controlPort,
+                controlPath: controlURL,
+                internalIP,
+            });
+            return;
+        } catch (e) {
+            console.error(e);
+        }
+    }
+    console.error("Failed to create port mapping, could not find controlURL");
+}
+
+function getLocalInterfaceAddress(): { internalIP: string; gatewayIP: string; } | undefined {
+    const interfaces = os.networkInterfaces() as any;
+    for (const name of Object.keys(interfaces)) {
+        for (const iface of interfaces[name]) {
+            if (iface.family === "IPv4" && !iface.internal) {
+                // TOOD: Correctly resolve the cidr?
+                let gatewayIP = iface.cidr.split(".").slice(0, 3).join(".") + ".1";
+                // TOOD: We try discovery on all gateways, so we can know for sure which one it is
+                //  (and maybe even port forward all gateway, if multiple respond?)
+                if (gatewayIP.startsWith("10.0.0") || gatewayIP.startsWith("10.0.1") || gatewayIP.startsWith("192.168.0")) {
+                    return { internalIP: iface.address, gatewayIP };
+                }
+            }
+        }
+    }
+    return undefined;
+}
+
+function discoverGateway(localAddress: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const socket = dgram.createSocket("udp4");
+        let isResolved = false;
+
+        if (!localAddress) {
+            reject(new Error("Could not find a suitable local address"));
+            return;
+        }
+
+        socket.on("message", (msg) => {
+            const response = msg.toString();
+            const location = response.match(/LOCATION: (.*)\r\n/i);
+            if (location && location[1]) {
+                isResolved = true;
+                socket.close();
+                resolve(location[1]);
+            }
+        });
+
+        socket.on("error", (err) => {
+            socket.close();
+            if (!isResolved) {
+                reject(err);
+            }
+        });
+
+        socket.on("listening", () => {
+            socket.addMembership("239.255.255.250", localAddress);
+        });
+
+        socket.bind({ address: localAddress }, () => {
+            socket.setBroadcast(true);
+            socket.send(SSDP_DISCOVER_MSG, 0, SSDP_DISCOVER_MSG.length, 1900, "239.255.255.250");
+        });
+
+        setTimeout(() => {
+            if (!isResolved) {
+                socket.close();
+                reject(new Error(`SSDP discovery timeout. Search on ${localAddress}`));
+            }
+        }, SSDP_DISCOVER_MX * 1000);
+    });
+}
+
+async function getControlPaths(gateway: string) {
+    let xml = await (await fetch(gateway)).text();
+    const controlURLRegex = /<controlURL>(.*?)<\/controlURL>/g;
+    const matches = [];
+    let match;
+    while ((match = controlURLRegex.exec(xml)) !== null) {
+        matches.push(match[1]);
+    }
+    matches.reverse();
+    return matches;
+}
+
+async function createPortMapping(config: {
+    externalPort: number;
+    internalPort: number;
+    gatewayIP: string;
+    controlPort: number;
+    controlPath: string;
+    internalIP: string;
+
+}): Promise<void> {
+    const { externalPort, internalPort, internalIP, controlPath, controlPort, gatewayIP } = config;
+    const action = "\"urn:schemas-upnp-org:service:WANIPConnection:1#AddPortMapping\"";
+
+    const soapBody = `
+        <?xml version="1.0"?>
+        <s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
+            <s:Body>
+                <u:AddPortMapping xmlns:u="urn:schemas-upnp-org:service:WANIPConnection:1">
+                    <NewRemoteHost></NewRemoteHost>
+                    <NewExternalPort>${externalPort}</NewExternalPort>
+                    <NewProtocol>TCP</NewProtocol>
+                    <NewInternalPort>${internalPort}</NewInternalPort>
+                    <NewInternalClient>${internalIP}</NewInternalClient>
+                    <NewEnabled>1</NewEnabled>
+                    <NewPortMappingDescription>My Port Mapping</NewPortMappingDescription>
+                    <NewLeaseDuration>0</NewLeaseDuration>
+                </u:AddPortMapping>
+            </s:Body>
+        </s:Envelope>
+    `;
+
+    const res = await fetch(`http://${gatewayIP}:${controlPort}${controlPath}`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "text/xml; charset=\"utf-8\"",
+            "SOAPAction": action,
+            "Content-Length": Buffer.byteLength(soapBody) + "",
+        },
+        body: soapBody
+    });
+
+    if (res.status !== 200) {
+        const data = await res.text();
+        throw new Error(`Failed to create port mapping: ${data}`);
+    }
+}

package/src/https.ts

@@ -0,0 +1,115 @@
+import * as https from "https";
+import * as http from "http";
+import { isNode } from "./misc";
+
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+
+export function httpsRequest(
+    url: string,
+    payload?: Buffer | Buffer[],
+    method = "GET",
+    sendSessionCookies = true,
+    config?: {
+        headers?: { [key: string]: string },
+    }
+): Promise<Buffer> {
+    if (isNode()) {
+        return (async () => {
+            let urlObj = new URL(url);
+
+            let requestor = url.startsWith("https") ? https : http;
+            let port = url.startsWith("https") ? 443 : 80;
+            if (urlObj.port) {
+                port = +urlObj.port;
+            }
+
+            return new Promise<Buffer>((resolve, reject) => {
+                let httpRequest = requestor.request(
+                    urlObj + "",
+                    {
+                        method,
+                        headers: config?.headers,
+                    },
+                    async httpResponse => {
+                        let data: Buffer[] = [];
+                        httpResponse.on("data", chunk => {
+                            data.push(chunk);
+                        });
+
+                        await new Promise<void>(resolve => {
+                            httpResponse.on("end", () => {
+                                resolve();
+                            });
+                        });
+
+                        if (!httpResponse.statusCode?.toString().startsWith("2")) {
+                            reject(new Error(`Error for ${url}, ${httpResponse.statusCode} ${httpResponse.statusMessage}\n` + Buffer.concat(data).toString()));
+                        } else {
+                            resolve(Buffer.concat(data));
+                        }
+                    }
+                );
+                httpRequest.on("error", reject);
+
+                if (payload) {
+                    if (Array.isArray(payload)) {
+                        payload = Buffer.concat(payload);
+                    }
+                    httpRequest.write(payload);
+                }
+                httpRequest.end();
+            });
+        })();
+
+    } else {
+        var request = new XMLHttpRequest();
+        request.open(method, url, true);
+        if (config?.headers) {
+            for (let key in config.headers) {
+                request.setRequestHeader(key, config.headers[key]);
+            }
+        }
+        request.responseType = "arraybuffer";
+        request.withCredentials = sendSessionCookies;
+        if (payload) {
+            if (Array.isArray(payload)) {
+                payload = Buffer.concat(payload);
+            }
+            request.send(payload);
+        } else {
+            request.send();
+        }
+        return new Promise((resolve, reject) => {
+            request.onload = () => {
+                if (request.status !== 200) {
+                    try {
+                        // It should be an error.stack. But if it isn't... just throw the status text...
+                        let responseText = textDecoder.decode(request.response);
+                        let message = responseText.split("\n")[0];
+
+                        let error = new Error(`For ${url}, ` + message);
+                        error.stack = `For ${url}, ` + responseText;
+
+                        reject(error);
+
+                    } catch (e: any) {
+                        reject(new Error(`For ${url}, ` + request.statusText));
+                    }
+                } else {
+                    resolve(Buffer.from(request.response));
+                }
+            };
+
+            request.onerror = (e) => {
+                reject(new Error(`Network error for request at ${url}`));
+            };
+            request.ontimeout = (e) => {
+                reject(new Error(`Network timeout for request at ${url}`));
+            };
+            request.onabort = (e) => {
+                reject(new Error(`Network abort for request at ${url}`));
+            };
+        });
+    }
+}

package/src/misc.ts

@@ -126,13 +126,6 @@ export function getStringKeys<T extends {}>(obj: T): ((keyof T) & string)[] {
     return Object.keys(obj) as any;
 }
 
-if (isNode()) {
-    // TODO: Find a better place for this...
-    process.on("unhandledRejection", async (reason: any, promise) => {
-        console.error(`Uncaught promise rejection: ${String(reason.stack || reason)}`);
-    });
-}
-
 export function keyBy<T, K>(arr: T[], getKey: (value: T) => K): Map<K, T> {
     let map = new Map<K, T>();
     for (let item of arr) {
@@ -159,43 +152,44 @@ export function deepCloneJSON<T>(obj: T): T {
     return JSON.parse(JSON.stringify(obj));
 }
 
+export class PromiseObj<T = void> {
+    public promise: Promise<T>;
+    public value: { value?: T; error?: string } | undefined;
+    /** Resolve called does not mean the value is ready, as it may be resolved with a promise. */
+    public resolveCalled?: boolean;
 
+    public resolve(value: T | Promise<T>) {
+        this.resolveCalled = true;
+        if (typeof value === "object" && value !== null && value instanceof Promise) {
+            value.then(
+                value => this.value = { value },
+                error => this.value = { error },
+            );
+        } else {
+            this.value = { value };
+        }
+        this.baseResolve(value);
+    }
+    public reject(error: any) {
+        this.baseReject(error);
+    }
 
-export interface PromiseObj<T = void> {
-    resolve: (value: T | Promise<T>) => void;
-    reject: (error: any) => void;
-    promise: Promise<T>;
-    value: { value?: T; error?: string } | undefined;
-    /** Resolve called does not mean the value is ready, as it may be resolved with a promise. */
-    resolveCalled?: boolean;
+    private baseResolve!: (value: T | Promise<T>) => void;
+    private baseReject!: (error: any) => void;
+    constructor() {
+        this.promise = new Promise<T>((resolve, reject) => {
+            this.baseResolve = resolve;
+            this.baseReject = reject;
+        });
+        this.promise.then(
+            value => this.value = { value },
+            error => this.value = { error }
+        );
+    }
 }
 
 export function promiseObj<T = void>(): PromiseObj<T> {
-    let resolve!: (value: T | Promise<T>) => void;
-    let reject!: (error: any) => void;
-    let promise = new Promise<T>((_resolve, _reject) => {
-        resolve = _resolve;
-        reject = _reject;
-    });
-    let obj: PromiseObj<T> = {
-        resolve(value: T | Promise<T>) {
-            obj.resolveCalled = true;
-            if (typeof value === "object" && value !== null && value instanceof Promise) {
-                value.then(
-                    value => obj.value = { value },
-                    error => obj.value = { error },
-                );
-            } else {
-                obj.value = { value };
-            }
-            resolve(value);
-        },
-        reject,
-        promise,
-        value: undefined
-    };
-    promise.then(value => obj.value = { value }, error => obj.value = { error });
-    return obj;
+    return new PromiseObj<T>();
 }
 
 
@@ -291,13 +285,29 @@ export function entries<Obj extends { [key: string]: unknown }>(obj: Obj): [keyo
     return Object.entries(obj) as any;
 }
 
+export function keys<Obj extends { [key: string]: unknown }>(obj: Obj): (keyof Obj)[] {
+    return Object.keys(obj) as any;
+}
+
 export function sort<T>(arr: T[], sortKey: (obj: T) => unknown) {
     if (arr.length <= 1) return arr;
     arr.sort((a, b) => compare(sortKey(a), sortKey(b)));
     return arr;
 }
 
-// NOTE: If there are duplicates, returns the first match.
+export function binarySearchBasic<T, V>(array: T[], getVal: (val: T) => V, searchValue: V): number {
+    return binarySearchIndex(array.length, i => compare(getVal(array[i]), searchValue));
+}
+
+/**
+ *  Searches indexes, allowing you to query structures that aren't arrays. To search an array, use:
+ *      `binarySearchIndex(array.length, i => compare(array[i], searchValue))`
+ * 
+ *      NOTE: If there are duplicates, returns the first match.
+ * 
+ *      NOTE: If the value can't be found, returns the bitwise negation of the index where it should be inserted.
+ *          - If you just want the index which is >=, use `if(index < 0) index = ~index;`
+ */
 export function binarySearchIndex(listCount: number, compare: (lhsIndex: number) => number): number {
     if (listCount === 0) {
         return ~0;
@@ -324,6 +334,8 @@ export function compare(lhs: unknown, rhs: unknown): number {
         return compare(typeof lhs, typeof rhs);
     }
     if (lhs === rhs) return 0;
+    if (lhs === null && rhs !== null) return -1;
+    if (lhs !== null && rhs === null) return 1;
     if (typeof lhs === "number") {
         if (Number.isNaN(lhs)) {
             if (Number.isNaN(rhs)) return 0;

package/src/networking.ts

@@ -0,0 +1,44 @@
+import net from "net";
+import { lazy } from "./caching";
+import { httpsRequest } from "./https";
+import { measureWrap } from "./profiling/measure";
+
+export const testTCPIsListening = measureWrap(async function testTCPIsListening(host: string, port: number): Promise<boolean> {
+    // We need to establish a TCP connection, then close it? Yeah... so it is
+    //  not even a SocketFunction call, because it can't be, because that woule be TLS,
+    //  which we can't do with an ip!
+    let socket = net.connect({ host, port });
+    return new Promise((resolve) => {
+        socket.on("connect", () => {
+            socket.end();
+            resolve(true);
+        });
+        socket.on("error", () => {
+            resolve(false);
+        });
+        setTimeout(() => {
+            socket.end();
+            resolve(false);
+        }, 1000 * 60);
+    });
+});
+
+
+const ipServers = [
+    "http://quentinbrooks.com:4283",
+    "https://ipinfo.io/ip",
+    "https://api.ipify.org"
+];
+
+export const getExternalIP = lazy(measureWrap(async function getExternalIP(): Promise<string> {
+    for (let server of ipServers) {
+        try {
+            return (await httpsRequest(server)).toString();
+        } catch (e) {
+            console.warn(`Failed to get external ip from ${server}: ${e}`);
+        }
+    }
+    throw new Error(`Failed to get external ip from any server`);
+}));
+
+export const getPublicIP = getExternalIP;

package/src/profiling/measure.ts

@@ -62,12 +62,14 @@ export function measureWrap<T extends (...args: any[]) => any>(fnc: T, name?: st
         return fnc;
     }
     let usedName = name || fnc.name || fnc.toString().slice(0, 100).replaceAll(/\s/g, " ");
-    return nameFunction(usedName, (function (this: any, ...args: unknown[]): unknown {
+    let output = nameFunction(usedName, (function (this: any, ...args: unknown[]): unknown {
         if (outstandingProfiles.length === 0) {
             return fnc.apply(this, args);
         }
         return getOwnTime(usedName, () => fnc.apply(this, args), recordOwnTime);
     })) as T;
+    (output as any).originalFnc = fnc;
+    return output;
 }
 export function measureBlock<T extends (...args: any[]) => any>(fnc: T, name?: string): ReturnType<T> {
     return measureWrap(fnc, name)();

package/src/sniTest.ts

@@ -0,0 +1,101 @@
+import { parseSNIExtension, parseTLSHello, SNIType } from "./tlsParsing";
+
+let tlsExtensionLookup: { [type: number]: string } = {
+    0: "server_name",
+    1: "max_fragment_length",
+    2: "client_certificate_url",
+    3: "trusted_ca_keys",
+    4: "truncated_hmac",
+    5: "status_request",
+    6: "user_mapping",
+    7: "client_authz",
+    8: "server_authz",
+    9: "cert_type",
+    10: "supported_groups",
+    11: "ec_point_formats",
+    12: "srp",
+    13: "signature_algorithms",
+    14: "use_srtp",
+    15: "heartbeat",
+    16: "application_layer_protocol_negotiation",
+    17: "status_request_v2",
+    18: "signed_certificate_timestamp",
+    19: "client_certificate_type",
+    20: "server_certificate_type",
+    21: "padding",
+    22: "encrypt_then_mac",
+    23: "extended_master_secret",
+    24: "token_binding",
+    25: "cached_info",
+    26: "tls_lts",
+    27: "compress_certificate",
+    28: "record_size_limit",
+    29: "pwd_protect",
+    30: "pwd_clear",
+    31: "password_salt",
+    32: "ticket_pinning",
+    33: "tls_cert_with_extern_psk",
+    34: "delegated_credential",
+    35: "session_ticket",
+    36: "TLMSP",
+    37: "TLMSP_proxying",
+    38: "TLMSP_delegate",
+    39: "supported_ekt_ciphers",
+    40: "Reserved",
+    41: "pre_shared_key",
+    42: "early_data",
+    43: "supported_versions",
+    44: "cookie",
+    45: "psk_key_exchange_modes",
+    46: "Reserved",
+    47: "certificate_authorities",
+    48: "oid_filters",
+    49: "post_handshake_auth",
+    50: "signature_algorithms_cert",
+    51: "key_share",
+    52: "transparency_info",
+    53: "connection_id",
+    54: "connection_id",
+    55: "external_id_hash",
+    56: "external_session_id",
+    57: "quic_transport_parameters",
+    58: "ticket_request",
+    59: "dnssec_chain",
+    60: "sequence_number_encryption_algorithms",
+    61: "rrc",
+    2570: "encrypted_client_hello?",
+    17513: "application_settings",
+    6682: "generated_random_extensions_and_sustain_extensibility",
+    10794: "generated_random_extensions_and_sustain_extensibility",
+    14906: "generated_random_extensions_and_sustain_extensibility",
+    19018: "generated_random_extensions_and_sustain_extensibility",
+    23130: "generated_random_extensions_and_sustain_extensibility",
+    27242: "generated_random_extensions_and_sustain_extensibility",
+    31354: "generated_random_extensions_and_sustain_extensibility",
+    35466: "generated_random_extensions_and_sustain_extensibility",
+    39578: "generated_random_extensions_and_sustain_extensibility",
+    43690: "generated_random_extensions_and_sustain_extensibility",
+    47802: "generated_random_extensions_and_sustain_extensibility",
+    51914: "generated_random_extensions_and_sustain_extensibility",
+    56026: "generated_random_extensions_and_sustain_extensibility",
+    60138: "generated_random_extensions_and_sustain_extensibility",
+    64250: "generated_random_extensions_and_sustain_extensibility",
+    65037: "encrypted_client_hello",
+    65281: "renegotiation_info" // 0xFF01
+};
+
+async function main() {
+    const packet = Buffer.from(
+        `FgMBBwgBAAcEAwNWbpKnOzI9CRp2ESvA5QzCXg5FYncEObckUkNoG3+/DyAwI4HL0havBXKvPlJtZJBgtZ+I/FqBlKGek8NGJcgdqQAgiooTARMCEwPAK8AvwCzAMMypzKjAE8AUAJwAnQAvADUBAAabSkoAAAAbAAMCAAIACgAMAArq6mOZAB0AFwAYAAUABQEAAAAAAC0AAgEB/wEAAQAAIwAAAAsAAgEAABIAAAANABIAEAQDCAQEAQUDCAUFAQgGBgEAFwAA/g0AugAAAQABCwAgIRJsXQHGN5cbIp9ucgXJ824RS/6hqIikNiMKw7/gsDAAkK8j0YYXYsSnpX4siEVnB/AxzQCPaGMoT4y1i63f4mzV0Sa6puc4sZqkxY7TyQMJLrGPL00HaJZh6ReT/ggPbCwg5f5/2hUCdZSx4aUpYMb7lAx9VVJjhrquZIRRVrk8yV0OjEivreTN02u1mgIgMY/0P8RzOea+iQluEF9ASOYxgNIU20xoKK608ktQrTafSQAQAAsACQhodHRwLzEuMQAzBO8E7erqAAEAY5kEwB8fuK+qumVLi67+BmQ5uQ556Opa122UZFmnfuoKQY1tiXkzM7Z+wmWOLTg7l3Ncu8ECL/Z9G6ZLP4N5OKGjiHE307sEvNRqdEnFYoQMVotF6hNbt3F4GrEBa6CuCEohg9HKg3RtIfMFYNc1lDC4rJCLwNETijo5MlOFnYVGpWOMcBZbdLsGv5JxN7idRUGvxsttaya3GYxSI1yFW9dZaysTsYXA2IzPp7M950GkJzBK5Zy3wBiXoZYGYKJGy5EA1XIE88Y6X2VNghPIatHHjdav+PpsLZCyUowEsAN49gh/yRrBNZxo6gNbfOIds/UpjlYtIEo1lch2ygWzVLS55qxm35IQUVKQjEQiPDNUI6g9RZZD+VSH5vB1ouMwTqbDAAN9EsUxigYmQggjz1jGFEm+DecEQTkXoQOIwWsAq+M39Eq7M3CwYVQ6yDQI+rQD9xxin8I9PrdVoXGaTVwnY9GFKFCYZyewSGeuPGSaPcdKwvBb/IgfFVRt2aU24XNmFsCkgLSzHEkBg1uYqjd+uIY7zTWo5WIevniL4SCIvRyvTQB7DICAZ+BxSUxio9x19OWQCitnoXdoGGwuTIQqSYuimFBedKa2b4Yk3nkPIRqa3jldbGMRwQmpaQFnsOJml/pEr3sQIuEcQ2shaIuqW+Zo86yamflqpCmCbQWNiipRYwQ/AmG7oOeS1iJomPiRJDwVNaoDZzI2ApYIcQckNPNrI9qOrGwww6x1T9EwKadz/1OcHQcbWtej5QkLm5Ss/9paxqYsu1kuohlvPOSd4txqKOGNvOhzxqqOweZ7p8l/H/JhnMkEUCZYvOW4JhthXsYMfnWn8rciE7KuMskf65mqI1tI5SZqzxAfH4MidyUJRNwR6mwWUgzMi7kB3eM3IPUbCDpUIpQBipt2SyenIMXNH9ZjnYtdCwVDe7m6mshYAJVV7oJDw5yn2pYH5nsZ+gM93vNWBvZVQjZZmRdZQ2YqhKaUyQgpOEhlE1w7XAXJp+um2kA5qElw9sNbxvOziNSPLyi17xgy1CSBSzB+o8JAfZamxUHFXEGSRYoR8qekMnlJeVRXlojEoaxMsspFX3itMOqIHNUL9Nltswaqggu7s+NQfshoLEg5gkfM/tok4sISLZlX7ACgcWZBQudMPTyn3FsQIbzNgXBzWTdiCZg9iZttoJtX8FygYbzEF9kFJyc5fIBA4NACVZhOf+onlMcFEbHCu6ilqMuzwQxf6vgV`
+            .replace(/\s/g, "")
+        , "base64"
+    );
+
+    let data = parseTLSHello(packet);
+    let sni = data.extensions.filter(x => x.type === SNIType).flatMap(x => parseSNIExtension(x.data))[0];
+    console.log(`Packet size ${packet.byteLength}, missing bytes ${data.missingBytes}`);
+    for (let ext of data.extensions) {
+        console.log(`Extension: ${tlsExtensionLookup[ext.type] || ext.type}, bytes length: ${ext.data.length}`);
+    }
+}
+main().catch(e => console.error(e)).finally(() => process.exit());

package/src/tlsParsing.ts

@@ -45,7 +45,6 @@ export function parseTLSHello(buffer: Buffer): {
         let sessionIdLength = buffer[pos++];
         pos += sessionIdLength;
 
-
         let cipherSuiteLength = readShort();
         pos += cipherSuiteLength;
 
@@ -53,11 +52,17 @@ export function parseTLSHello(buffer: Buffer): {
         pos += compressionLength;
 
         let extensionsLength = readShort();
-        output.missingBytes = contentLength - (pos + extensionsLength);
         let extensionsEnd = pos + extensionsLength;
         while (pos < extensionsEnd) {
             let extensionType = readShort();
             let length = readShort();
+            // Break if we only have part of the extension
+            if (Number.isNaN(extensionType) || Number.isNaN(length)) {
+                break;
+            }
+            if (pos + length > buffer.length) {
+                break;
+            }
 
             output.extensions.push({
                 type: extensionType, data: viewSliceBuffer(buffer, pos, length)
@@ -65,6 +70,7 @@ export function parseTLSHello(buffer: Buffer): {
 
             pos += length;
         }
+        output.missingBytes = contentLength - pos;
     } catch { }
 
     return output;

package/src/webSocketServer.ts

@@ -11,12 +11,14 @@ import { parseSNIExtension, parseTLSHello, SNIType } from "./tlsParsing";
 import debugbreak from "debugbreak";
 import { getNodeId } from "./nodeCache";
 import crypto from "crypto";
-import { Watchable } from "./misc";
-import { delay, runInfinitePoll } from "./batching";
-import { magenta } from "./formatting/logColors";
+import { Watchable, timeInHour } from "./misc";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "./batching";
+import { magenta, red } from "./formatting/logColors";
 import { yellow } from "./formatting/logColors";
 import { green } from "./formatting/logColors";
 import { formatTime } from "./formatting/format";
+import { getExternalIP, testTCPIsListening } from "./networking";
+import { forwardPort } from "./forwardPort";
 
 export type SocketServerConfig = (
     https.ServerOptions & {
@@ -30,6 +32,12 @@ export type SocketServerConfig = (
         // public sets ip to "0.0.0.0", otherwise it defaults to "127.0.0.1", which
         //  causes the server to only accept local connections.
         public?: boolean;
+        /** Tries forwarding ports (using UPnP), if we detect they aren't externally reachable.
+         *      - This causes an extra request and delay during startup, so should only be used
+         *          during development.
+         *      - Ignored if public is false
+        */
+        autoForwardPort?: boolean;
         ip?: string;
 
         // NOTE: Any same origin accesses are allowed (header.origin === header.host)
@@ -187,6 +195,10 @@ export async function startSocketServer(
                 if (!SocketFunction.silent) {
                     console.log(`Received TCP connection with SNI ${JSON.stringify(sni)}`);
                 }
+                if (!sni) {
+                    console.warn(`No SNI found in TLS hello, using main server. Packets ${packetCount}`);
+                    console.log(buffer.toString("base64"));
+                }
                 server = sniServers.get(sni) || mainHTTPSServer;
             }
 
@@ -214,49 +226,78 @@ export async function startSocketServer(
     });
 
 
-    let listenPromise = new Promise<void>((resolve, error) => {
-        realServer.on("listening", () => {
-            resolve();
-        });
-        realServer.on("error", e => {
-            error(e);
-        });
-    });
-
     let host = config.public ? "0.0.0.0" : "127.0.0.1";
     if (config.ip) {
         host = config.ip;
     }
 
     let port = config.port;
-    async function isPortInUse(port: number): Promise<boolean> {
-        return new Promise<boolean>((resolve, reject) => {
-            let server = net.createServer();
-            server.listen(port, host)
-                .on("listening", function () {
-                    server.close();
-                    resolve(false);
-                }).on("close", function () {
-                    resolve(true);
-                }).on("error", function (e) {
-                    resolve(true);
-                });
+    if (!SocketFunction.silent) {
+        console.log(yellow(`Trying to listening on ${host}:${port}`));
+    }
+
+    let listeningPromise = waitUntilListening();
+    listeningPromise.catch(e => { });
+
+    // Return true if we are listening, false if the address is in use, and throws on other errors
+    async function waitUntilListening() {
+        return await new Promise<boolean>((resolve, reject) => {
+            realServer.once("error", e => {
+                reject(e);
+            });
+            realServer.once("listening", () => {
+                resolve(false);
+            });
         });
     }
+
     if (config.useAvailablePortIfPortInUse && port) {
-        if (await isPortInUse(port)) {
+        realServer.listen(port, host);
+        let isListening = await new Promise<boolean>((resolve, reject) => {
+            if (realServer.listening) {
+                resolve(true);
+                return;
+            }
+            realServer.once("error", e => {
+                if (e.message.includes("EADDRINUSE")) {
+                    resolve(true);
+                } else {
+                    reject(e);
+                }
+            });
+            realServer.once("listening", () => {
+                resolve(false);
+            });
+        });
+        if (!isListening) {
             port = 0;
+            realServer.listen(port, host);
+            listeningPromise = waitUntilListening();
         }
+    } else {
+        realServer.listen(port, host);
     }
 
-    if (!SocketFunction.silent) {
-        console.log(yellow(`Trying to listening on ${host}:${port}`));
-    }
-    realServer.listen(port, host);
+    await listeningPromise;
+    port = (realServer.address() as net.AddressInfo).port;
 
-    await listenPromise;
+    if (config.autoForwardPort && config.public) {
+        // let externalIP = await getExternalIP();
+        // let isListening = await testTCPIsListening(externalIP, port);
+        // if (!isListening) {
+        //     console.log(magenta(`Port ${port} is not externally reachable, trying to forward it`));
+        //     await forwardPort({ externalPort: port, internalPort: port });
+        // }
+        // Even if they are listening, they might not stay listening. Forward every 8 hours
+        //      (including at the start, in case the forward is about to expire).
+        async function forward() {
+            await forwardPort({ externalPort: port, internalPort: port });
+            console.log(magenta(`Forwarded port ${port} to our machine`));
+        }
+        // Every hour, in case our network configuration changes
+        runInfinitePollCallAtStart(timeInHour * 1, forward).catch(e => console.error(red(`Error in port forwarding ${e.stack}`)));
+    }
 
-    port = (realServer.address() as net.AddressInfo).port;
     let nodeId = getNodeId(getCommonName(config.cert), port);
     console.log(green(`Started Listening on ${nodeId} after ${formatTime(process.uptime() * 1000)}`));
 

package/test.ts

@@ -0,0 +1,33 @@
+import { getExternalIP } from "./src/networking";
+import http from "http";
+import * as dgram from "dgram";
+import os from "os";
+import debugbreak from "debugbreak";
+import { forwardPort } from "./src/forwardPort";
+
+
+// Usage example:
+async function main() {
+    const externalPort = 8088;
+    const internalPort = externalPort;
+
+    await forwardPort({ externalPort, internalPort });
+
+    // Listen on the external port
+    const server = http.createServer((req, res) => {
+        console.log("Request received");
+        res.end("Hello, world!");
+    });
+    server.listen(externalPort, "0.0.0.0");
+
+    {
+        const externalIP = await getExternalIP();
+        let test = await fetch(`http://${externalIP}:${externalPort}`);
+        console.log(await test.text());
+    }
+
+
+    //await createPortMapping({ externalPort, internalPort, gateWayIP, internalIP, });
+}
+
+main().catch(e => console.error(e));

package/time/trueTimeShim.ts

@@ -128,7 +128,9 @@ async function updateTimeOffset() {
                 || Math.abs(offset) > 300 && yellow(offsetRound + "ms")
                 || green(offsetRound + "ms")
             );
-            console.log(`${blue("Synchronized time")}, local clock was ${offset > 0 ? "behind" : "ahead"} by ${offsetColored}`);
+            if (Math.abs(offset) > 500) {
+                console.log(`${blue("Synchronized time")}, local clock was ${offset > 0 ? "behind" : "ahead"} by ${offsetColored} @ ${blue(Date.now() + "")}`);
+            }
             for (let i = 0; i < currentSmearCount; i++) {
                 let fraction = (i + 1) / currentSmearCount;
                 trueTimeOffset = prevOffset * (1 - fraction) + offset * fraction;