socket-function 0.65.0 → 0.67.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "socket-function",
-	"version": "0.65.0",
+	"version": "0.67.0",
 	"main": "index.js",
 	"license": "MIT",
 	"dependencies": {

package/require/RequireController.ts

@@ -134,8 +134,9 @@ class RequireControllerBase {
 
     public async requireHTML(config?: {
         requireCalls?: string[];
+        cacheTime?: number;
     }) {
-        let { requireCalls } = config || {};
+        let { requireCalls, cacheTime } = config || {};
         let result = resolvedHTMLFile();
         if (beforeEntryText.length > 0) {
             let resolved: string[] = [];
@@ -166,7 +167,13 @@ class RequireControllerBase {
         } else {
             result = result.replace(ENTRY_TEMPLATE, "");
         }
-        return setHTTPResultHeaders(Buffer.from(result), { "Content-Type": "text/html" });
+        let headers: Record<string, string> = {
+            "Content-Type": "text/html"
+        };
+        if (cacheTime) {
+            headers["Cache-Control"] = `max-age=${Math.floor(cacheTime / 1000)}`;
+        }
+        return setHTTPResultHeaders(Buffer.from(result), headers);
     }
 
     public async getModules(
@@ -347,8 +354,14 @@ class RequireControllerBase {
                 }]),
             };
             let key = sha256Hash(JSON.stringify(simplifiedResult));
-            let buffer = await compressCached(key, () => Buffer.from(JSON.stringify(result)));
-            setHTTPResultHeaders(buffer, { "Content-Type": "application/json", "Content-Encoding": "gzip" });
+            let uncompressedBuffer = Buffer.from(JSON.stringify(result));
+            let buffer = await compressCached(key, () => uncompressedBuffer);
+            setHTTPResultHeaders(buffer, {
+                "Content-Type": "application/json",
+                "Content-Encoding": "gzip",
+                "Content-Length": buffer.length.toString(),
+                "X-Uncompressed-Content-Length": uncompressedBuffer.length.toString(),
+            });
             return buffer as any;
         }
 

package/require/require.ts

@@ -1,11 +1,86 @@
 import { GetModulesArgs, SerializedModule } from "./RequireController";
 
+
+declare global {
+    var onProgressHandler: undefined | ((progress: {
+        type: string;
+        addValue: number;
+        addMax: number;
+    }) => void);
+    var onErrorHandler: undefined | ((error: string) => void);
+
+
+    var BOOT_TIME: number;
+    var builtInModuleExports: {
+        [key: string]: unknown;
+    };
+}
+
 export function requireMain() {
-    //# sourceURL=require.js
+    async function requestText(endpoint: string, values: any) {
+        let url = new URL(endpoint);
+
+        let json = JSON.stringify(values);
+        let response: Response;
+        if (json.length < 6000) {
+            // NOTE: Try to use a GET, as GETs can be cached! However, if the data is too large,
+            //  we have to use a post, or else the request url will be too large
+            for (let key in values) {
+                url.searchParams.set(key, JSON.stringify(values[key]));
+            }
+            response = await fetch(url.toString(), {
+                method: "GET",
+                credentials: "include",
+            });
+        } else {
+            response = await fetch(url.toString(), {
+                method: "POST",
+                body: json,
+                credentials: "include",
+            });
+        }
+
+        let compressionRatio = 1;
+        let uncompressedLength = response.headers.get("X-Uncompressed-Content-Length");
+        let compressedLength = response.headers.get("Content-Length");
+        if (uncompressedLength && compressedLength) {
+            compressionRatio = +uncompressedLength / +compressedLength;
+        }
+        let totalLength = +(uncompressedLength || compressedLength || 0);
+        if (totalLength) {
+            globalThis.onProgressHandler?.({
+                type: "Download",
+                addValue: 0,
+                addMax: Math.floor(totalLength / compressionRatio),
+            });
+        }
+
+        // If it's an error, set requestError
+        if (!response.ok) {
+            globalThis.onErrorHandler?.(response.statusText);
+        }
+
+        // Stream it, so we can get progrss
+        let reader = response.body!.getReader();
+        let result = "";
+        while (true) {
+            let { done, value } = await reader.read();
+            if (done) break;
+            result += new TextDecoder().decode(value);
+            let cur = (value?.length || 0) / compressionRatio;
+            globalThis.onProgressHandler?.({
+                type: "Download",
+                addValue: cur,
+                addMax: 0,
+            });
+        }
+
+        return result;
+    }
 
     const g = globalThis as any;
     let startTime = Date.now();
-    g.BOOT_TIME = startTime;
+    globalThis.BOOT_TIME = startTime;
 
     (Symbol as any).dispose = Symbol.dispose || Symbol("dispose");
     (Symbol as any).asyncDispose = Symbol.asyncDispose || Symbol("asyncDispose");
@@ -54,7 +129,8 @@ export function requireMain() {
         child_process: {},
         events: {},
     };
-    g.builtInModuleExports = builtInModuleExports;
+    globalThis.builtInModuleExports = globalThis.builtInModuleExports || {};
+    Object.assign(globalThis.builtInModuleExports, builtInModuleExports);
 
     let lastTime = 0;
     function nextTime() {
@@ -286,6 +362,12 @@ export function requireMain() {
             rootResolveCache[requests[i]] = requestsResolvedPaths[i];
         }
 
+        globalThis.onProgressHandler?.({
+            type: "Compile",
+            addValue: 0,
+            addMax: Object.keys(modules).length,
+        });
+
         // Store the function, so we only call it if it exists BEFORE we import
         //  (which means we already have something loading, so this is likely hot reloading...)
         let observerOnHotReload = g.observerOnHotReload;
@@ -300,13 +382,13 @@ export function requireMain() {
         let requireModuleCount = Object.values(modules).filter((x) => !x.source).length;
         let dependenciesOnlyText = requireModuleCount ? ` (+${requireModuleCount} dependencies only)` : "";
         console.log(
-            `%cimport(${requests.join(", ")}) finished download ${time}ms, ${Math.ceil(
+            `%cimport(${requests.join(", ")}) finished download ${time.toFixed(0)}ms, ${Math.ceil(
                 rawText.length / 1024
-            )}KB, ${moduleCount} modules${dependenciesOnlyText} at ${Date.now() - startTime}ms`,
-            "color: green"
+            )}KB, ${moduleCount} modules${dependenciesOnlyText} at ${(Date.now() - startTime).toFixed(0)}ms`,
+            "color: lightgreen"
         );
 
-        time = Date.now();
+
 
         if (alreadyHave?.requireSeqNumProcessId !== requireSeqNumProcessId) {
             alreadyHave = undefined;
@@ -319,13 +401,37 @@ export function requireMain() {
             serializedModules[id] = module;
         }
 
+        time = Date.now();
+        let lastWaitTime = time;
+        for (let key of Object.keys(modules)) {
+            getModule(key, "preload");
+            // Wait, so we can render progress (and in generals, so the UI remains somewhat responsive)
+            if (Date.now() - lastWaitTime > 10) {
+                await new Promise(resolve => setTimeout(resolve, 0));
+                lastWaitTime = Date.now();
+            }
+        }
+
+        time = Date.now() - time;
+        console.log(
+            `%cimport(${requests.join(", ")}) finished compile ${time.toFixed(0)}ms (${moduleCount} modules) at ${(Date.now() - startTime).toFixed(0)}ms`,
+            "color: hotpink"
+        );
+
+
+        globalThis.onProgressHandler?.({
+            type: "Evaluate",
+            addValue: 0,
+            addMax: Object.keys(modules).length,
+        });
+
+        time = Date.now();
         try {
             return requestsResolvedPaths.map((x) => getModule(x));
         } finally {
             time = Date.now() - time;
             console.log(
-                `%cimport(${requests.join(", ")}) finished evaluate ${time}ms (${moduleCount} modules) at ${Date.now() - startTime
-                }ms`,
+                `%cimport(${requests.join(", ")}) finished evaluate ${time.toFixed(0)}ms (${moduleCount} modules) at ${(Date.now() - startTime).toFixed(0)}ms`,
                 "color: lightblue"
             );
         }
@@ -392,7 +498,6 @@ export function requireMain() {
                         "color: red",
                         "color: unset"
                     );
-                    debugger;
                 }
                 return rootRequire(resolvedPath);
             }
@@ -495,12 +600,17 @@ export function requireMain() {
 
     let currentModuleEvaluationStack: string[] = [];
     // See https://nodejs.org/api/modules.html
-    function getModule(resolvedId: string) {
+    function getModule(resolvedId: string, preload?: "preload") {
         if (resolvedId === "") {
             return {};
         }
         if (resolvedId in moduleCache) {
-            return moduleCache[resolvedId];
+            let module = moduleCache[resolvedId];
+            if (!preload && !module.loaded) {
+                module.loaded = true;
+                module.load();
+            }
+            return module;
         }
 
         let serializedModule = serializedModules[resolvedId];
@@ -513,13 +623,25 @@ export function requireMain() {
         module.exports.default = module.exports;
         module.children = [];
         for (let key in serializedModule.flags || {}) {
+            if (key === "loaded") continue;
             module[key] = true;
         }
 
         module.load = load;
 
-        module.loaded = true;
-        module.load();
+        let originalSource = serializedModule.source;
+        let moduleFnc = wrapSafe(module.id, originalSource);
+
+        globalThis.onProgressHandler?.({
+            type: "Compile",
+            addValue: 1,
+            addMax: 0,
+        });
+
+        if (!preload) {
+            module.loaded = true;
+            module.load();
+        }
 
         function load() {
             let serializedModule = serializedModules[resolvedId];
@@ -560,7 +682,10 @@ export function requireMain() {
             module.size = source.length;
             module.source = source;
 
-            let moduleFnc = wrapSafe(module.id, source);
+            if (source !== originalSource) {
+                originalSource = source;
+                moduleFnc = wrapSafe(module.id, originalSource);
+            }
 
             let dirname = module.filename.replace(/\\/g, "/").split("/").slice(0, -1).join("/");
 
@@ -597,34 +722,15 @@ export function requireMain() {
             } finally {
                 module.isPreloading = false;
                 currentModuleEvaluationStack.pop();
+
+                globalThis.onProgressHandler?.({
+                    type: "Evaluate",
+                    addValue: 1,
+                    addMax: 0,
+                });
             }
         }
 
         return module;
     }
-
-    async function requestText(endpoint: string, values: any) {
-        let url = new URL(endpoint);
-
-        let json = JSON.stringify(values);
-        if (json.length < 6000) {
-            // NOTE: Try to use a GET, as GETs can be cached! However, if the data is too large,
-            //  we have to use a post, or else the request url will be too large
-            for (let key in values) {
-                url.searchParams.set(key, JSON.stringify(values[key]));
-            }
-            let response = await fetch(url.toString(), {
-                method: "GET",
-                credentials: "include",
-            });
-            return await response.text();
-        } else {
-            let response = await fetch(url.toString(), {
-                method: "POST",
-                body: json,
-                credentials: "include",
-            });
-            return await response.text();
-        }
-    }
-}
+}

package/src/callHTTPHandler.ts

@@ -61,6 +61,53 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
         // Don't keep alive, to prevent issues with zombie sockets.
         response.setHeader("Connection", "close");
 
+        // CORS bs (due to having to explictly allow subdomains)
+        {
+            response.setHeader("Strict-Transport-Security", "max-age=63072000; includeSubDomains; preload");
+            response.setHeader("Cross-Origin-Opener-Policy", "same-origin-allow-popups");
+            // NOTE: "credentialless" would work here too
+            response.setHeader("Cross-Origin-Embedder-Policy", "require-corp");
+
+            let origin = request.headers.origin || request.headers.referer;
+            let allowed = false;
+            if (!origin) {
+                // I guess it's a script, so just allow it (as it could easily set any header it wanted anyways)
+                allowed = true;
+                origin = "*";
+            } else {
+                let host = request.headers.host;
+                if (!host) {
+                    throw new Error("Missing host in request headers");
+                }
+                function rootDomain(hostname: string) {
+                    let parts = hostname.split(".");
+                    if (parts.length > 2) {
+                        return parts.slice(-2).join(".");
+                    }
+                    return hostname;
+                }
+                allowed = rootDomain(host) === rootDomain(origin);
+            }
+
+            // Allow if it has no origin, as that means it isn't a CORS request?
+            if (allowed) {
+                response.setHeader("Cross-Origin-Resource-Policy", "cross-origin");
+            } else {
+                response.setHeader("Cross-Origin-Resource-Policy", "same-site");
+            }
+
+            response.setHeader("vary", "Access-Control-Request-Headers");
+            response.setHeader("Access-Control-Allow-Origin", allowed ? origin : "");
+
+            if (allowed) {
+                response.setHeader("Access-Control-Allow-Credentials", "true");
+                response.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS");
+                response.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization, Content-Length, X-Requested-With, x-uncompressed-content-length, Cookie");
+            }
+            response.setHeader("Access-Control-Expose-Headers", "x-uncompressed-content-length");
+        }
+
+
         let urlBase = request.url;
         if (!urlBase) {
             throw new Error("Missing URL");
@@ -189,6 +236,7 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
                 return;
             }
         }
+        let uncompressedLength = resultBuffer.length;
         if (SocketFunction.HTTP_COMPRESS && request.headers["accept-encoding"]?.includes("gzip") && !headers?.["Content-Encoding"]) {
             // NOTE: This is a BIT slow. To speed it up, functions can use an internal cache, according to their function,
             //  and return a Buffer (which they can as any cast to make the returned type allowed, as returned Buffers will
@@ -205,6 +253,10 @@ export async function httpCallHandler(request: http.IncomingMessage, response: h
                 });
             });
         }
+        response.setHeader("Content-Length", resultBuffer.length.toString());
+        if (!response.getHeader("X-Uncompressed-Content-Length")) {
+            response.setHeader("X-Uncompressed-Content-Length", uncompressedLength.toString());
+        }
         response.write(resultBuffer);
         if (SocketFunction.logMessages) {
             console.log(`HTTP response  ${formatNumberSuffixed(resultBuffer.length)}B  (${request.method}) ${url}`);

package/temp.md.ai.temp

@@ -0,0 +1,41 @@
+{
+    "instructions": [
+        {
+            "messages": [
+                {
+                    "role": "user",
+                    "header": "# user",
+                    "content": "How secure is comparing the Host and Origin headers, checking if they have the same root domain, and then adding \"Access-Control-Allow-Origin\"\n\n",
+                    "slot": "",
+                    "chStart": 0,
+                    "chEnd": 7
+                }
+            ],
+            "params": {
+                "model": "openrouter-deepseek/deepseek-chat",
+                "max_tokens": 9999
+            },
+            "startIndex": 150,
+            "endIndex": 150
+        },
+        {
+            "messages": [
+                {
+                    "role": "user",
+                    "header": "# user",
+                    "content": "How secure is comparing the Host and Origin headers, checking if they have the same root domain, and then adding \"Access-Control-Allow-Origin\"\n\n",
+                    "slot": "",
+                    "chStart": 0,
+                    "chEnd": 7
+                }
+            ],
+            "params": {
+                "model": "openrouter-deepseek/deepseek-r1",
+                "max_tokens": 9999
+            },
+            "startIndex": 219,
+            "endIndex": 219
+        }
+    ],
+    "rawResult": "Comparing the `Host` and `Origin` headers to check if they have the same root domain and then dynamically setting the `Access-Control-Allow-Origin` header can be a reasonable approach to enforce cross-origin resource sharing (CORS) policies, but it has its nuances and potential pitfalls. Here's a detailed analysis of its security implications:\n\n### How It Works\n1. **Host Header**: Represents the domain of the server receiving the request.\n2. **Origin Header**: Represents the domain from which the request originates (e.g., in a cross-origin request).\n3. **Root Domain Comparison**: Check if the `Origin` and `Host` share the same root domain (e.g., `example.com` and `api.example.com`).\n4. **Access-Control-Allow-Origin**: If they match, set this header to the value of the `Origin` header to allow the request.\n\n### Security Considerations\n\n#### **1. Same-Origin Policy**\nThis approach ensures that only requests from the same root domain are allowed, which aligns with the same-origin policy. This can prevent unauthorized cross-origin requests.\n\n#### **2. Root Domain Matching**\nComparing the root domain ensures that subdomains of the same root domain are allowed. For example, `api.example.com` and `app.example.com` share the same root domain (`example.com`). This can be useful for microservices or APIs hosted on subdomains.\n\n#### **3. Potential Misconfigurations**\n- **Weak Root Domain Matching**: If the root domain matching logic is not strict, it could lead to vulnerabilities. For example, improperly parsing domains could allow `example.com.evil.com` to be treated as the same root domain.\n- **HTTP Header Spoofing**: If the server blindly trusts the `Origin` header, an attacker could spoof it to bypass CORS restrictions. Always validate the `Origin` header against a whitelist of allowed domains.\n\n#### **4. Security Best Practices**\n- **Whitelist Allowed Origins**: Instead of dynamically allowing any origin that matches the root domain, maintain a whitelist of trusted origins. This reduces the risk of allowing unintended domains.\n- **Validate Headers**: Ensure that the `Origin` and `Host` headers are properly validated to prevent spoofing or injection attacks.\n- **Use HTTPS**: Always enforce HTTPS to prevent man-in-the-middle attacks that could tamper with headers.\n- **Limit Scopes**: Only allow CORS for specific endpoints that require it, and restrict methods (e.g., `GET`, `POST`) and headers as needed.\n- **Avoid Wildcards**: Avoid using `Access-Control-Allow-Origin: *` unless absolutely necessary, as it allows access from any domain.\n\n#### **5. Additional Security Headers**\n- **Access-Control-Allow-Credentials**: Only set this to `true` if the request includes credentials (e.g., cookies) and you explicitly need to allow them.\n- **Vary: Origin**: Include this header to prevent caching issues with CORS responses.\n\n### Example Implementation (Pseudocode)\n```pseudocode\nif request.headers['Origin'] is not null:\n    root_domain_origin = extract_root_domain(request.headers['Origin'])\n    root_domain_host = extract_root_domain(request.headers['Host'])\n    \n    if root_domain_origin == root_domain_host:\n        response.headers['Access-Control-Allow-Origin'] = request.headers['Origin']\n        response.headers['Vary'] = 'Origin'\n    else:\n        deny_request()\nelse:\n    deny_request()\n```\n\n### Conclusion\nThis approach can be secure if implemented carefully, but it's essential to follow best practices like strict domain validation, whitelisting origins, and enforcing HTTPS. Always test your CORS implementation thoroughly to ensure it doesn't introduce vulnerabilities."
+}