sliftutils 1.5.6 → 1.6.0

package/index.d.ts

@@ -150,6 +150,34 @@ declare module "sliftutils/misc/https/dns" {
     export declare function setRecord(type: string, key: string, value: string, proxied?: "proxied"): Promise<void>;
     /** Keeps existing records */
     export declare function addRecord(type: string, key: string, value: string, proxied?: "proxied"): Promise<void>;
+    /** Provide Cloudflare credentials directly (an API token, or a path to a file containing one), instead of relying on ./cloudflare.json */
+    export declare function setCloudflareCredentials(config: {
+        key?: string;
+        path?: string;
+    }): void;
+
+}
+
+declare module "sliftutils/misc/https/hostServer" {
+    export type HostServerConfig = {
+        /** Full domain to host on (e.g. "testsite.example.com"). The HTTPS cert is created for this domain and *.domain, so using a subdomain never touches the root domain (beyond its _acme-challenge TXT record). */
+        domain: string;
+        port: number;
+        /** Cloudflare API token (or a path to a file containing one). If neither is given, ./cloudflare.json is used. */
+        cloudflareApiToken?: string;
+        cloudflareApiTokenPath?: string;
+        /** Creates an unproxied A record pointing domain at this machine (publicIp, or our detected external IP) */
+        setDNSRecord?: boolean;
+        publicIp?: string;
+        allowHostnames?: string[];
+    };
+    /** Hosts a SocketFunction server on a real domain, with an automatically created and renewed Let's Encrypt HTTPS certificate (cached in the home folder, shared between processes on the machine). Expose your controllers (and any RequireController setup) before calling this. Returns the mounted nodeId. */
+    export declare function hostServer(config: HostServerConfig): Promise<string>;
+    /** Returns the cached HTTPS cert for the domain, creating/renewing it first if it is past this process's renewal threshold. Reads the disk cache on every call, so a renewal done by a parallel process is picked up instead of renewing again. */
+    export declare function getFreshHTTPSCert(domain: string): Promise<{
+        key: string;
+        cert: string;
+    }>;
 
 }
 

package/misc/https/dns.d.ts

@@ -11,3 +11,8 @@ export declare function deleteRecord(type: string, key: string, value: string):
 export declare function setRecord(type: string, key: string, value: string, proxied?: "proxied"): Promise<void>;
 /** Keeps existing records */
 export declare function addRecord(type: string, key: string, value: string, proxied?: "proxied"): Promise<void>;
+/** Provide Cloudflare credentials directly (an API token, or a path to a file containing one), instead of relying on ./cloudflare.json */
+export declare function setCloudflareCredentials(config: {
+    key?: string;
+    path?: string;
+}): void;

package/misc/https/dns.ts

@@ -114,16 +114,32 @@ export async function addRecord(type: string, key: string, value: string, proxie
 }
 
 
-const getCloudflareCreds = lazy(async (): Promise<{ key: string; }> => {
+let credsOverride: { key: string } | undefined;
+/** Provide Cloudflare credentials directly (an API token, or a path to a file containing one), instead of relying on ./cloudflare.json */
+export function setCloudflareCredentials(config: { key?: string; path?: string }) {
+    let key = config.key;
+    if (!key && config.path) {
+        key = fs.readFileSync(config.path, "utf8").trim();
+    }
+    if (!key) {
+        throw new Error(`Must provide either key or path in setCloudflareCredentials, received ${JSON.stringify(Object.keys(config))}`);
+    }
+    credsOverride = { key };
+}
+
+const getCloudflareCredsFromFile = lazy(async (): Promise<{ key: string; }> => {
     const path = "cloudflare.json";
     if (!fs.existsSync(path)) {
-        throw new Error(`Must add cloudflare.json file to root of project.`);
+        throw new Error(`Must add cloudflare.json file to root of project (or call setCloudflareCredentials).`);
     }
     let creds = JSON.parse(fs.readFileSync(path, "utf8")) as { key: string; };
     return {
         key: creds.key,
     };
 });
+async function getCloudflareCreds() {
+    return credsOverride || await getCloudflareCredsFromFile();
+}
 
 async function cloudflareGETCall<T>(path: string, params?: { [key: string]: string }): Promise<T> {
     let url = new URL(`https://api.cloudflare.com/client/v4` + path);

package/misc/https/hostServer.d.ts

@@ -0,0 +1,19 @@
+export type HostServerConfig = {
+    /** Full domain to host on (e.g. "testsite.example.com"). The HTTPS cert is created for this domain and *.domain, so using a subdomain never touches the root domain (beyond its _acme-challenge TXT record). */
+    domain: string;
+    port: number;
+    /** Cloudflare API token (or a path to a file containing one). If neither is given, ./cloudflare.json is used. */
+    cloudflareApiToken?: string;
+    cloudflareApiTokenPath?: string;
+    /** Creates an unproxied A record pointing domain at this machine (publicIp, or our detected external IP) */
+    setDNSRecord?: boolean;
+    publicIp?: string;
+    allowHostnames?: string[];
+};
+/** Hosts a SocketFunction server on a real domain, with an automatically created and renewed Let's Encrypt HTTPS certificate (cached in the home folder, shared between processes on the machine). Expose your controllers (and any RequireController setup) before calling this. Returns the mounted nodeId. */
+export declare function hostServer(config: HostServerConfig): Promise<string>;
+/** Returns the cached HTTPS cert for the domain, creating/renewing it first if it is past this process's renewal threshold. Reads the disk cache on every call, so a renewal done by a parallel process is picked up instead of renewing again. */
+export declare function getFreshHTTPSCert(domain: string): Promise<{
+    key: string;
+    cert: string;
+}>;

package/misc/https/hostServer.ts

@@ -0,0 +1,132 @@
+import os from "os";
+import fs from "fs";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { timeInMinute } from "socket-function/src/misc";
+import { delay } from "socket-function/src/batching";
+import { getExternalIP } from "socket-function/src/networking";
+import { magenta } from "socket-function/src/formatting/logColors";
+import { getThreadKeyCert, loadIdentityCA } from "./certs";
+import { generateCert, getAccountKey, parseCert } from "./httpsCerts";
+import { setCloudflareCredentials, setRecord } from "./dns";
+
+// Renew somewhere randomly between 40% and 60% of the way through the cert lifetime. The random threshold staggers parallel processes on the same machine, so usually one process renews early, and the others see the renewed cert on disk (we re-read the disk before every renewal check) and never renew themselves.
+const RENEW_THRESHOLD_MIN = 0.4;
+const RENEW_THRESHOLD_MAX = 0.6;
+const CERT_CHECK_INTERVAL = timeInMinute * 15;
+
+// One threshold per process, so a process is consistently early or late relative to its siblings
+const renewThreshold = RENEW_THRESHOLD_MIN + Math.random() * (RENEW_THRESHOLD_MAX - RENEW_THRESHOLD_MIN);
+
+export type HostServerConfig = {
+    /** Full domain to host on (e.g. "testsite.example.com"). The HTTPS cert is created for this domain and *.domain, so using a subdomain never touches the root domain (beyond its _acme-challenge TXT record). */
+    domain: string;
+    port: number;
+    /** Cloudflare API token (or a path to a file containing one). If neither is given, ./cloudflare.json is used. */
+    cloudflareApiToken?: string;
+    cloudflareApiTokenPath?: string;
+    /** Creates an unproxied A record pointing domain at this machine (publicIp, or our detected external IP) */
+    setDNSRecord?: boolean;
+    publicIp?: string;
+    allowHostnames?: string[];
+};
+
+/** Hosts a SocketFunction server on a real domain, with an automatically created and renewed Let's Encrypt HTTPS certificate (cached in the home folder, shared between processes on the machine). Expose your controllers (and any RequireController setup) before calling this. Returns the mounted nodeId. */
+export async function hostServer(config: HostServerConfig): Promise<string> {
+    let { domain, port } = config;
+    if (config.cloudflareApiToken || config.cloudflareApiTokenPath) {
+        setCloudflareCredentials({ key: config.cloudflareApiToken, path: config.cloudflareApiTokenPath });
+    }
+    // The identity CA always lives on the root domain (nodeIds are threadHash.machineHash.root.tld)
+    let rootDomain = domain.split(".").slice(-2).join(".");
+    await loadIdentityCA(rootDomain);
+
+    if (config.setDNSRecord) {
+        let ip = config.publicIp || await getExternalIP();
+        await setRecord("A", domain, ip);
+    }
+
+    let keyCert = await getFreshHTTPSCert(domain);
+    let certListeners: ((value: { key: string; cert: string }) => void)[] = [];
+    void runCertRenewalLoop(domain, newKeyCert => {
+        keyCert = newKeyCert;
+        for (let listener of certListeners) {
+            listener(newKeyCert);
+        }
+    });
+
+    let nodeId = await SocketFunction.mount({
+        public: true,
+        port,
+        ...getThreadKeyCert(rootDomain),
+        SNICerts: {
+            [domain]: callback => {
+                callback(keyCert);
+                certListeners.push(callback);
+            },
+        },
+        allowHostnames: config.allowHostnames,
+    });
+    console.log(magenta(`Hosting https://${domain}:${port} (nodeId ${nodeId})`));
+    return nodeId;
+}
+
+function getCertDiskPath(domain: string) {
+    return os.homedir() + `/httpscert_${domain}.json`;
+}
+function readCertFromDisk(domain: string): { key: string; cert: string } | undefined {
+    try {
+        return JSON.parse(fs.readFileSync(getCertDiskPath(domain), "utf8")) as { key: string; cert: string };
+    } catch {
+        return undefined;
+    }
+}
+function getRenewTime(certPem: string, threshold: number) {
+    let certObj = parseCert(certPem);
+    let start = +new Date(certObj.validity.notBefore);
+    let end = +new Date(certObj.validity.notAfter);
+    return start + (end - start) * threshold;
+}
+
+/** Returns the cached HTTPS cert for the domain, creating/renewing it first if it is past this process's renewal threshold. Reads the disk cache on every call, so a renewal done by a parallel process is picked up instead of renewing again. */
+export async function getFreshHTTPSCert(domain: string): Promise<{ key: string; cert: string }> {
+    let keyCert = readCertFromDisk(domain);
+    if (keyCert && getRenewTime(keyCert.cert, renewThreshold) > Date.now()) {
+        return keyCert;
+    }
+    if (keyCert) {
+        console.log(magenta(`HTTPS cert for ${domain} is past ${(renewThreshold * 100).toFixed(0)}% of its lifetime, renewing`));
+    } else {
+        console.log(magenta(`No HTTPS cert on disk for ${domain}, creating one`));
+    }
+    let accountKey = await getAccountKey(domain);
+    try {
+        keyCert = await generateCert({ accountKey, domain, altDomains: ["*." + domain] });
+    } catch (e) {
+        if (String(e).includes("authorization must be pending")) {
+            // Another process is mid-renewal. Wait for it to finish, then re-check the disk.
+            console.log(`Certificate authorization is pending in another process, waiting 2 minutes`);
+            await delay(timeInMinute * 2);
+            return await getFreshHTTPSCert(domain);
+        }
+        throw e;
+    }
+    fs.writeFileSync(getCertDiskPath(domain), JSON.stringify({ key: keyCert.key, cert: keyCert.cert }));
+    return { key: keyCert.key, cert: keyCert.cert };
+}
+
+async function runCertRenewalLoop(domain: string, onNewCert: (keyCert: { key: string; cert: string }) => void) {
+    let lastCert = readCertFromDisk(domain)?.cert;
+    while (true) {
+        await delay(CERT_CHECK_INTERVAL);
+        try {
+            let keyCert = await getFreshHTTPSCert(domain);
+            if (keyCert.cert !== lastCert) {
+                lastCert = keyCert.cert;
+                console.log(magenta(`HTTPS cert for ${domain} updated, applying to running server`));
+                onNewCert(keyCert);
+            }
+        } catch (e) {
+            console.error(`Failed to check/renew HTTPS cert for ${domain}`, e);
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.5.6",
+    "version": "1.6.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/render-utils/observer.tsx

@@ -1,3 +1,5 @@
+module.allowclient = true;
+
 import * as preact from "preact";
 import { observable, Reaction } from "mobx";
 import { measureBlock } from "socket-function/src/profiling/measure";

package/testsite/browser.tsx

@@ -0,0 +1,46 @@
+module.allowclient = true;
+
+import preact from "preact";
+import { observable } from "mobx";
+import { observer } from "../render-utils/observer";
+import { isNode } from "socket-function/src/misc";
+import { css } from "typesafecss";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { ServerInfoController } from "./serverInfoController";
+
+@observer
+class TestPage extends preact.Component {
+    synced = observable({
+        serverOSName: "",
+        error: "",
+    });
+
+    componentDidMount() {
+        void (async () => {
+            try {
+                let nodeId = SocketFunction.connect({ address: location.hostname, port: +location.port || 443 });
+                this.synced.serverOSName = await ServerInfoController.nodes[nodeId].getServerOSName();
+            } catch (e) {
+                this.synced.error = String(e);
+            }
+        })();
+    }
+
+    render() {
+        return <div className={css.vbox(8).pad2(16)}>
+            <div>sliftutils hostServer test page</div>
+            <div>
+                {this.synced.error && `Error: ${this.synced.error}`
+                    || this.synced.serverOSName && `Server OS (via SocketFunction): ${this.synced.serverOSName}`
+                    || "Asking the server for its OS name..."}
+            </div>
+        </div>;
+    }
+}
+
+async function main() {
+    if (isNode()) return;
+    preact.render(<TestPage />, document.body);
+}
+
+main().catch(console.error);

package/testsite/server.ts

@@ -0,0 +1,41 @@
+process.env.NODE_ENV = "production";
+import os from "os";
+import path from "path";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { RequireController } from "socket-function/require/RequireController";
+import { hostServer } from "../misc/https/hostServer";
+import { ServerInfoController } from "./serverInfoController";
+// Import browser code, so it is allowed to be required by the client
+import "./browser";
+
+const DOMAIN = "testsite.vidgridweb.com";
+const PORT = 4443;
+
+process.on("unhandledRejection", (error) => {
+    console.error("Unhandled promise rejection:", error);
+});
+process.on("uncaughtException", (error) => {
+    console.error("Uncaught exception:", error);
+});
+
+async function main() {
+    RequireController.allowAllNodeModules();
+    SocketFunction.expose(RequireController);
+    SocketFunction.expose(ServerInfoController);
+    SocketFunction.setDefaultHTTPCall(RequireController, "requireHTML", {
+        requireCalls: ["./testsite/browser.tsx"],
+    });
+    RequireController.addStaticRoot(path.resolve("."));
+
+    await hostServer({
+        domain: DOMAIN,
+        port: PORT,
+        cloudflareApiTokenPath: os.homedir() + "/vidgridweb.com.key",
+        setDNSRecord: true,
+    });
+}
+
+main().catch(e => {
+    console.error(e);
+    process.exit(1);
+});

package/testsite/serverInfoController.ts

@@ -0,0 +1,19 @@
+module.allowclient = true;
+
+import os from "os";
+import { SocketFunction } from "socket-function/SocketFunction";
+
+class ServerInfoControllerBase {
+    // Innocuous server-only information, to prove the browser is really talking to the server
+    async getServerOSName(): Promise<string> {
+        return `${os.type()} ${os.release()} (${os.platform()})`;
+    }
+}
+
+export const ServerInfoController = SocketFunction.register(
+    "ServerInfoController-7f3b1c2a",
+    new ServerInfoControllerBase(),
+    () => ({
+        getServerOSName: {},
+    })
+);