querysub 0.2.0

package/src/-e-certs/EdgeCertController.ts

@@ -0,0 +1,288 @@
+/*
+    Maintains an HTTPS certificate which all nodes may use (for access of nodes in the browser)
+    - Also maintains the multi-ip A record which all nodes contribute to
+
+    IMPORTANT! HTTPS certificates have to be shared, and therefore we should (and do) only use them for the browser
+        (using SNI to determine if calls are browser or server calls).
+        - They have to be shared because letsencrypt has a limit on how many certificates you can generate per domain,
+            and our server count could easily exceed that limit (and we can't reuse certificates cross machine,
+            both for security, and for routing reasons).
+*/
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { lazy } from "socket-function/src/caching";
+import { createX509, generateKeyPair, getThreadKeyCert, parseCert } from "../-a-auth/certs";
+import { backoffRetryLoop, logErrors } from "../errors";
+import { getHTTPSKeyCert } from "./certAuthority";
+import { getControllerNodeId } from "../-g-core-values/NodeCapabilities";
+import * as forge from "node-forge";
+import { getNodeIdIP, getNodeIdLocation } from "socket-function/src/nodeCache";
+import { addRecord, deleteRecord, getRecords, hasDNSWritePermissions, setRecord } from "../-b-authorities/dnsAuthority";
+import { SocketServerConfig } from "socket-function/src/webSocketServer";
+import debugbreak from "debugbreak";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { getDomain, isNoNetwork } from "../config";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getExternalIP, testTCPIsListening } from "../misc/networking";
+import { magenta, yellow } from "socket-function/src/formatting/logColors";
+
+let publicPort = -1;
+
+// Figure out how to hook up our watch so it actually updates the underlying server
+export function getSNICerts(config: {
+    publicPort: number
+}): Exclude<SocketServerConfig["SNICerts"], undefined> {
+    if (config.publicPort > 0) {
+        if (publicPort !== -1 && publicPort !== config.publicPort) {
+            throw new Error(`getSNICerts called with different publicPort ${publicPort} and ${config.publicPort}. We require the same port, so we can correctly maintain the A record for the edge node.`);
+        }
+        publicPort = config.publicPort;
+    }
+    let certs: Exclude<SocketServerConfig["SNICerts"], undefined> = {
+        [getDomain()]: async (callback) => {
+            await EdgeCertController_watchHTTPSKeyCert(callback);
+        }
+    };
+    certs["noproxy." + getDomain()] = certs[getDomain()];
+    certs["127-0-0-1." + getDomain()] = certs[getDomain()];
+    return certs;
+}
+
+function createGetBaseCert() {
+    return async () => {
+        /*
+        // Code to create a self signed cert, for testing of expiration dates
+        let keyPair = generateKeyPair();
+        let certPair = createX509({
+            keyPair,
+            domain: rootCertDomain,
+            // Very short duration for testing
+            lifeSpan: 1000 * 60 * 15,
+            issuer: "self",
+        });
+        return { key: certPair.key.toString(), cert: certPair.cert.toString() };
+        //*/
+
+        return getHTTPSKeyCert(getDomain());
+    };
+}
+
+let getBaseCert = lazy(createGetBaseCert()) as () => Promise<{ key: string, cert: string }>;
+
+const certUpdateLoop = lazy(() => {
+    logErrors((async () => {
+        let firstLoop = true;
+        while (true) {
+            let curCert = await getCertFromRemote();
+            if (!curCert) {
+                throw new Error(`Internal error, certUpdateLoop called before lastPromise was set`);
+            }
+            let certObj = parseCert(curCert.cert);
+            // Get expiration date
+            let expirationTime = +new Date(certObj.validity.notAfter);
+            let createTime = +new Date(certObj.validity.notBefore);
+
+            // If 75% of the lifetime has passed, renew the cert
+            let renewDate = createTime + (expirationTime - createTime) * 0.75;
+            let timeToExpire = renewDate - Date.now();
+            if (timeToExpire < 0) {
+                console.log(`HTTPS certificate is looking too old. Renewing from remote.`);
+                getCertFromRemote = createGetCertFromRemote();
+                continue;
+            }
+
+            if (!firstLoop) {
+                for (let callback of callbacks) {
+                    callback(curCert);
+                }
+            }
+            firstLoop = false;
+
+            // Max timeout a signed integer, but lower is fine too
+            timeToExpire = Math.min(timeToExpire, 2 ** 30);
+            console.log(`Certicates up to date, renewing on ${new Date(Date.now() + timeToExpire)}`);
+            await delay(timeToExpire);
+            console.log(`Woke up to renew`);
+        }
+    })());
+});
+
+function createGetCertFromRemote() {
+    return lazy(async () => {
+        return backoffRetryLoop(async () => {
+            // Skip the remote call if we have DNS write permissions, to
+            //  make bootstrapping easier.
+            if (await hasDNSWritePermissions()) {
+                let ip = SocketFunction.mountedIP;
+                if (ip === "0.0.0.0") {
+                    ip = await getExternalIP();
+                }
+                return await getHTTPSKeyCertInner(ip);
+            }
+
+            let edgeNodeId = await getControllerNodeId(EdgeCertController);
+            if (!edgeNodeId) {
+                throw new Error("No EdgeCertController found");
+            }
+            return await EdgeCertController.nodes[edgeNodeId].getHTTPSKeyCert();
+        });
+    });
+}
+let getCertFromRemote = createGetCertFromRemote();
+
+// NOTE: Only works if the machine is already listening. Needed for some special websocket
+//  stuff related to debugging.
+export function debugGetRawEdgeCert() {
+    return getCertFromRemote();
+}
+
+let callbacks: ((newCertPair: { cert: string; key: string }) => void)[] = [];
+async function EdgeCertController_watchHTTPSKeyCert(
+    callback: (newCertPair: { cert: string; key: string }) => void
+) {
+    certUpdateLoop();
+
+    callbacks.push(callback);
+    let certPair = await getCertFromRemote();
+    callback(certPair);
+}
+
+/** NOTE: Any SocketFunction.mount calls should probably call this, otherwise they won't be able to be called. */
+export async function publishMachineARecords() {
+    if (!SocketFunction.isMounted()) {
+        throw new Error(`Must mount before publishing machine A records`);
+    }
+
+    let ip = SocketFunction.mountedIP;
+    if (ip === "0.0.0.0") {
+        ip = await getExternalIP();
+    }
+
+    let selfNodeId = SocketFunction.mountedNodeId;
+    let nodeObj = getNodeIdLocation(selfNodeId);
+    if (!nodeObj) throw new Error(`Invalid nodeId ${selfNodeId}`);
+    let machineAddress = nodeObj.address.split(".").slice(1).join(".");
+    if (ip === "127.0.0.1") {
+        let prev = await getRecords("A", machineAddress);
+        if (prev.length > 0 && prev[0] !== "127.0.0.1") {
+            // NOTE: We NEED to publish some records. HOWEVER, it is very likely they will run some services
+            //  and not set public (such as the FunctionRunner). SO... just ignore this, leaving the records as
+            //  public, even though the current run doesn't have public set.
+            if (process.argv[1].includes("server.ts")) {
+                console.log(yellow(`Current process is not marked as public, but machine previous had public services. NOT publishing A records to point to 127.0.0.1, which will make service inaccessible if the port is not forwarded open on the public ip.`));
+            }
+            return;
+        }
+    }
+    await setRecord("A", machineAddress, ip);
+    await setRecord("A", "*." + machineAddress, ip);
+}
+
+export const verifyServicesAlive = lazy(() => {
+
+    // NOTE: Our DNS TTL of 1 minute means we can't avoid downtime of 1 minute,
+    //  so shorter polling intervals will have very little benefit.
+    runInfinitePoll(1000 * 60, removeDeadARecords);
+});
+async function removeDeadARecords() {
+    if (isNoNetwork()) return;
+    if (publicPort === -1) return;
+    const edgeDomain = getDomain();
+    let ips = await getRecords("A", edgeDomain);
+    // Don't check 127.0.0.1, if we are in development this will still be set,
+    //  and it will be removed when we leave development
+    ips = ips.filter(ip => ip !== "127.0.0.1");
+
+    let results = await Promise.all(ips.map(async ip => {
+        return await testTCPIsListening(ip, publicPort);
+    }));
+
+    let deadIPs = ips.filter((ip, i) => !results[i]);
+    for (let deadIP of deadIPs) {
+        await deleteRecord("A", edgeDomain, deadIP);
+    }
+}
+
+async function getHTTPSKeyCertInner(callerIP: string) {
+    const edgeDomain = getDomain();
+    if (callerIP) {
+        let proxied = isNoNetwork() ? undefined : "proxied" as const;
+        try {
+            let promises: Promise<void>[] = [];
+            let existingIPs = await getRecords("A", edgeDomain);
+            if (callerIP === "127.0.0.1") {
+                if (existingIPs.length === 0) {
+                    promises.push(addRecord("A", edgeDomain, callerIP, proxied));
+                } else if (existingIPs.length === 1 && existingIPs[0] === "127.0.0.1") {
+                    // Good, do nothing
+                } else {
+                    // Don't remove the public servers, this is probably just us accidentally running a
+                    //  test script for a public domain.
+                    /*
+                    // Maybe they took down all the public servers?
+                    await removeDeadARecords();
+                    existingIPs = await getRecords("A", edgeDomain);
+                    if (existingIPs.length === 0) {
+                        await addRecord("A", edgeDomain, callerIP);
+                    } else {
+                        await addRecord("A", "127-0-0-1." + edgeDomain, "127.0.0.1");
+                        console.warn(`Tried to serve an edge node on the local domain, but SocketFunction.mount did NOT specify a public ip (ex, { ip: "0.0.0.0" }) AND there are already existing public servers. You can't load balance between real ips and 127.0.0.1! ${existingIPs.join(", ")}. You will need to use 127-0-0-1.${edgeDomain} to access the local server (instead of just ${edgeDomain}).`);
+                    }
+                    */
+                }
+            } else {
+                if (existingIPs.includes("127.0.0.1")) {
+                    console.log(magenta(`Switching from local development to production development (removing A record for 127.0.0.1, and using a real ip)`));
+                    promises.push(deleteRecord("A", edgeDomain, "127.0.0.1"));
+                }
+                promises.push(addRecord("A", edgeDomain, callerIP, proxied));
+            }
+            promises.push(addRecord("A", "noproxy." + edgeDomain, "127.0.0.1"));
+            promises.push(addRecord("A", "127-0-0-1." + edgeDomain, "127.0.0.1"));
+            // Add records in parallel, so we can wait for DNS propagation in parallel
+            await Promise.all(promises);
+        } catch (e) {
+            console.error(`Error updating DNS records, continuing without updating them`, e);
+        }
+    }
+
+    let cert = await getBaseCert();
+    // If the cert is 50% expired generate a new one
+    let certObj = parseCert(cert.cert);
+
+    // Get expiration date
+    let expirationTime = +new Date(certObj.validity.notAfter);
+    let createTime = +new Date(certObj.validity.notBefore);
+
+    // If 50% of the lifetime has passed, renew the cert
+    let renewDate = createTime + (expirationTime - createTime) * 0.5;
+    if (renewDate < Date.now()) {
+        console.log(`HTTPS certificate is looking too old, forcefully renewing`);
+        getHTTPSKeyCert.clear(getDomain());
+        getBaseCert = createGetBaseCert();
+        cert = await getBaseCert();
+    }
+
+    return cert;
+}
+
+class EdgeCertControllerBase {
+    public async getHTTPSKeyCert() {
+        // Ensure the a record is subscribed
+        const caller = SocketFunction.getCaller();
+        let callerIP = getNodeIdIP(caller.nodeId);
+        return await getHTTPSKeyCertInner(callerIP);
+    }
+}
+
+
+const EdgeCertController = SocketFunction.register(
+    "EdgeCertController-694c925b-fb10-4656-aed0-b53a48ded548",
+    new EdgeCertControllerBase(),
+    () => ({
+        getHTTPSKeyCert: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    })
+);

package/src/-e-certs/certAuthority.ts

@@ -0,0 +1,192 @@
+import acme from "acme-client";
+import { generateRSAKeyPair, parseCert, privateKeyToPem } from "../-a-auth/certs";
+import { cache, lazy } from "socket-function/src/caching";
+import { hasDNSWritePermissions, setRecord } from "../-b-authorities/dnsAuthority";
+import { magenta } from "socket-function/src/formatting/logColors";
+import { createKeyStore } from "../persistentLocalStore";
+import { getArchives } from "../-a-archives/archives";
+import { isNoNetwork } from "../config";
+import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import { delay } from "socket-function/src/batching";
+import { timeInMinute } from "socket-function/src/misc";
+
+const archives = lazy(() => getArchives(`https_certs_2/`));
+// Expire EXPIRATION_THRESHOLD% of the way through the certificate's lifetime
+const EXPIRATION_THRESHOLD = 0.4;
+
+export const getHTTPSKeyCert = cache(async (domain: string): Promise<{ key: string; cert: string }> => {
+    if (!await hasDNSWritePermissions()) {
+        throw new Error(`Cannot generate HTTPS key and cert unless we have DNS write permissions (need credentials in archives, at b2:/keys/cloudflare.json)`);
+    }
+    if (!domain.endsWith(".")) {
+        domain = domain + ".";
+    }
+    let keyCert: { key: string; cert: string } | undefined;
+
+    try {
+        let keyCertJSON = await archives().get(domain);
+        if (keyCertJSON) {
+            keyCert = JSON.parse(keyCertJSON.toString());
+        }
+    } catch { }
+    if (keyCert) {
+        // If 40% of the lifetime has passed, renew it (has to be < the threshold
+        //  in EdgeCertController).
+        let certObj = parseCert(keyCert.cert);
+        let expirationTime = +new Date(certObj.validity.notAfter);
+        let createTime = +new Date(certObj.validity.notBefore);
+        let renewDate = createTime + (expirationTime - createTime) * EXPIRATION_THRESHOLD;
+        if (renewDate < Date.now()) {
+            console.log(magenta(`Renewing domain ${domain} (renew target is ${formatDateTime(renewDate)}).`));
+            keyCert = undefined;
+        }
+    } else {
+        console.log(magenta(`No cert found for domain ${domain}, generating shortly.`));
+    }
+    if (keyCert) {
+        return keyCert;
+    }
+
+    const accountKey = getAccountKey();
+    let altDomains: string[] = [];
+
+    // altDomains.push("noproxy." + domain);
+    // // NOTE: Allowing local access is just an optimization, not to avoid having to forward ports
+    // //  (unless you type 127-0-0-1.domain into the browser... then I guess you don't have to forward ports?)
+    // altDomains.push("127-0-0-1." + domain);
+
+    // NOTE: I forget why we were not allowing wildcard domains. I think it was to prevent
+    //  any HTTPS domains from impersonating servers. But... servers have two levels, so that isn't
+    //  an issue. And even if they didn't they store their public key in their domain, so you
+    //  can't really impersonate them anyways...
+    altDomains.push("*." + domain);
+
+    let isPending = await archives().getInfo(domain + "_pending");
+    if (isPending && isPending.writeTime > Date.now() - timeInMinute * 10) {
+        let pendingTime = +isPending.toString();
+        // Wait 2 minutes
+        let waitTime = Math.max(0, pendingTime - Date.now() + timeInMinute * 2);
+        if (waitTime) {
+            console.log(`getHTTPSKeyCert pending in another process, waiting ${formatTime(waitTime)} for other process to create certificate`);
+            await delay(waitTime);
+            return await getHTTPSKeyCert(domain);
+        }
+    }
+    let pendingTime = Date.now().toString();
+    await archives().set(domain + "_pending", Buffer.from(pendingTime));
+    // Wait a bit, and then make sure we are the last to write to pending. This helps a lot with
+    //      race conditions (although they can still happen).
+    {
+        await delay(10 * 1000);
+        let pendingValue = await archives().get(domain + "_pending");
+        if (pendingValue && pendingValue.toString() !== pendingTime) {
+            return await getHTTPSKeyCert(domain);
+        }
+    }
+    try {
+        keyCert = await generateCert({ accountKey, domain, altDomains });
+        await archives().del(domain + "_pending");
+    } catch (e) {
+        if (String(e).includes("authorization must be pending")) {
+            console.log(`Authorization appears to be pending, waiting 2 minutes for other process to create certificate`);
+            await delay(timeInMinute * 2);
+            return await getHTTPSKeyCert(domain);
+        }
+        throw e;
+    }
+    await archives().set(domain, Buffer.from(JSON.stringify(keyCert)));
+    return keyCert;
+});
+
+let accountKeyCache = createKeyStore<{ key: string }>("letsEncryptAccountKey_1");
+let getAccountKey: () => string = lazy(() => {
+    let keyCache = accountKeyCache.value;
+    if (!keyCache) {
+        console.log(`Generating new letsencrypt account key`);
+        const keyPair = generateRSAKeyPair();
+        keyCache = { key: privateKeyToPem(keyPair.privateKey) };
+        accountKeyCache.value = keyCache;
+    }
+    return keyCache.key;
+});
+
+async function generateCert(config: {
+    accountKey: string;
+    domain: string;
+    altDomains?: string[];
+}): Promise<{
+    domains: string[];
+    key: string;
+    cert: string;
+}> {
+    let { accountKey, domain } = config;
+
+    console.log(magenta(`Generating new cert for ${domain}`));
+
+    let domainList = [domain, ...config.altDomains || []];
+    // Strip trailing "."
+    domainList = domainList.map(x => x.endsWith(".") ? x.slice(0, -1) : x);
+
+    const [certificateKey, certificateCsr] = await acme.forge.createCsr({
+        commonName: domainList[0],
+        altNames: domainList.slice(1),
+    });
+
+    // So... acme-client is fine. Just re-implement the "auto" mode ourselves, to have more control over it.
+    const client = new acme.Client({
+        directoryUrl: acme.directory.letsencrypt.production,
+        accountKey: accountKey,
+    });
+
+    const accountPayload = {
+        termsOfServiceAgreed: true,
+        contact: [`mailto:sliftist@gmail.com`],
+    };
+
+    try {
+        await client.getAccountUrl();
+    } catch {
+        await client.createAccount(accountPayload);
+    }
+
+    const orderPayload = {
+        identifiers: domainList.map(domain => ({ type: "dns", value: domain })),
+    };
+    const order = await client.createOrder(orderPayload);
+    const authorizations = await client.getAuthorizations(order);
+
+    for (let auth of authorizations) {
+        if (auth.status === "valid") {
+            console.log(`Authorization already valid for ${auth.identifier.value}`);
+            continue;
+        }
+        console.log(`Starting authorization for ${auth.identifier.value}`);
+
+        // Only use DNS authorization
+        let challenge = auth.challenges.find(x => x.type === "dns-01");
+        if (!challenge) {
+            throw new Error("No DNS challenge found");
+        }
+        const keyAuthorization = await client.getChallengeKeyAuthorization(challenge);
+
+        let hostname = auth.identifier.value;
+        let challengeRecordName = "_acme-challenge." + hostname + ".";
+        await setRecord("TXT", challengeRecordName, keyAuthorization);
+
+        await client.completeChallenge(challenge);
+        console.log(`Challenge completed`);
+
+        await client.waitForValidStatus(challenge);
+        console.log(`Status of order is valid`);
+    }
+
+    const finalized = await client.finalizeOrder(order, certificateCsr);
+    console.log(`Order finalized`);
+
+    let cert = await client.getCertificate(finalized);
+    return {
+        domains: domainList,
+        key: certificateKey.toString(),
+        cert: cert,
+    };
+}