querysub 0.433.0 → 0.436.0

package/src/-e-certs/EdgeCertController.ts

@@ -1,252 +1,252 @@
-/*
-    Maintains an HTTPS certificate which all nodes may use (for access of nodes in the browser)
-    - Also maintains the multi-ip A record which all nodes contribute to
-
-    IMPORTANT! HTTPS certificates have to be shared, and therefore we should (and do) only use them for the browser
-        (using SNI to determine if calls are browser or server calls).
-        - They have to be shared because letsencrypt has a limit on how many certificates you can generate per domain,
-            and our server count could easily exceed that limit (and we can't reuse certificates cross machine,
-            both for security, and for routing reasons).
-*/
-
-import { SocketFunction } from "socket-function/SocketFunction";
-import { lazy } from "socket-function/src/caching";
-import { createX509, generateKeyPair, getMachineId, getOwnMachineId, getThreadKeyCert, parseCert } from "../-a-auth/certs";
-import { backoffRetryLoop, logErrors } from "../errors";
-import { getHTTPSKeyCert } from "./certAuthority";
-import { getControllerNodeId } from "../-g-core-values/NodeCapabilities";
-import * as forge from "node-forge";
-import { getNodeIdIP, getNodeIdLocation } from "socket-function/src/nodeCache";
-import { addRecord, deleteRecord, getRecords, hasDNSWritePermissions, setRecord } from "../-b-authorities/dnsAuthority";
-import { SocketServerConfig } from "socket-function/src/webSocketServer";
-import debugbreak from "debugbreak";
-import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { getDomain, isBootstrapOnly, isNoNetwork, isPublic } from "../config";
-import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
-import { getExternalIP, testTCPIsListening } from "../misc/networking";
-import { magenta, yellow } from "socket-function/src/formatting/logColors";
-import { timeInMinute, timeInSecond } from "socket-function/src/misc";
-import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
-import { shutdown } from "../diagnostics/periodic";
-import { formatDateTime } from "socket-function/src/formatting/format";
-
-let publicPort = -1;
-
-export const getHostedIP = lazy(async () => {
-    if (!isPublic()) {
-        return "127.0.0.1";
-    }
-    return await getExternalIP();
-});
-export const getIPDomain = lazy(async () => {
-    let ip = await getHostedIP();
-    return ip.replaceAll(".", "-") + "." + getDomain();
-});
-
-// Figure out how to hook up our watch so it actually updates the underlying server
-export async function getSNICerts(config: {
-    publicPort: number
-}): Promise<Exclude<SocketServerConfig["SNICerts"], undefined>> {
-    if (config.publicPort > 0) {
-        if (publicPort !== -1 && publicPort !== config.publicPort) {
-            throw new Error(`getSNICerts called with different publicPort ${publicPort} and ${config.publicPort}. We require the same port, so we can correctly maintain the A record for the edge node.`);
-        }
-        publicPort = config.publicPort;
-    }
-    let certs: Exclude<SocketServerConfig["SNICerts"], undefined> = {
-        [getDomain()]: async (callback) => {
-            await EdgeCertController_watchHTTPSKeyCert(callback);
-        },
-        [getOwnMachineId() + "." + getDomain()]: async (callback) => {
-            let threadCert = await getThreadKeyCert();
-            callback({
-                key: threadCert.key,
-                cert: threadCert.cert,
-            });
-        },
-    };
-    return certs;
-}
-
-const certUpdateLoop = lazy(() => {
-    logErrors((async () => {
-        let firstLoop = true;
-        while (true) {
-            let curCert = await getHTTPSKeyCert(getDomain());
-            if (!curCert) {
-                throw new Error(`Internal error, certUpdateLoop called before lastPromise was set`);
-            }
-            let certObj = parseCert(curCert.cert);
-            // Get expiration date
-            let expirationTime = +new Date(certObj.validity.notAfter);
-            let createTime = +new Date(certObj.validity.notBefore);
-
-            // If 75% of the lifetime has passed, getHTTPSKeyCert should have updated it, so wait until that, then get the new cert, and update our watchers (which should be the servers).
-            let renewDate = createTime + (expirationTime - createTime) * 0.75;
-            let timeToExpire = renewDate - Date.now();
-
-            if (!firstLoop) {
-                for (let callback of callbacks) {
-                    callback(curCert);
-                }
-            }
-            firstLoop = false;
-
-            if (timeToExpire < 0) {
-                console.warn(`getHTTPSKeyCert gave as an almost expired. It is not supposed to do this. It should have updated it by now... Expires on ${formatDateTime(expirationTime)}`);
-                timeToExpire = timeInMinute * 15;
-            }
-            // Max timeout a signed integer, but lower is fine too
-            timeToExpire = Math.min(Math.floor(timeToExpire), 2 ** 30);
-            await delay(timeToExpire);
-            console.log(`Woke up to propagate new certs`);
-        }
-    })());
-});
-
-
-
-let callbacks: ((newCertPair: { cert: string; key: string }) => void)[] = [];
-async function EdgeCertController_watchHTTPSKeyCert(
-    callback: (newCertPair: { cert: string; key: string }) => void
-) {
-    certUpdateLoop();
-
-    callbacks.push(callback);
-    let certPair = await getHTTPSKeyCert(getDomain());
-    callback(certPair);
-}
-
-/** NOTE: Any SocketFunction.mount calls should probably call this, otherwise they won't be able to be called. */
-export async function publishMachineARecords() {
-    if (!SocketFunction.isMounted()) {
-        throw new Error(`Must mount before publishing machine A records`);
-    }
-
-    let ip = await getHostedIP();
-
-    let selfNodeId = SocketFunction.mountedNodeId;
-    let nodeObj = getNodeIdLocation(selfNodeId);
-    if (!nodeObj) throw new Error(`Invalid nodeId ${selfNodeId}`);
-    let machineAddress = nodeObj.address.split(".").slice(1).join(".");
-    let prevMachineIP = await getRecords("A", machineAddress);
-    let ipDomain = await getIPDomain();
-    let promises: Promise<void>[] = [];
-    promises.push(setRecord("A", ipDomain, ip));
-
-
-    if (ip === "127.0.0.1" && prevMachineIP.length > 0 && !prevMachineIP.includes("127.0.0.1")) {
-        console.log(yellow(`Not setting A record for ${machineAddress} to ${ip}, as we previously had a public IP. IF you want to switch back to 127.0.0.1, manually go in and delete the A records for ${machineAddress}. Port forwarding should allow this to work anyways, and the bootstrapper should be smart enough to try 127-0-0-1 style addresses to allow fast development (ex, if you want to download a large file from the local development server quickly).`));
-    } else {
-        promises.push(setRecord("A", machineAddress, ip));
-        promises.push(setRecord("A", "*." + machineAddress, ip));
-    }
-
-    promises.push(publishEdgeDomain());
-    await Promise.all(promises);
-
-    return {
-        ip,
-        ipDomain,
-    };
-}
-
-const runEdgeDomainAliveLoop = lazy(() => {
-    // NOTE: Our DNS TTL is 1 minute, which means no matter how fast we poll,
-    //  we can't get below that. Of course the worst case is that + our poll rate,
-    //  but still, this means there is less and less benefit the lower this value is.
-    runInfinitePoll(timeInMinute * 3, checkEdgeDomainsAlive);
-});
-async function checkEdgeDomainsAlive() {
-    if (isNoNetwork()) return;
-    if (publicPort === -1) return;
-    // Only check once we are actually running on real sites (not just developer sites, which
-    //  use non-port 443)
-    if (publicPort !== 443) return;
-    const edgeDomain = getDomain();
-    let ips = await getRecords("A", edgeDomain);
-    // Don't check 127.0.0.1, if we are in development this will still be set,
-    //  and it will be removed when we leave development
-    ips = ips.filter(ip => ip !== "127.0.0.1");
-
-    let results = await Promise.all(ips.map(async ip => {
-        // It must be not listening for 3 times in a row
-        for (let i = 0; i < 3; i++) {
-            let isListening = await testTCPIsListening(ip, publicPort);
-            if (isListening) return true;
-            let areWeOnline = await testTCPIsListening("1.1.1.1", publicPort);
-            if (!areWeOnline) {
-                // NOTE: I mean, while this case can happen, it also won't matter because we can't really delete the A record if we're not on the internet. I guess it's useful if we're offline when we check the IPs, but then we come back online right before we delete the records. But I doubt that will happen.t
-                console.warn(`Ignoring down ip, as we cannot even connect to 1.1.1.1:${publicPort}, so the other node isn't down, we're down!`);
-                return true;
-            }
-            console.warn(`IP ${ip}:${publicPort} is not listening, waiting 10 seconds before retrying. If it fails 3 times in a row, the A record will be removed.`);
-            await delay(timeInSecond * 10);
-        }
-        return false;
-    }));
-
-    let deadIPs = ips.filter((ip, i) => !results[i]);
-    if (deadIPs.length > 0) {
-        console.error(`Found dead IPs for ${edgeDomain}:${publicPort}, removing their A records: ${JSON.stringify(deadIPs)}`);
-    }
-    for (let deadIP of deadIPs) {
-        await deleteRecord("A", edgeDomain, deadIP);
-    }
-}
-
-async function publishEdgeDomain() {
-    let callerIP = SocketFunction.mountedIP;
-    if (callerIP === "0.0.0.0") {
-        callerIP = await getExternalIP();
-    }
-    if (!callerIP) {
-        console.warn(`publishEdgeDomain called before SocketFunction.mount was called, so we don't know the public ip.`);
-        return;
-    }
-    // IMPORTANT! We have to set our A record AFTER we create our cert, otherwise we might wait a while
-    //  with our A record public while we create our cert.
-    runEdgeDomainAliveLoop();
-    const edgeDomain = getDomain();
-    console.log(`Starting publish check for A record for ${edgeDomain} to ${callerIP}`);
-    try {
-        let promises: Promise<void>[] = [];
-        let existingIPs = await getRecords("A", edgeDomain);
-        if (!isPublic()) {
-            if (existingIPs.length === 0) {
-                promises.push(addRecord("A", edgeDomain, callerIP));
-            } else if (existingIPs.length === 1 && existingIPs[0] === "127.0.0.1") {
-                // Good, do nothing
-            } else {
-                // Don't remove the public servers, this is probably just us accidentally running a
-                //  test script for a public domain.
-                /*
-                // Maybe they took down all the public servers?
-                await removeDeadARecords();
-                existingIPs = await getRecords("A", edgeDomain);
-                if (existingIPs.length === 0) {
-                    await addRecord("A", edgeDomain, callerIP);
-                } else {
-                    await addRecord("A", "127-0-0-1." + edgeDomain, "127.0.0.1");
-                    console.warn(`Tried to serve an edge node on the local domain, but SocketFunction.mount did NOT specify a public ip (ex, { ip: "0.0.0.0" }) AND there are already existing public servers. You can't load balance between real ips and 127.0.0.1! ${existingIPs.join(", ")}. You will need to use 127-0-0-1.${edgeDomain} to access the local server (instead of just ${edgeDomain}).`);
-                }
-                */
-                console.log(yellow(`Current process is not marked as public, but machine previous had public services. NOT setting ${edgeDomain} to 127.0.0.1, as this would make the public services inaccessible. Assuming the current process is for development, I recommend using 127-0-0-1.${edgeDomain} or 127-0-0-1.${edgeDomain} to access the server.`));
-            }
-        } else {
-            if (existingIPs.includes("127.0.0.1")) {
-                console.log(magenta(`Switching from local development to production development (removing A record for 127.0.0.1, and using a real ip)`));
-                promises.push(deleteRecord("A", edgeDomain, "127.0.0.1"));
-            }
-            if (isBootstrapOnly()) {
-                promises.push(addRecord("A", edgeDomain, callerIP, "proxied"));
-            }
-        }
-        promises.push(addRecord("A", "127-0-0-1." + edgeDomain, "127.0.0.1"));
-        // Add records in parallel, so we can wait for DNS propagation in parallel
-        await Promise.all(promises);
-    } catch (e) {
-        console.error(`Error updating DNS records, continuing without updating them`, e);
-    }
-}
-
+/*
+    Maintains an HTTPS certificate which all nodes may use (for access of nodes in the browser)
+    - Also maintains the multi-ip A record which all nodes contribute to
+
+    IMPORTANT! HTTPS certificates have to be shared, and therefore we should (and do) only use them for the browser
+        (using SNI to determine if calls are browser or server calls).
+        - They have to be shared because letsencrypt has a limit on how many certificates you can generate per domain,
+            and our server count could easily exceed that limit (and we can't reuse certificates cross machine,
+            both for security, and for routing reasons).
+*/
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { lazy } from "socket-function/src/caching";
+import { createX509, generateKeyPair, getMachineId, getOwnMachineId, getThreadKeyCert, parseCert } from "../-a-auth/certs";
+import { backoffRetryLoop, logErrors } from "../errors";
+import { getHTTPSKeyCert } from "./certAuthority";
+import { getControllerNodeId } from "../-g-core-values/NodeCapabilities";
+import * as forge from "node-forge";
+import { getNodeIdIP, getNodeIdLocation } from "socket-function/src/nodeCache";
+import { addRecord, deleteRecord, getRecords, hasDNSWritePermissions, setRecord } from "../-b-authorities/dnsAuthority";
+import { SocketServerConfig } from "socket-function/src/webSocketServer";
+import debugbreak from "debugbreak";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { getDomain, isBootstrapOnly, isNoNetwork, isPublic } from "../config";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getExternalIP, testTCPIsListening } from "../misc/networking";
+import { magenta, yellow } from "socket-function/src/formatting/logColors";
+import { timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
+import { shutdown } from "../diagnostics/periodic";
+import { formatDateTime } from "socket-function/src/formatting/format";
+
+let publicPort = -1;
+
+export const getHostedIP = lazy(async () => {
+    if (!isPublic()) {
+        return "127.0.0.1";
+    }
+    return await getExternalIP();
+});
+export const getIPDomain = lazy(async () => {
+    let ip = await getHostedIP();
+    return ip.replaceAll(".", "-") + "." + getDomain();
+});
+
+// Figure out how to hook up our watch so it actually updates the underlying server
+export async function getSNICerts(config: {
+    publicPort: number
+}): Promise<Exclude<SocketServerConfig["SNICerts"], undefined>> {
+    if (config.publicPort > 0) {
+        if (publicPort !== -1 && publicPort !== config.publicPort) {
+            throw new Error(`getSNICerts called with different publicPort ${publicPort} and ${config.publicPort}. We require the same port, so we can correctly maintain the A record for the edge node.`);
+        }
+        publicPort = config.publicPort;
+    }
+    let certs: Exclude<SocketServerConfig["SNICerts"], undefined> = {
+        [getDomain()]: async (callback) => {
+            await EdgeCertController_watchHTTPSKeyCert(callback);
+        },
+        [getOwnMachineId() + "." + getDomain()]: async (callback) => {
+            let threadCert = await getThreadKeyCert();
+            callback({
+                key: threadCert.key,
+                cert: threadCert.cert,
+            });
+        },
+    };
+    return certs;
+}
+
+const certUpdateLoop = lazy(() => {
+    logErrors((async () => {
+        let firstLoop = true;
+        while (true) {
+            let curCert = await getHTTPSKeyCert(getDomain());
+            if (!curCert) {
+                throw new Error(`Internal error, certUpdateLoop called before lastPromise was set`);
+            }
+            let certObj = parseCert(curCert.cert);
+            // Get expiration date
+            let expirationTime = +new Date(certObj.validity.notAfter);
+            let createTime = +new Date(certObj.validity.notBefore);
+
+            // If 75% of the lifetime has passed, getHTTPSKeyCert should have updated it, so wait until that, then get the new cert, and update our watchers (which should be the servers).
+            let renewDate = createTime + (expirationTime - createTime) * 0.75;
+            let timeToExpire = renewDate - Date.now();
+
+            if (!firstLoop) {
+                for (let callback of callbacks) {
+                    callback(curCert);
+                }
+            }
+            firstLoop = false;
+
+            if (timeToExpire < 0) {
+                console.warn(`getHTTPSKeyCert gave as an almost expired. It is not supposed to do this. It should have updated it by now... Expires on ${formatDateTime(expirationTime)}`);
+                timeToExpire = timeInMinute * 15;
+            }
+            // Max timeout a signed integer, but lower is fine too
+            timeToExpire = Math.min(Math.floor(timeToExpire), 2 ** 30);
+            await delay(timeToExpire);
+            console.log(`Woke up to propagate new certs`);
+        }
+    })());
+});
+
+
+
+let callbacks: ((newCertPair: { cert: string; key: string }) => void)[] = [];
+async function EdgeCertController_watchHTTPSKeyCert(
+    callback: (newCertPair: { cert: string; key: string }) => void
+) {
+    certUpdateLoop();
+
+    callbacks.push(callback);
+    let certPair = await getHTTPSKeyCert(getDomain());
+    callback(certPair);
+}
+
+/** NOTE: Any SocketFunction.mount calls should probably call this, otherwise they won't be able to be called. */
+export async function publishMachineARecords() {
+    if (!SocketFunction.isMounted()) {
+        throw new Error(`Must mount before publishing machine A records`);
+    }
+
+    let ip = await getHostedIP();
+
+    let selfNodeId = SocketFunction.mountedNodeId;
+    let nodeObj = getNodeIdLocation(selfNodeId);
+    if (!nodeObj) throw new Error(`Invalid nodeId ${selfNodeId}`);
+    let machineAddress = nodeObj.address.split(".").slice(1).join(".");
+    let prevMachineIP = await getRecords("A", machineAddress);
+    let ipDomain = await getIPDomain();
+    let promises: Promise<void>[] = [];
+    promises.push(setRecord("A", ipDomain, ip));
+
+
+    if (ip === "127.0.0.1" && prevMachineIP.length > 0 && !prevMachineIP.includes("127.0.0.1")) {
+        console.log(yellow(`Not setting A record for ${machineAddress} to ${ip}, as we previously had a public IP. IF you want to switch back to 127.0.0.1, manually go in and delete the A records for ${machineAddress}. Port forwarding should allow this to work anyways, and the bootstrapper should be smart enough to try 127-0-0-1 style addresses to allow fast development (ex, if you want to download a large file from the local development server quickly).`));
+    } else {
+        promises.push(setRecord("A", machineAddress, ip));
+        promises.push(setRecord("A", "*." + machineAddress, ip));
+    }
+
+    promises.push(publishEdgeDomain());
+    await Promise.all(promises);
+
+    return {
+        ip,
+        ipDomain,
+    };
+}
+
+const runEdgeDomainAliveLoop = lazy(() => {
+    // NOTE: Our DNS TTL is 1 minute, which means no matter how fast we poll,
+    //  we can't get below that. Of course the worst case is that + our poll rate,
+    //  but still, this means there is less and less benefit the lower this value is.
+    runInfinitePoll(timeInMinute * 3, checkEdgeDomainsAlive);
+});
+async function checkEdgeDomainsAlive() {
+    if (isNoNetwork()) return;
+    if (publicPort === -1) return;
+    // Only check once we are actually running on real sites (not just developer sites, which
+    //  use non-port 443)
+    if (publicPort !== 443) return;
+    const edgeDomain = getDomain();
+    let ips = await getRecords("A", edgeDomain);
+    // Don't check 127.0.0.1, if we are in development this will still be set,
+    //  and it will be removed when we leave development
+    ips = ips.filter(ip => ip !== "127.0.0.1");
+
+    let results = await Promise.all(ips.map(async ip => {
+        // It must be not listening for 3 times in a row
+        for (let i = 0; i < 3; i++) {
+            let isListening = await testTCPIsListening(ip, publicPort);
+            if (isListening) return true;
+            let areWeOnline = await testTCPIsListening("1.1.1.1", publicPort);
+            if (!areWeOnline) {
+                // NOTE: I mean, while this case can happen, it also won't matter because we can't really delete the A record if we're not on the internet. I guess it's useful if we're offline when we check the IPs, but then we come back online right before we delete the records. But I doubt that will happen.t
+                console.warn(`Ignoring down ip, as we cannot even connect to 1.1.1.1:${publicPort}, so the other node isn't down, we're down!`);
+                return true;
+            }
+            console.warn(`IP ${ip}:${publicPort} is not listening, waiting 10 seconds before retrying. If it fails 3 times in a row, the A record will be removed.`);
+            await delay(timeInSecond * 10);
+        }
+        return false;
+    }));
+
+    let deadIPs = ips.filter((ip, i) => !results[i]);
+    if (deadIPs.length > 0) {
+        console.error(`Found dead IPs for ${edgeDomain}:${publicPort}, removing their A records: ${JSON.stringify(deadIPs)}`);
+    }
+    for (let deadIP of deadIPs) {
+        await deleteRecord("A", edgeDomain, deadIP);
+    }
+}
+
+async function publishEdgeDomain() {
+    let callerIP = SocketFunction.mountedIP;
+    if (callerIP === "0.0.0.0") {
+        callerIP = await getExternalIP();
+    }
+    if (!callerIP) {
+        console.warn(`publishEdgeDomain called before SocketFunction.mount was called, so we don't know the public ip.`);
+        return;
+    }
+    // IMPORTANT! We have to set our A record AFTER we create our cert, otherwise we might wait a while
+    //  with our A record public while we create our cert.
+    runEdgeDomainAliveLoop();
+    const edgeDomain = getDomain();
+    console.log(`Starting publish check for A record for ${edgeDomain} to ${callerIP}`);
+    try {
+        let promises: Promise<void>[] = [];
+        let existingIPs = await getRecords("A", edgeDomain);
+        if (!isPublic()) {
+            if (existingIPs.length === 0) {
+                promises.push(addRecord("A", edgeDomain, callerIP));
+            } else if (existingIPs.length === 1 && existingIPs[0] === "127.0.0.1") {
+                // Good, do nothing
+            } else {
+                // Don't remove the public servers, this is probably just us accidentally running a
+                //  test script for a public domain.
+                /*
+                // Maybe they took down all the public servers?
+                await removeDeadARecords();
+                existingIPs = await getRecords("A", edgeDomain);
+                if (existingIPs.length === 0) {
+                    await addRecord("A", edgeDomain, callerIP);
+                } else {
+                    await addRecord("A", "127-0-0-1." + edgeDomain, "127.0.0.1");
+                    console.warn(`Tried to serve an edge node on the local domain, but SocketFunction.mount did NOT specify a public ip (ex, { ip: "0.0.0.0" }) AND there are already existing public servers. You can't load balance between real ips and 127.0.0.1! ${existingIPs.join(", ")}. You will need to use 127-0-0-1.${edgeDomain} to access the local server (instead of just ${edgeDomain}).`);
+                }
+                */
+                console.log(yellow(`Current process is not marked as public, but machine previous had public services. NOT setting ${edgeDomain} to 127.0.0.1, as this would make the public services inaccessible. Assuming the current process is for development, I recommend using 127-0-0-1.${edgeDomain} or 127-0-0-1.${edgeDomain} to access the server.`));
+            }
+        } else {
+            if (existingIPs.includes("127.0.0.1")) {
+                console.log(magenta(`Switching from local development to production development (removing A record for 127.0.0.1, and using a real ip)`));
+                promises.push(deleteRecord("A", edgeDomain, "127.0.0.1"));
+            }
+            if (isBootstrapOnly()) {
+                promises.push(addRecord("A", edgeDomain, callerIP, "proxied"));
+            }
+        }
+        promises.push(addRecord("A", "127-0-0-1." + edgeDomain, "127.0.0.1"));
+        // Add records in parallel, so we can wait for DNS propagation in parallel
+        await Promise.all(promises);
+    } catch (e) {
+        console.error(`Error updating DNS records, continuing without updating them`, e);
+    }
+}
+