sliftutils 1.7.125 → 1.7.126

package/bin/derivekey.js

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+"use strict";
+// Derives a second ed25519 key from an existing one, so one key can stand behind several
+// identities without any of them being stored.
+require("typenode");
+
+require("../security/keys/deriveKey").main().catch(e => {
+    console.error(`${e}`);
+    process.exitCode = 1;
+}).finally(() => process.exit());

package/bin/portsecuredaemon.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+"use strict";
+// The daemon that keeps a machine's root authorized_keys equal to its key repos. Installed and
+// started by securessh, and normally only run by systemd.
+//
+// Deliberately without the usual finally(process.exit) of the other entry points: this one is
+// meant to keep running after main resolves, and exiting there would stop it dead on startup.
+require("typenode");
+
+require("../security/authorizedKeys/daemon/daemon").main().catch(e => {
+    console.error(`portsecure: failed to start. ${e && e.stack || e}`);
+    process.exit(1);
+});

package/bin/unrevoke.js

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+"use strict";
+// Undoes revocations, by writing a file into the keys repo naming the ones to undo.
+require("typenode");
+
+require("../security/authorizedKeys/unrevoke").main().catch(e => {
+    console.error(`${e}`);
+    process.exitCode = 1;
+}).finally(() => process.exit());

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.125",
+    "version": "1.7.126",
     "main": "index.js",
     "license": "MIT",
     "files": [
@@ -31,7 +31,9 @@
         "autohost": "node ./bin/autohost.js",
         "setupnotify": "node ./bin/setupnotify.js",
         "securessh": "node ./bin/securessh.js",
-        "signfiles": "node ./bin/signfiles.js"
+        "signfiles": "node ./bin/signfiles.js",
+        "derivekey": "node ./bin/derivekey.js",
+        "unrevoke": "node ./bin/unrevoke.js"
     },
     "bin": {
         "filehoster": "./bin/filehoster.js",
@@ -51,7 +53,10 @@
         "sliftsetup": "./builders/setupRun.js",
         "setupnotify": "./bin/setupnotify.js",
         "securessh": "./bin/securessh.js",
-        "signfiles": "./bin/signfiles.js"
+        "signfiles": "./bin/signfiles.js",
+        "derivekey": "./bin/derivekey.js",
+        "portsecuredaemon": "./bin/portsecuredaemon.js",
+        "unrevoke": "./bin/unrevoke.js"
     },
     "dependencies": {
         "@types/chrome": "^0.0.237",

package/security/README.md

@@ -22,10 +22,18 @@ off every other way in.
     yarn securessh <host> add <repo-private-key> [repo-url]
     yarn securessh <host> remove [repo-url]
     yarn securessh <host> list
+    yarn securessh <host> update
 
 With no repo url, the repo you are standing in is used, as long as it holds keys and has an
 origin to clone from.
 
+The daemon runs on the host, so `update` is how a host picks up a newer build of it. It changes
+nothing about which keys that host trusts.
+
+`update` installs the daemon from this checkout, not from anything the host fetches, so it pulls
+this checkout first and stops if that pull does not fast forward. It never installs code it
+cannot account for.
+
 Each source repo keeps its own deploy key. Both verbs refuse to run if the key you log in with
 would not survive the change, since that would lock you out of the host permanently.
 
@@ -67,10 +75,66 @@ When a source starts being signed by a different key, the daemon warns on Discor
 applying the keys it last accepted. It applies the new ones only after 24 hours of that same new
 signer. Any different signer restarts the wait, so publishing twice in a row gains nothing, and a
 return to the accepted signer cancels it. Losing a signature entirely counts as a change too, so
-stripping it does not get anything through faster.
+stripping it does not get anything through faster. Going the other way, from unsigned to signed,
+is only ever an improvement and applies right away.
+
+Every one of those messages names the public key, in the same `type base64` form `signfiles`
+prints, or `<no public key>` when there is none.
 
 A signature that does not verify, or a manifest that does not match the files on disk, is never
-treated as an identity - that content is ignored and the last accepted keys stay.
+treated as an identity - that content is ignored and the last accepted keys stay. Which of the
+two it is gets reported:
+
+- the signature is byte for byte the one we already accepted, so the repo changed and nobody
+  re-signed it. The changes are ignored until someone runs `signfiles` again.
+- the signature did change and does not hold up, so it is corrupt.
+
+Either way it is reported once, not every time it is polled.
+
+## keys
+
+Derives a second ed25519 key from an existing one, by mixing a label into the source key's secret.
+The same label and source always give the same key, so a derived key is something you can work out
+again rather than something you have to keep a backup of.
+
+    yarn derivekey <label> <source-key> <derived-key>
+    yarn derivekey revokegithubkey ~/authorized_keys_access/id_ed25519 ~/authorized_keys_access/id_ed25519_revoke
+
+It writes an ordinary OpenSSH key pair, so the public key can be handed to anything that takes one.
+`deriveEd25519Key` in `deriveKey.ts` is the part to import elsewhere, and `sshKeyFile.ts` reads and
+writes the OpenSSH private key container that node itself cannot.
+
+### Revoking keys
+
+A key that is used from an address its `from=` restriction does not allow is revoked everywhere,
+not just refused. The daemon reads sshd's log for exactly that refusal, takes the fingerprint sshd
+names for the same connection, and writes a revocation to the source's revoke repo.
+
+The revoke repo is derived from the source: `…/authorized_keys.git` gets `…/authorized_keys_revoked.git`,
+reached with a key derived from the source's deploy key under the label `revokegithubkey`. Github
+will not take one public key on two repos, which is why it is derived rather than reused, and it
+means nothing extra has to be configured or uploaded - anything holding the source key can work it
+out. `securessh add` checks that repo exists and is writable, and prints the deploy key to add if
+it is not.
+
+- One revocation per key, ever. The file is named after the fingerprint, and the key is checked
+  against local state before any network work, so a flood of unknown keys cannot become a flood of
+  commits.
+- A revocation is sticky once seen. Deleting it from the repo does not bring the key back: the key
+  that writes revocations is on every server, so whoever stole one could otherwise erase the record
+  that locked them out. Recovering from that means making a new key, which you would want anyway.
+- Each machine says so on Discord when a revoked key actually leaves its authorized_keys, once.
+
+    yarn unrevoke [keys-repo]
+
+Run in the keys repo, or name one. It uses whatever git credentials the machine already has, since
+the derived deploy key is for servers rather than for people. It reads the revoke repo and writes
+one file under `unrevoked/` naming the revocations to undo, which then needs signing and pushing. Machines report when they see it, hold
+it for an hour, then report again when it takes effect - so a signing key that was itself stolen
+cannot instantly undo the revocation that shut it out.
+
+Deleting a revoked key from the repo is usually the right answer instead. Both `signfiles` and
+`securessh` refuse while a repo still holds a revoked key, and say which one it is.
 
 ## helpers
 

package/security/authorizedKeys/authorizedKeys.ts

@@ -1,3 +1,4 @@
+import crypto from "crypto";
 import fs from "fs/promises";
 import path from "path";
 
@@ -10,6 +11,28 @@ export function normalizeKeys(contents: string) {
     return contents.split("\n").map(line => line.trim()).filter(line => line && !line.startsWith("#"));
 }
 
+/** The fingerprint ssh itself reports for a key, which is what the sshd log names and therefore
+    what a revocation is keyed by. Returns "" for a line that holds no key. */
+export function keyFingerprint(keyLine: string) {
+    let parts = keyLine.trim().split(/\s+/);
+    let typeIndex = parts.findIndex(part => /^(ssh-|ecdsa-|sk-)/.test(part));
+    let blob = typeIndex >= 0 && parts[typeIndex + 1] || "";
+    if (!blob) {
+        return "";
+    }
+    return "SHA256:" + crypto.createHash("sha256").update(Buffer.from(blob, "base64")).digest("base64").replace(/=+$/, "");
+}
+
+/** The addresses a key may be used from, which is the part of an authorized_keys line that
+    decides how much a stolen key is worth. A key with no restriction says so loudly. */
+export function keyRestriction(keyLine: string) {
+    let match = keyLine.match(/from="([^"]*)"/);
+    if (!match) {
+        return "ANY ADDRESS (no from= restriction)";
+    }
+    return match[1];
+}
+
 /** Enough to recognise whose key this is without printing the whole blob. */
 export function summarizeKey(keyLine: string) {
     let parts = keyLine.trim().split(/\s+/);

package/security/authorizedKeys/daemon/authLog.ts

@@ -0,0 +1,117 @@
+import crypto from "crypto";
+import fs from "fs/promises";
+import { spawnPromise } from "../../helpers/spawn";
+import { AUTH_LOG_PATH } from "./paths";
+import { getState, saveState } from "./state";
+import { Attempt } from "./revocation";
+
+// sshd says an attempt was refused in one line and names the key in another, both for the same
+// connection, so they are tied together by the process id the log puts on every line.
+const REFUSED = /Authentication tried for (\S+) with correct key but not from a permitted host \(host=([^,]*), ip=([^,]*), required=([^)]*)\)/;
+const FAILED_KEY = /Failed publickey for (\S+) from (\S+) port (\d+) ssh2: \S+ (SHA256:[A-Za-z0-9+/=]+)/;
+const PROCESS_ID = /(?:sshd|sshd-session)\[(\d+)\]/;
+// Enough of the head of the file to notice it was rotated out from under us.
+const SIGNATURE_LENGTH = 512;
+
+export type RefusedAttempt = { fingerprint: string; attempt: Attempt };
+
+/** Pairs each refusal with the fingerprint sshd logged for the same connection. A refusal we
+    cannot tie to a key is dropped: revoking the wrong key would lock out the wrong person. */
+export function parseAuthLog(contents: string) {
+    let refusals = new Map<string, { user: string; ip: string; required: string; line: string }[]>();
+    let fingerprints = new Map<string, { fingerprint: string; port: string }>();
+    for (let line of contents.split("\n")) {
+        let processMatch = line.match(PROCESS_ID);
+        if (!processMatch) {
+            continue;
+        }
+        let processId = processMatch[1];
+        let refused = line.match(REFUSED);
+        if (refused) {
+            let existing = refusals.get(processId) || [];
+            existing.push({ user: refused[1], ip: refused[3], required: refused[4], line: line.trim() });
+            refusals.set(processId, existing);
+            continue;
+        }
+        let failed = line.match(FAILED_KEY);
+        if (failed) {
+            fingerprints.set(processId, { fingerprint: failed[4], port: failed[3] });
+        }
+    }
+
+    let attempts: RefusedAttempt[] = [];
+    for (let [processId, entries] of refusals) {
+        let key = fingerprints.get(processId);
+        if (!key) {
+            console.log(`A refused attempt named no key, so nothing is being revoked for it: ${entries[0].line}`);
+            continue;
+        }
+        for (let entry of entries) {
+            attempts.push({
+                fingerprint: key.fingerprint,
+                attempt: {
+                    ip: entry.ip,
+                    user: entry.user,
+                    port: key.port,
+                    required: entry.required,
+                    line: entry.line,
+                },
+            });
+        }
+    }
+    return attempts;
+}
+
+/** Only what has been written since last time. A rotated file starts again from the beginning,
+    and a file that only grew is read from where we stopped. */
+export async function readNewAuthLog() {
+    let state = getState();
+    let handle;
+    try {
+        handle = await fs.open(AUTH_LOG_PATH, "r");
+    } catch (e) {
+        // Some machines only keep this in the journal.
+        let result = await spawnPromise({
+            command: "journalctl",
+            args: ["-u", "ssh", "-u", "sshd", "--no-pager", "--since", "-10min"],
+        });
+        if (result.status !== 0) {
+            console.log(`No auth log to read: ${AUTH_LOG_PATH} is unreadable and journalctl exited ${result.status}`);
+            return "";
+        }
+        return result.stdout;
+    }
+    try {
+        let stats = await handle.stat();
+        let head = Buffer.alloc(Math.min(SIGNATURE_LENGTH, stats.size));
+        await handle.read(head, 0, head.length, 0);
+        let signature = crypto.createHash("sha256").update(head).digest("hex");
+
+        if (!state.authLogSignature) {
+            // First time we have ever looked. Start from the end: the log holds history from
+            // before this machine watched it, and revoking keys over refusals nobody was watching
+            // for could take away access that is still in use.
+            state.authLogOffset = stats.size;
+            state.authLogSignature = signature;
+            await saveState();
+            console.log(`Watching ${AUTH_LOG_PATH} from its current end, ${stats.size} bytes in`);
+            return "";
+        }
+        let offset = state.authLogOffset;
+        if (signature !== state.authLogSignature || stats.size < offset) {
+            // Rotated, or replaced. Everything in the new file is new.
+            offset = 0;
+        }
+        if (stats.size === offset) {
+            return "";
+        }
+        let contents = Buffer.alloc(stats.size - offset);
+        await handle.read(contents, 0, contents.length, offset);
+        state.authLogOffset = stats.size;
+        state.authLogSignature = signature;
+        await saveState();
+        return contents.toString("utf8");
+    } finally {
+        await handle.close();
+    }
+}

package/security/authorizedKeys/daemon/daemon.ts

@@ -0,0 +1,308 @@
+import fs from "fs/promises";
+import os from "os";
+import { configureDiscordNotifications, DEFAULT_WEBHOOK_FILE_PATH } from "../../notifications/discord";
+import { sourceKeyPath, sourceRepoPath } from "../sources";
+import { cloneRepo, syncRepo } from "./git";
+import {
+    CHECK_INTERVAL,
+    CONFIG_PATH,
+    MAX_REPO_FAILURES_BEFORE_RECLONE,
+    ROOT_AUTHORIZED_KEYS,
+} from "./paths";
+import { notify, setHostLabel } from "./notify";
+import { enforceRootKeys } from "./rootKeys";
+import { enforceSSHDConfig } from "./sshdConfig";
+import { getState, loadState, saveState, sourceState } from "./state";
+import { resolveSourceKeys } from "./trust";
+import { parseAuthLog, readNewAuthLog } from "./authLog";
+import { absorbRevocations, applyUnrevokes, recordRevocation, removeRevokedKeys, syncRevokeRepo } from "./revocation";
+import { keyFingerprint } from "../authorizedKeys";
+import { checkOtherUserKeys, seedUserKeys } from "./userKeys";
+
+// portsecure authorized-keys daemon.
+//
+// It owns root's authorized_keys: the contents come from one or more git repos, anything else is
+// reverted, and password authentication is turned off so those repos are the only way in.
+//
+// The complete list of things that send a Discord message. Nothing else may be added to it
+// without the user asking for that specific case - everything else goes to the log.
+//   1. root's authorized_keys was changed outside portsecure, and was reverted.
+//   2. root's authorized_keys was updated because a source changed.
+//   3. Another user's authorized_keys changed.
+//   4. A source's history was rewritten.
+//   5. A source started being signed by a different key, so its new keys are being held.
+//   6. A source is now signed when it was not before, applied right away.
+//   7. A source changed without its signature being updated, so the change is ignored.
+//   8. A source has a corrupted signature, so its contents are ignored.
+//   9. The webhook file itself changed, reported to the webhook being replaced.
+//  10. A key was revoked here, after being used from an address it is not allowed from.
+//  11. A revoked key was removed from root's authorized_keys, said once per key.
+//  12. An unrevoke was published, and is being held for an hour.
+//  13. An unrevoke was applied once that hour passed.
+
+export type DaemonConfig = {
+    repoSources: string[];
+    hostLabel: string;
+};
+
+let config: DaemonConfig = { repoSources: [], hostLabel: "" };
+let repoFailureCounts: { [repoURL: string]: number } = {};
+
+export function getConfig() {
+    return config;
+}
+
+export function setConfig(value: DaemonConfig) {
+    config = value;
+    setHostLabel(value.hostLabel);
+}
+
+async function loadConfig(): Promise<DaemonConfig> {
+    let contents;
+    try {
+        contents = await fs.readFile(CONFIG_PATH, "utf8");
+    } catch (e) {
+        console.error(`portsecure: expected a config file at ${CONFIG_PATH}, ${e}`);
+        process.exit(1);
+    }
+    let parsed = JSON.parse(contents) as { repoSources?: string[]; hostLabel?: string };
+    if (!Array.isArray(parsed.repoSources)) {
+        console.error(`portsecure: expected a repoSources array in ${CONFIG_PATH}, was ${JSON.stringify(parsed.repoSources)}`);
+        process.exit(1);
+    }
+    for (let repoURL of parsed.repoSources) {
+        try {
+            await fs.access(sourceKeyPath(repoURL));
+        } catch (e) {
+            console.error(`portsecure: expected the private key for ${repoURL} at ${sourceKeyPath(repoURL)}, ${e}`);
+            process.exit(1);
+        }
+    }
+    return {
+        repoSources: parsed.repoSources,
+        // The machine knows its own name, the config only overrides it when a nicer label helps.
+        hostLabel: parsed.hostLabel || os.hostname(),
+    };
+}
+
+/** The union of every source, in source order, with duplicates dropped. A source that cannot be
+    read is skipped rather than emptying the merged set, so one broken repo cannot revoke the keys
+    that came from the others. */
+export async function readAllowedKeys() {
+    let keys: string[] = [];
+    let seen = new Set<string>();
+    for (let repoURL of config.repoSources) {
+        let sourceKeys: string[];
+        try {
+            sourceKeys = await resolveSourceKeys(repoURL);
+        } catch (e) {
+            console.log(`Skipping ${repoURL}, its checkout could not be read. ${e}`);
+            continue;
+        }
+        for (let key of sourceKeys) {
+            if (seen.has(key)) {
+                continue;
+            }
+            seen.add(key);
+            keys.push(key);
+        }
+    }
+    return keys;
+}
+
+/** Which source contributed a key, so its revocation is written to that source's revoke repo. */
+function sourceOfFingerprint(fingerprint: string) {
+    for (let repoURL of config.repoSources) {
+        if (sourceState(repoURL).acceptedKeys.some(key => keyFingerprint(key) === fingerprint)) {
+            return repoURL;
+        }
+    }
+    return config.repoSources[0] || "";
+}
+
+/** Anything sshd refused because of a from= restriction gets that key revoked everywhere. Reading
+    the log is cheap and local, and the fingerprint is checked against what we already revoked
+    before any network work happens.
+
+    A refusal is queued rather than acted on directly, because the log is read once and moves past:
+    if the revoke repo is unreachable at that moment, dropping the refusal would leave a key that
+    was misused accepted forever. The queue is retried until it is written down. */
+async function queueRefusedKeys(allowedKeys: string[]) {
+    let contents = await readNewAuthLog();
+    if (!contents.trim()) {
+        return;
+    }
+    let state = getState();
+    for (let found of parseAuthLog(contents)) {
+        // One key is revoked once, no matter how many addresses it was tried from.
+        if (state.revocations[found.fingerprint]) {
+            continue;
+        }
+        if (state.pendingRevocations.some(pending => pending.fingerprint === found.fingerprint)) {
+            continue;
+        }
+        let sourceURL = sourceOfFingerprint(found.fingerprint);
+        if (!sourceURL) {
+            console.log(`Nowhere to record the revocation of ${found.fingerprint}, no sources are configured`);
+            continue;
+        }
+        console.log(`Queued the revocation of ${found.fingerprint}, used from ${found.attempt.ip}`);
+        state.pendingRevocations.push({
+            fingerprint: found.fingerprint,
+            keyLine: allowedKeys.find(key => keyFingerprint(key) === found.fingerprint) || "",
+            sourceURL,
+            attempt: found.attempt,
+        });
+    }
+    await saveState();
+}
+
+/** Writes down everything queued that we have not managed to write down yet. */
+async function writeQueuedRevocations() {
+    let state = getState();
+    if (!state.pendingRevocations.length) {
+        return;
+    }
+    let remaining = [];
+    for (let pending of state.pendingRevocations) {
+        if (state.revocations[pending.fingerprint]) {
+            continue;
+        }
+        let recorded = await recordRevocation({ ...pending, hostLabel: config.hostLabel });
+        if (!recorded && !state.revocations[pending.fingerprint]) {
+            remaining.push(pending);
+        }
+    }
+    state.pendingRevocations = remaining;
+    await saveState();
+}
+
+/** Syncs one source. Returns whether the merged keys need reapplying. */
+async function pollSource(repoURL: string) {
+    let result;
+    try {
+        result = await syncRepo(repoURL);
+        repoFailureCounts[repoURL] = 0;
+    } catch (e) {
+        let failures = (repoFailureCounts[repoURL] || 0) + 1;
+        repoFailureCounts[repoURL] = failures;
+        console.log(`Sync of ${repoURL} failed (${failures} in a row). ${e}`);
+        if (failures < MAX_REPO_FAILURES_BEFORE_RECLONE) {
+            return false;
+        }
+        // Availability over tidiness: throw the working copy away and start again.
+        console.log(`Discarding the checkout of ${repoURL} and cloning from scratch`);
+        try {
+            await cloneRepo({ repoURL, repoPath: sourceRepoPath(repoURL), keyPath: sourceKeyPath(repoURL) });
+            repoFailureCounts[repoURL] = 0;
+            return true;
+        } catch (cloneError) {
+            console.log(`${repoURL} cannot be reached or cloned, its last known keys stay in place. ${cloneError}`);
+            return false;
+        }
+    }
+
+    if (result.historyRewritten) {
+        await notify(
+            `the history of \`${repoURL}\` was rewritten. Commit \`${result.previousSha.slice(0, 12)}\` is no`
+            + ` longer an ancestor of \`${result.remoteSha.slice(0, 12)}\`, so history was force pushed or`
+            + ` tampered with. The new state has been applied.`
+        );
+    }
+    if (!result.changed) {
+        return false;
+    }
+    sourceState(repoURL).lastSha = result.remoteSha;
+    return true;
+}
+
+async function everyCheck() {
+    let anyChanged = false;
+    for (let repoURL of config.repoSources) {
+        // One unreachable source must not stop the others from being checked.
+        try {
+            anyChanged = await pollSource(repoURL) || anyChanged;
+        } catch (e) {
+            console.log(`Polling ${repoURL} failed. ${e && (e as Error).stack || e}`);
+        }
+    }
+    if (anyChanged) {
+        await saveState();
+    }
+
+    // What other machines have revoked, and anything published to undo a revocation.
+    for (let repoURL of config.repoSources) {
+        await syncRevokeRepo(repoURL);
+    }
+    await absorbRevocations(config.repoSources);
+    await applyUnrevokes(config.repoSources);
+
+    let mergedKeys = await readAllowedKeys();
+    await queueRefusedKeys(mergedKeys);
+    await writeQueuedRevocations();
+
+    // The repo is checked first, so a change that came from it is reported as an update rather
+    // than as somebody having edited the file locally.
+    await enforceRootKeys({ keys: await removeRevokedKeys(mergedKeys), reason: anyChanged && "repo" || "manual" });
+    await checkOtherUserKeys();
+    await enforceSSHDConfig();
+}
+
+function startInterval(config: { intervalTime: number; run: () => Promise<void>; name: string }) {
+    let { intervalTime, run, name } = config;
+    let running = false;
+    let tick = async () => {
+        if (running) {
+            console.log(`Skipping ${name}, the previous run has not finished`);
+            return;
+        }
+        running = true;
+        try {
+            await run();
+        } catch (e) {
+            // Every scheduled job swallows its own errors, the daemon must outlive any single one.
+            console.log(`${name} failed. ${e && (e as Error).stack || e}`);
+        }
+        running = false;
+    };
+    setInterval(tick, intervalTime);
+    return tick;
+}
+
+export async function main() {
+    setConfig(await loadConfig());
+    await loadState();
+    await configureDiscordNotifications({ filePath: DEFAULT_WEBHOOK_FILE_PATH });
+
+    console.log(`Starting, ${config.repoSources.length} source(s), keys applied to ${ROOT_AUTHORIZED_KEYS}`);
+
+    // A first pass has to happen before the intervals, so a machine is correct immediately after
+    // boot rather than a minute later.
+    for (let repoURL of config.repoSources) {
+        try {
+            sourceState(repoURL).lastSha = (await syncRepo(repoURL)).remoteSha;
+        } catch (e) {
+            console.log(`Initial sync of ${repoURL} failed, continuing with whatever is on disk. ${e}`);
+        }
+    }
+    await saveState();
+    await seedUserKeys();
+
+    for (let repoURL of config.repoSources) {
+        await syncRevokeRepo(repoURL);
+    }
+    await absorbRevocations(config.repoSources);
+    await enforceRootKeys({ keys: await removeRevokedKeys(await readAllowedKeys()), reason: "repo" });
+    await enforceSSHDConfig();
+
+    // configureDiscordNotifications watches the webhook file on its own, so there is nothing to
+    // schedule for it here.
+    startInterval({ name: "check", intervalTime: CHECK_INTERVAL, run: everyCheck });
+}
+
+process.on("uncaughtException", e => console.log(`Uncaught exception, staying up. ${e && e.stack || e}`));
+process.on("unhandledRejection", e => console.log(`Unhandled rejection, staying up. ${e}`));
+process.on("SIGTERM", () => {
+    console.log("Received SIGTERM, exiting");
+    process.exit(0);
+});