sliftutils 1.7.124 → 1.7.126

package/security/authorizedKeys/daemon/daemon.ts

@@ -0,0 +1,308 @@
+import fs from "fs/promises";
+import os from "os";
+import { configureDiscordNotifications, DEFAULT_WEBHOOK_FILE_PATH } from "../../notifications/discord";
+import { sourceKeyPath, sourceRepoPath } from "../sources";
+import { cloneRepo, syncRepo } from "./git";
+import {
+    CHECK_INTERVAL,
+    CONFIG_PATH,
+    MAX_REPO_FAILURES_BEFORE_RECLONE,
+    ROOT_AUTHORIZED_KEYS,
+} from "./paths";
+import { notify, setHostLabel } from "./notify";
+import { enforceRootKeys } from "./rootKeys";
+import { enforceSSHDConfig } from "./sshdConfig";
+import { getState, loadState, saveState, sourceState } from "./state";
+import { resolveSourceKeys } from "./trust";
+import { parseAuthLog, readNewAuthLog } from "./authLog";
+import { absorbRevocations, applyUnrevokes, recordRevocation, removeRevokedKeys, syncRevokeRepo } from "./revocation";
+import { keyFingerprint } from "../authorizedKeys";
+import { checkOtherUserKeys, seedUserKeys } from "./userKeys";
+
+// portsecure authorized-keys daemon.
+//
+// It owns root's authorized_keys: the contents come from one or more git repos, anything else is
+// reverted, and password authentication is turned off so those repos are the only way in.
+//
+// The complete list of things that send a Discord message. Nothing else may be added to it
+// without the user asking for that specific case - everything else goes to the log.
+//   1. root's authorized_keys was changed outside portsecure, and was reverted.
+//   2. root's authorized_keys was updated because a source changed.
+//   3. Another user's authorized_keys changed.
+//   4. A source's history was rewritten.
+//   5. A source started being signed by a different key, so its new keys are being held.
+//   6. A source is now signed when it was not before, applied right away.
+//   7. A source changed without its signature being updated, so the change is ignored.
+//   8. A source has a corrupted signature, so its contents are ignored.
+//   9. The webhook file itself changed, reported to the webhook being replaced.
+//  10. A key was revoked here, after being used from an address it is not allowed from.
+//  11. A revoked key was removed from root's authorized_keys, said once per key.
+//  12. An unrevoke was published, and is being held for an hour.
+//  13. An unrevoke was applied once that hour passed.
+
+export type DaemonConfig = {
+    repoSources: string[];
+    hostLabel: string;
+};
+
+let config: DaemonConfig = { repoSources: [], hostLabel: "" };
+let repoFailureCounts: { [repoURL: string]: number } = {};
+
+export function getConfig() {
+    return config;
+}
+
+export function setConfig(value: DaemonConfig) {
+    config = value;
+    setHostLabel(value.hostLabel);
+}
+
+async function loadConfig(): Promise<DaemonConfig> {
+    let contents;
+    try {
+        contents = await fs.readFile(CONFIG_PATH, "utf8");
+    } catch (e) {
+        console.error(`portsecure: expected a config file at ${CONFIG_PATH}, ${e}`);
+        process.exit(1);
+    }
+    let parsed = JSON.parse(contents) as { repoSources?: string[]; hostLabel?: string };
+    if (!Array.isArray(parsed.repoSources)) {
+        console.error(`portsecure: expected a repoSources array in ${CONFIG_PATH}, was ${JSON.stringify(parsed.repoSources)}`);
+        process.exit(1);
+    }
+    for (let repoURL of parsed.repoSources) {
+        try {
+            await fs.access(sourceKeyPath(repoURL));
+        } catch (e) {
+            console.error(`portsecure: expected the private key for ${repoURL} at ${sourceKeyPath(repoURL)}, ${e}`);
+            process.exit(1);
+        }
+    }
+    return {
+        repoSources: parsed.repoSources,
+        // The machine knows its own name, the config only overrides it when a nicer label helps.
+        hostLabel: parsed.hostLabel || os.hostname(),
+    };
+}
+
+/** The union of every source, in source order, with duplicates dropped. A source that cannot be
+    read is skipped rather than emptying the merged set, so one broken repo cannot revoke the keys
+    that came from the others. */
+export async function readAllowedKeys() {
+    let keys: string[] = [];
+    let seen = new Set<string>();
+    for (let repoURL of config.repoSources) {
+        let sourceKeys: string[];
+        try {
+            sourceKeys = await resolveSourceKeys(repoURL);
+        } catch (e) {
+            console.log(`Skipping ${repoURL}, its checkout could not be read. ${e}`);
+            continue;
+        }
+        for (let key of sourceKeys) {
+            if (seen.has(key)) {
+                continue;
+            }
+            seen.add(key);
+            keys.push(key);
+        }
+    }
+    return keys;
+}
+
+/** Which source contributed a key, so its revocation is written to that source's revoke repo. */
+function sourceOfFingerprint(fingerprint: string) {
+    for (let repoURL of config.repoSources) {
+        if (sourceState(repoURL).acceptedKeys.some(key => keyFingerprint(key) === fingerprint)) {
+            return repoURL;
+        }
+    }
+    return config.repoSources[0] || "";
+}
+
+/** Anything sshd refused because of a from= restriction gets that key revoked everywhere. Reading
+    the log is cheap and local, and the fingerprint is checked against what we already revoked
+    before any network work happens.
+
+    A refusal is queued rather than acted on directly, because the log is read once and moves past:
+    if the revoke repo is unreachable at that moment, dropping the refusal would leave a key that
+    was misused accepted forever. The queue is retried until it is written down. */
+async function queueRefusedKeys(allowedKeys: string[]) {
+    let contents = await readNewAuthLog();
+    if (!contents.trim()) {
+        return;
+    }
+    let state = getState();
+    for (let found of parseAuthLog(contents)) {
+        // One key is revoked once, no matter how many addresses it was tried from.
+        if (state.revocations[found.fingerprint]) {
+            continue;
+        }
+        if (state.pendingRevocations.some(pending => pending.fingerprint === found.fingerprint)) {
+            continue;
+        }
+        let sourceURL = sourceOfFingerprint(found.fingerprint);
+        if (!sourceURL) {
+            console.log(`Nowhere to record the revocation of ${found.fingerprint}, no sources are configured`);
+            continue;
+        }
+        console.log(`Queued the revocation of ${found.fingerprint}, used from ${found.attempt.ip}`);
+        state.pendingRevocations.push({
+            fingerprint: found.fingerprint,
+            keyLine: allowedKeys.find(key => keyFingerprint(key) === found.fingerprint) || "",
+            sourceURL,
+            attempt: found.attempt,
+        });
+    }
+    await saveState();
+}
+
+/** Writes down everything queued that we have not managed to write down yet. */
+async function writeQueuedRevocations() {
+    let state = getState();
+    if (!state.pendingRevocations.length) {
+        return;
+    }
+    let remaining = [];
+    for (let pending of state.pendingRevocations) {
+        if (state.revocations[pending.fingerprint]) {
+            continue;
+        }
+        let recorded = await recordRevocation({ ...pending, hostLabel: config.hostLabel });
+        if (!recorded && !state.revocations[pending.fingerprint]) {
+            remaining.push(pending);
+        }
+    }
+    state.pendingRevocations = remaining;
+    await saveState();
+}
+
+/** Syncs one source. Returns whether the merged keys need reapplying. */
+async function pollSource(repoURL: string) {
+    let result;
+    try {
+        result = await syncRepo(repoURL);
+        repoFailureCounts[repoURL] = 0;
+    } catch (e) {
+        let failures = (repoFailureCounts[repoURL] || 0) + 1;
+        repoFailureCounts[repoURL] = failures;
+        console.log(`Sync of ${repoURL} failed (${failures} in a row). ${e}`);
+        if (failures < MAX_REPO_FAILURES_BEFORE_RECLONE) {
+            return false;
+        }
+        // Availability over tidiness: throw the working copy away and start again.
+        console.log(`Discarding the checkout of ${repoURL} and cloning from scratch`);
+        try {
+            await cloneRepo({ repoURL, repoPath: sourceRepoPath(repoURL), keyPath: sourceKeyPath(repoURL) });
+            repoFailureCounts[repoURL] = 0;
+            return true;
+        } catch (cloneError) {
+            console.log(`${repoURL} cannot be reached or cloned, its last known keys stay in place. ${cloneError}`);
+            return false;
+        }
+    }
+
+    if (result.historyRewritten) {
+        await notify(
+            `the history of \`${repoURL}\` was rewritten. Commit \`${result.previousSha.slice(0, 12)}\` is no`
+            + ` longer an ancestor of \`${result.remoteSha.slice(0, 12)}\`, so history was force pushed or`
+            + ` tampered with. The new state has been applied.`
+        );
+    }
+    if (!result.changed) {
+        return false;
+    }
+    sourceState(repoURL).lastSha = result.remoteSha;
+    return true;
+}
+
+async function everyCheck() {
+    let anyChanged = false;
+    for (let repoURL of config.repoSources) {
+        // One unreachable source must not stop the others from being checked.
+        try {
+            anyChanged = await pollSource(repoURL) || anyChanged;
+        } catch (e) {
+            console.log(`Polling ${repoURL} failed. ${e && (e as Error).stack || e}`);
+        }
+    }
+    if (anyChanged) {
+        await saveState();
+    }
+
+    // What other machines have revoked, and anything published to undo a revocation.
+    for (let repoURL of config.repoSources) {
+        await syncRevokeRepo(repoURL);
+    }
+    await absorbRevocations(config.repoSources);
+    await applyUnrevokes(config.repoSources);
+
+    let mergedKeys = await readAllowedKeys();
+    await queueRefusedKeys(mergedKeys);
+    await writeQueuedRevocations();
+
+    // The repo is checked first, so a change that came from it is reported as an update rather
+    // than as somebody having edited the file locally.
+    await enforceRootKeys({ keys: await removeRevokedKeys(mergedKeys), reason: anyChanged && "repo" || "manual" });
+    await checkOtherUserKeys();
+    await enforceSSHDConfig();
+}
+
+function startInterval(config: { intervalTime: number; run: () => Promise<void>; name: string }) {
+    let { intervalTime, run, name } = config;
+    let running = false;
+    let tick = async () => {
+        if (running) {
+            console.log(`Skipping ${name}, the previous run has not finished`);
+            return;
+        }
+        running = true;
+        try {
+            await run();
+        } catch (e) {
+            // Every scheduled job swallows its own errors, the daemon must outlive any single one.
+            console.log(`${name} failed. ${e && (e as Error).stack || e}`);
+        }
+        running = false;
+    };
+    setInterval(tick, intervalTime);
+    return tick;
+}
+
+export async function main() {
+    setConfig(await loadConfig());
+    await loadState();
+    await configureDiscordNotifications({ filePath: DEFAULT_WEBHOOK_FILE_PATH });
+
+    console.log(`Starting, ${config.repoSources.length} source(s), keys applied to ${ROOT_AUTHORIZED_KEYS}`);
+
+    // A first pass has to happen before the intervals, so a machine is correct immediately after
+    // boot rather than a minute later.
+    for (let repoURL of config.repoSources) {
+        try {
+            sourceState(repoURL).lastSha = (await syncRepo(repoURL)).remoteSha;
+        } catch (e) {
+            console.log(`Initial sync of ${repoURL} failed, continuing with whatever is on disk. ${e}`);
+        }
+    }
+    await saveState();
+    await seedUserKeys();
+
+    for (let repoURL of config.repoSources) {
+        await syncRevokeRepo(repoURL);
+    }
+    await absorbRevocations(config.repoSources);
+    await enforceRootKeys({ keys: await removeRevokedKeys(await readAllowedKeys()), reason: "repo" });
+    await enforceSSHDConfig();
+
+    // configureDiscordNotifications watches the webhook file on its own, so there is nothing to
+    // schedule for it here.
+    startInterval({ name: "check", intervalTime: CHECK_INTERVAL, run: everyCheck });
+}
+
+process.on("uncaughtException", e => console.log(`Uncaught exception, staying up. ${e && e.stack || e}`));
+process.on("unhandledRejection", e => console.log(`Unhandled rejection, staying up. ${e}`));
+process.on("SIGTERM", () => {
+    console.log("Received SIGTERM, exiting");
+    process.exit(0);
+});

package/security/authorizedKeys/daemon/git.ts

@@ -0,0 +1,119 @@
+import fs from "fs/promises";
+import path from "path";
+import { spawnPromise } from "../../helpers/spawn";
+import { sourceKeyPath, sourceRepoPath } from "../sources";
+import { GIT_TIMEOUT, MAX_ERROR_BODY_LENGTH } from "./paths";
+import { sourceState } from "./state";
+
+async function pathExists(filePath: string) {
+    try {
+        await fs.access(filePath);
+        return true;
+    } catch (e) {
+        return false;
+    }
+}
+
+/** core.sshCommand keeps the key selection with the command instead of in the environment. */
+export async function runGit(config: { args: string[]; cwd?: string; keyPath: string; allowFailure?: boolean }) {
+    let { args, cwd, keyPath, allowFailure } = config;
+    let sshCommand = `ssh -i ${keyPath} -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new`;
+    let result = await spawnPromise({
+        command: "git",
+        args: ["-c", `core.sshCommand=${sshCommand}`, ...args],
+        cwd,
+        timeoutTime: GIT_TIMEOUT,
+    });
+    if (result.error) {
+        throw new Error(`Expected git ${args.join(" ")} to run, failed with ${result.error.message}`);
+    }
+    if (result.status !== 0 && !allowFailure) {
+        throw new Error(
+            `Expected git ${args.join(" ")} to exit 0, was ${result.status}. `
+            + `${(result.stdout + result.stderr).trim().slice(0, MAX_ERROR_BODY_LENGTH)}`
+        );
+    }
+    return result;
+}
+
+export async function repoIsUsable(config: { repoPath: string; keyPath: string }) {
+    let { repoPath, keyPath } = config;
+    if (!await pathExists(path.join(repoPath, ".git"))) {
+        return false;
+    }
+    let result = await runGit({ args: ["rev-parse", "--git-dir"], cwd: repoPath, keyPath, allowFailure: true });
+    if (result.status !== 0) {
+        console.log(`Repo at ${repoPath} is not usable. ${(result.stdout + result.stderr).trim()}`);
+        return false;
+    }
+    return true;
+}
+
+/** Cloned beside the old checkout and swapped in, so a clone that fails leaves the copy we are
+    already using untouched rather than deleting the only keys we have. */
+export async function cloneRepo(config: { repoURL: string; repoPath: string; keyPath: string }) {
+    let { repoURL, repoPath, keyPath } = config;
+    let incomingPath = `${repoPath}.incoming`;
+    await fs.rm(incomingPath, { recursive: true, force: true });
+    await fs.mkdir(path.dirname(repoPath), { recursive: true });
+    await runGit({ args: ["clone", repoURL, incomingPath], keyPath });
+    await fs.rm(repoPath, { recursive: true, force: true });
+    await fs.rename(incomingPath, repoPath);
+    console.log(`Cloned ${repoURL} into ${repoPath}`);
+}
+
+export async function currentBranch(config: { repoPath: string; keyPath: string }) {
+    return (await runGit({ args: ["rev-parse", "--abbrev-ref", "HEAD"], ...config })).stdout.trim();
+}
+
+async function ensureSourceRepo(repoURL: string) {
+    let repoPath = sourceRepoPath(repoURL);
+    let keyPath = sourceKeyPath(repoURL);
+    if (!await repoIsUsable({ repoPath, keyPath })) {
+        await cloneRepo({ repoURL, repoPath, keyPath });
+    }
+    if (!sourceState(repoURL).branch) {
+        sourceState(repoURL).branch = await currentBranch({ repoPath, keyPath });
+    }
+}
+
+/** Returns what changed, so the caller can report it. A rewritten history is called out
+    separately - it means the remote no longer contains the commits we already had. */
+export async function syncRepo(repoURL: string) {
+    await ensureSourceRepo(repoURL);
+    let repoPath = sourceRepoPath(repoURL);
+    let keyPath = sourceKeyPath(repoURL);
+    let branch = sourceState(repoURL).branch;
+    let localSha = (await runGit({ args: ["rev-parse", "HEAD"], cwd: repoPath, keyPath })).stdout.trim();
+
+    // A ref listing is a few hundred bytes and no objects, so the usual case of nothing having
+    // changed costs almost nothing and we only fetch when there is something to fetch.
+    let listing = (await runGit({ args: ["ls-remote", "origin", branch], cwd: repoPath, keyPath })).stdout;
+    let remoteSha = (listing.split(/\s+/)[0] || "").trim();
+    if (!remoteSha) {
+        throw new Error(`Expected origin to report a sha for ${branch}, listed ${listing.slice(0, MAX_ERROR_BODY_LENGTH)}`);
+    }
+    if (remoteSha === localSha && remoteSha === sourceState(repoURL).lastSha) {
+        return { changed: false, historyRewritten: false, remoteSha, previousSha: localSha };
+    }
+
+    await runGit({ args: ["fetch", "--prune", "origin", branch], cwd: repoPath, keyPath });
+    remoteSha = (await runGit({ args: ["rev-parse", `origin/${branch}`], cwd: repoPath, keyPath })).stdout.trim();
+
+    let previousSha = sourceState(repoURL).lastSha || localSha;
+    let historyRewritten = false;
+    if (previousSha && previousSha !== remoteSha) {
+        // If what we already had is no longer an ancestor of the remote tip, commits were removed
+        // or rewritten rather than added.
+        let ancestry = await runGit({
+            args: ["merge-base", "--is-ancestor", previousSha, remoteSha],
+            cwd: repoPath,
+            keyPath,
+            allowFailure: true,
+        });
+        historyRewritten = ancestry.status !== 0;
+    }
+    await runGit({ args: ["reset", "--hard", `origin/${branch}`], cwd: repoPath, keyPath });
+    await runGit({ args: ["clean", "-fdx"], cwd: repoPath, keyPath });
+    return { changed: true, historyRewritten, remoteSha, previousSha };
+}

package/security/authorizedKeys/daemon/notify.ts

@@ -0,0 +1,25 @@
+import os from "os";
+import { sendDiscordNotification } from "../../notifications/discord";
+
+let hostLabelValue = "";
+
+export function setHostLabel(value: string) {
+    hostLabelValue = value;
+}
+
+// DO NOT add new calls to this. Every message goes to a real Discord server someone reads, so a
+// notification is only ever added when the user explicitly asks for that specific case. Startup,
+// success, errors, retries and recoveries all belong in console.log instead. The complete list of
+// cases that are allowed to notify is at the top of daemon.ts.
+export async function notify(message: string) {
+    let full = `**portsecure [${hostLabelValue || os.hostname()}]**: ${message}`;
+    // Logged before it is sent, and whether or not it arrives, so the journal is a complete record
+    // of what this machine had to say even when Discord is unreachable or the webhook is wrong.
+    console.log(`Discord: ${full}`);
+    try {
+        await sendDiscordNotification(full);
+    } catch (e) {
+        // A failed notification must never take the daemon down, the local log is the fallback.
+        console.log(`Failed to send the Discord notification above. ${e}`);
+    }
+}

package/security/authorizedKeys/daemon/paths.ts

@@ -0,0 +1,26 @@
+// Locations the daemon owns. Fixed rather than configurable, so every machine looks the same and
+// the config file only carries what genuinely differs between them.
+
+export const CONFIG_PATH = "/etc/portsecure/daemon.json";
+export const STATE_PATH = "/var/lib/portsecure/state.json";
+export const KEYS_HISTORY_PATH = "/var/lib/portsecure/authorized-keys-history";
+export const ROOT_AUTHORIZED_KEYS = "/root/.ssh/authorized_keys";
+export const SSHD_CONFIG_PATH = "/etc/ssh/sshd_config";
+export const SSHD_DROPIN_DIR = "/etc/ssh/sshd_config.d";
+export const SSHD_DROPIN_PATH = "/etc/ssh/sshd_config.d/00-portsecure.conf";
+export const PASSWD_PATH = "/etc/passwd";
+export const AUTH_LOG_PATH = "/var/log/auth.log";
+
+export const CHECK_INTERVAL = 60 * 1000;
+export const WEBHOOK_CHECK_INTERVAL = 5 * 60 * 1000;
+export const GIT_TIMEOUT = 120 * 1000;
+export const MAX_ERROR_BODY_LENGTH = 500;
+// A source that starts being signed by someone new is held at arm's length for this long, so a
+// stolen signing key cannot push keys onto a machine before anyone notices the warning.
+export const SIGNER_CHANGE_DELAY = 24 * 60 * 60 * 1000;
+// An unrevoke waits this long before taking effect, so a compromised signing key cannot instantly
+// undo the revocation that locked it out.
+export const UNREVOKE_DELAY = 60 * 60 * 1000;
+// After this many consecutive failures a repo is thrown away and cloned from scratch, which
+// recovers from corruption and interrupted fetches. Counted in checks, so about a quarter hour.
+export const MAX_REPO_FAILURES_BEFORE_RECLONE = 15;

package/security/authorizedKeys/daemon/portsecure.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=portsecure authorized keys daemon
+Documentation=https://github.com/sliftist/sliftutils
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/env node /opt/portsecure/sliftutils/bin/portsecuredaemon.js
+WorkingDirectory=/opt/portsecure/sliftutils
+User=root
+# Availability is the point of this daemon, so it always comes back.
+Restart=always
+RestartSec=10
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target