sliftutils 1.9.7 → 1.9.9

package/index.d.ts

@@ -1182,7 +1182,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
     export type BulkDatabase2Config = {
         maxTriggerThrottleMs?: number;
     };
-    export type MergeSkipReason = "mergeInFlight" | "tabLockHeld" | "fileLockHeld" | "nothingToMerge";
+    export type MergeSkipReason = "mergeInFlight" | "tabLockHeld" | "fileLockHeld" | "fileLockUnwritable" | "nothingToMerge";
     export type MergeAttemptResult = {
         merged: boolean;
         skipReason?: MergeSkipReason;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.9.7",
+    "version": "1.9.9",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/security/authorizedKeys/daemon/daemon.ts

@@ -247,7 +247,6 @@ async function pollSource(repoURL: string) {
     // Said when the file is written, and only if it came out different. A commit that does not
     // touch the keys is not worth telling anyone about.
     addChangeReason(`The key repo \`${repoURL}\` moved to \`${result.remoteSha.slice(0, 12)}\`.`);
-    sourceState(repoURL).lastSha = result.remoteSha;
     return true;
 }
 

package/security/authorizedKeys/daemon/git.ts

@@ -71,12 +71,14 @@ export async function currentBranch(config: { repoPath: string; keyPath: string
     for good: it sits in a stash if it is ever wanted. */
 export async function setGitRef(config: { repoPath: string; gitRef: string; keyPath: string }) {
     let { repoPath, gitRef, keyPath } = config;
-    // Staging the untracked files first is what lets the stash carry them off, so the reset lands
-    // on a genuinely clean tree.
-    await runGit({ args: ["add", "--all"], cwd: repoPath, keyPath, allowFailure: true });
-    // A checkout with nothing to stash makes git stash exit non-zero, which is harmless here - the
-    // reset below is what actually forces the tree.
-    await runGit({ args: ["stash"], cwd: repoPath, keyPath, allowFailure: true });
+    let dirty = (await runGit({ args: ["status", "--porcelain"], cwd: repoPath, keyPath })).stdout.trim();
+    if (dirty) {
+        console.log(`[pid ${process.pid}] Stashing local changes in ${repoPath} before resetting it to ${gitRef}:\n${dirty.slice(0, MAX_ERROR_BODY_LENGTH)}`);
+        // Staging the untracked files first is what lets the stash carry them off, so the reset lands
+        // on a genuinely clean tree.
+        await runGit({ args: ["add", "--all"], cwd: repoPath, keyPath, allowFailure: true });
+        await runGit({ args: ["stash"], cwd: repoPath, keyPath, allowFailure: true });
+    }
     await runGit({ args: ["fetch", "--prune", "origin"], cwd: repoPath, keyPath });
     await runGit({ args: ["reset", "--hard", gitRef], cwd: repoPath, keyPath });
     // Drops objects nothing references any more, so a file committed by mistake cannot bloat the
@@ -140,5 +142,6 @@ export async function syncRepo(repoURL: string, options?: { forceUpdate?: boolea
         });
         historyRewritten = ancestry.status !== 0;
     }
+    sourceState(repoURL).lastSha = remoteSha;
     return { changed: true, historyRewritten, remoteSha, previousSha };
 }

package/security/authorizedKeys/unrevoke.ts

@@ -8,6 +8,7 @@ import { spawnPromise } from "../helpers/spawn";
 
 export const UNREVOKES_DIR = "unrevoked";
 const REVOCATIONS_DIR = "revocations";
+const MAX_LOGGED_CONTENTS = 500;
 
 /** One revocation event. Either an ssh key used from an address it is not allowed from, or a
     machine that talked to us from one. Both are the same shape - an identity and an address - and
@@ -38,25 +39,20 @@ export function revocationIdentity(revocation: Revocation) {
     return revocation.fingerprint || revocation.machineId || "";
 }
 
-/** Every revocation the revoke repo lists. Cloned read only into a temp directory.
-
-    A person running this has their own access to the repo, so their credentials are what is used.
-    On a machine that is already a portsecure host there may be no such credentials, but the source
-    deploy key is right there, and the key derived from it is one that repo definitely accepts. */
-export async function readRemoteRevocations(config: { sourceURL: string }) {
+async function cloneRevokeRepo(config: { sourceURL: string }) {
     let { sourceURL } = config;
     let temporaryDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "unrevoke-"));
     let repoPath = path.join(temporaryDirectory, "repo");
-    let cloneArgs = ["clone", "--depth", "1", revokeRepoURL(sourceURL), repoPath];
+    let gitConfig: string[] = [];
     let sourceKey = await findSourceKey(sourceURL);
     if (sourceKey) {
         let derived = deriveRevokeKey(await fs.readFile(sourceKey, "utf8"));
         let derivedKeyPath = path.join(temporaryDirectory, "key");
         await fs.writeFile(derivedKeyPath, derived.privateKeyFile, { mode: 0o600 });
         let sshCommand = `ssh -i ${derivedKeyPath} -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new`;
-        cloneArgs = ["-c", `core.sshCommand=${sshCommand}`, ...cloneArgs];
+        gitConfig = ["-c", `core.sshCommand=${sshCommand}`];
     }
-    let clone = await spawnPromise({ command: "git", args: cloneArgs });
+    let clone = await spawnPromise({ command: "git", args: [...gitConfig, "clone", "--depth", "1", revokeRepoURL(sourceURL), repoPath] });
     if (clone.status !== 0) {
         await fs.rm(temporaryDirectory, { recursive: true, force: true });
         throw new Error(
@@ -64,18 +60,63 @@ export async function readRemoteRevocations(config: { sourceURL: string }) {
             + `${(clone.stdout + clone.stderr).trim()}`
         );
     }
-    let revocations: Revocation[] = [];
-    let directory = path.join(repoPath, REVOCATIONS_DIR);
-    let names: string[] = [];
+    return { temporaryDirectory, repoPath, gitConfig };
+}
+
+async function listRevocationFiles(repoPath: string) {
     try {
-        names = (await fs.readdir(directory)).sort();
+        return (await fs.readdir(path.join(repoPath, REVOCATIONS_DIR))).filter(name => name.endsWith(".json")).sort();
     } catch (e) {
         // No revocations directory at all just means nothing has ever been revoked.
+        return [];
     }
-    for (let name of names) {
-        if (!name.endsWith(".json")) {
-            continue;
+}
+
+/** Deletes the revocation files in the revoke repo that are not JSON at all, and pushes that.
+    Nothing is lost: a file that does not parse names no key and no address, so every machine
+    already ignores it, and all it does is print the same complaint on every read forever.
+
+    Judged on a fresh clone, never on a machine's own checkout, so a file only goes if the repo
+    itself holds it broken. */
+export async function removeUnreadableRevocations(config: { sourceURL: string }) {
+    let { sourceURL } = config;
+    let { temporaryDirectory, repoPath, gitConfig } = await cloneRevokeRepo({ sourceURL });
+    try {
+        let removed: string[] = [];
+        for (let name of await listRevocationFiles(repoPath)) {
+            let relativePath = `${REVOCATIONS_DIR}/${name}`;
+            let contents = await fs.readFile(path.join(repoPath, REVOCATIONS_DIR, name), "utf8");
+            try {
+                JSON.parse(contents);
+                continue;
+            } catch (e) {
+                console.log(`Removing unreadable revocation ${relativePath} from ${revokeRepoURL(sourceURL)}, ${contents.length} characters: ${JSON.stringify(contents.slice(0, MAX_LOGGED_CONTENTS))}. ${e}`);
+            }
+            await fs.rm(path.join(repoPath, REVOCATIONS_DIR, name));
+            removed.push(relativePath);
         }
+        if (!removed.length) return;
+        let git = async (args: string[]) => {
+            let result = await spawnPromise({ command: "git", args: [...gitConfig, ...args], cwd: repoPath });
+            if (result.status !== 0) {
+                throw new Error(`Expected git ${args.join(" ")} to succeed in a clone of ${revokeRepoURL(sourceURL)}, exited ${result.status}.\n${(result.stdout + result.stderr).trim()}`);
+            }
+        };
+        await git(["add", "-A"]);
+        await git(["-c", "user.email=portsecure@localhost", "-c", "user.name=portsecure", "commit", "-m", `remove ${removed.length} unreadable revocation(s)`]);
+        await git(["push", "origin", "HEAD"]);
+        console.log(`Removed ${removed.length} unreadable revocation(s) from ${revokeRepoURL(sourceURL)}: ${removed.join(", ")}`);
+    } finally {
+        await fs.rm(temporaryDirectory, { recursive: true, force: true });
+    }
+}
+
+export async function readRemoteRevocations(config: { sourceURL: string }) {
+    let { sourceURL } = config;
+    let { temporaryDirectory, repoPath } = await cloneRevokeRepo({ sourceURL });
+    let revocations: Revocation[] = [];
+    let directory = path.join(repoPath, REVOCATIONS_DIR);
+    for (let name of await listRevocationFiles(repoPath)) {
         try {
             revocations.push(JSON.parse(await fs.readFile(path.join(directory, name), "utf8")));
         } catch (e) {

package/security/authorizedKeys/unrevokeCli.ts

@@ -6,13 +6,13 @@ import { revokeRepoURL } from "./revokeSource";
 import { signRepo } from "../signedFiles/signFiles";
 import { getMachines, setMachines } from "../machines/machines";
 import { resolveKeysRepo } from "./keysRepo";
-import { readRemoteRevocations, readUnrevokedInRepo, Revocation, revocationIdentity, revocationIP, stillRevokedBy, UNREVOKES_DIR } from "./unrevoke";
+import { readRemoteRevocations, readUnrevokedInRepo, removeUnreadableRevocations,Revocation, revocationIdentity, revocationIP, stillRevokedBy, UNREVOKES_DIR } from "./unrevoke";
 
 const GIT_KEYWORD = "git";
 const COMMIT_MESSAGE = "unrevoke keys";
 const USAGE = `Usage: yarn unrevoke [keys-repo] [${GIT_KEYWORD}]
 
-Run this in a keys repo, or name one. It reads that repo's revoke repo and writes one unrevoke file
+Run this in a keys repo, or name one. It deletes any revocation file in that repo's revoke repo that is not valid JSON, pushing that straight away. It reads the revoke repo and writes one unrevoke file
 naming every revocation in it, so the keys are accepted again as each machine picks it up. It signs
 the result, and with ${GIT_KEYWORD} it commits and pushes it too.
 
@@ -98,6 +98,12 @@ async function main() {
     }
     let { repoPath, sourceURL: originURL } = await resolveKeysRepo(positional[0]);
 
+    try {
+        await removeUnreadableRevocations({ sourceURL: originURL });
+    } catch (e) {
+        console.log(`Could not remove the unreadable revocations from ${revokeRepoURL(originURL)}, unrevoking anyway.`, (e as Error).stack ?? e);
+    }
+
     let revocations = await readRemoteRevocations({ sourceURL: originURL });
     if (!revocations.length) {
         console.log(`${revokeRepoURL(originURL)} lists no revocations, so there is nothing to undo.`);

package/security/machines/machines.ts

@@ -16,6 +16,7 @@ import { describeHost, runOverSSH, writeRemoteFile } from "../helpers/remoteSSH"
 import { lazy } from "socket-function/src/caching";
 import { runInfinitePoll, runInSerial } from "socket-function/src/batching";
 import { spawnPromise } from "../helpers/spawn";
+import { formatTime } from "socket-function/src/formatting/format";
 
 // Which machines this system talks to, kept in the same repo as the ssh keys. That repo is already
 // signed, already distributed to every machine, and already has somewhere to record a rejection,
@@ -26,6 +27,9 @@ const REVOCATION_REASON = "a machine talked to us from an unapproved IP";
 // The revoke repo is pulled at most this often. A check happens per request, and a request must
 // not cost a round trip to github.
 const REVOKE_SYNC_INTERVAL = 60 * 1000;
+const SLOW_REVOKE_SYNC_TIME = 10 * 1000;
+const REVOKE_SYNC_RETRY_INTERVAL = 5 * 60 * 1000;
+const ACCEPTED_SYNC_WAIT = 5 * 1000;
 // And the machine list is re-read from the checkout on this interval, in the background. The
 // checkout itself only moves when the daemon pulls it, which is on the same sort of interval, so
 // reading more often than this would only re-read the same bytes.
@@ -235,19 +239,56 @@ async function readMachines(repoPath: string) {
     return machines;
 }
 
-let lastRevokeSync = 0;
+let nextRevokeSyncTime = 0;
+let revokeSyncFailing = false;
+let revokeSyncInFlight: Promise<boolean> | undefined;
 
 /** Pulls the keys repo and its revoke repo, at most once a REVOKE_SYNC_INTERVAL, because this is
     asked per request. Both, since either changing changes the answer: the keys repo carries the
     machines and unrevokes, the revoke repo carries the revocations. A sync that fails leaves the
     checkouts we already have. */
-async function syncRevocations(sourceURL: string) {
-    if (Date.now() - lastRevokeSync < REVOKE_SYNC_INTERVAL) {
-        return;
+function startRevocationSync(sourceURL: string) {
+    if (revokeSyncInFlight || Date.now() < nextRevokeSyncTime) {
+        return revokeSyncInFlight;
     }
-    lastRevokeSync = Date.now();
-    await syncRepo(sourceURL).catch(e => console.log(`Could not sync ${sourceURL}, using the checkout already here. ${e}`));
-    await syncRepoFiles(revokeRepo(sourceURL)).catch(e => console.log(`Could not sync ${revokeRepoURL(sourceURL)}, using the revocations already absorbed. ${e}`));
+    nextRevokeSyncTime = Date.now() + REVOKE_SYNC_INTERVAL;
+    revokeSyncInFlight = (async () => {
+        let start = Date.now();
+        let succeeded = true;
+        await syncRepo(sourceURL).catch(e => {
+            succeeded = false;
+            console.log(`[pid ${process.pid}] Could not sync ${sourceURL}, using the checkout already here. ${(e as Error).stack ?? e}`);
+        });
+        await syncRepoFiles(revokeRepo(sourceURL)).catch(e => {
+            succeeded = false;
+            console.log(`[pid ${process.pid}] Could not sync ${revokeRepoURL(sourceURL)}, using the revocations already absorbed. ${(e as Error).stack ?? e}`);
+        });
+        let elapsed = Date.now() - start;
+        if (elapsed > SLOW_REVOKE_SYNC_TIME) {
+            console.log(`[pid ${process.pid}] Syncing ${sourceURL} and ${revokeRepoURL(sourceURL)} took ${formatTime(elapsed)}, over ${formatTime(SLOW_REVOKE_SYNC_TIME)}`);
+        }
+        if (!succeeded) {
+            nextRevokeSyncTime = Date.now() + REVOKE_SYNC_RETRY_INTERVAL;
+            if (!revokeSyncFailing) {
+                console.log(`[pid ${process.pid}] Syncing ${sourceURL} failed, retrying every ${formatTime(REVOKE_SYNC_RETRY_INTERVAL)} instead of every ${formatTime(REVOKE_SYNC_INTERVAL)} until it works`);
+            }
+            revokeSyncFailing = true;
+            return false;
+        }
+        if (revokeSyncFailing) {
+            console.log(`[pid ${process.pid}] Syncing ${sourceURL} works again, back to every ${formatTime(REVOKE_SYNC_INTERVAL)}`);
+        }
+        revokeSyncFailing = false;
+        try {
+            trustedMachines.set(Promise.resolve(await readTrustedMachines()));
+        } catch (e) {
+            console.log(`[pid ${process.pid}] Could not re-read the trusted machines after syncing, keeping the ones we have. ${(e as Error).stack ?? e}`);
+        }
+        return true;
+    })().finally(() => {
+        revokeSyncInFlight = undefined;
+    });
+    return revokeSyncInFlight;
 }
 
 /** What to run to trust a machine that has just been refused.
@@ -390,12 +431,34 @@ export async function isMachineAccepted(config: {
     // there is nothing to gain from looking again. Any other rejection might just be a stale view -
     // a machine added seconds ago that our cached list has not caught up to - so it is worth one
     // fresh look, which is what the retry below does.
-    let evaluate = async (): Promise<{ verdict: MachineVerdict; froze: boolean }> => {
-        if (LOOPBACK.includes(ip) && !!domain && machineId === getOwnMachineId(domain)) {
-            return { verdict: { accepted: true, reason: "" }, froze: false };
+    if (LOOPBACK.includes(ip) && !!domain && machineId === getOwnMachineId(domain)) {
+        return { accepted: true, reason: "" };
+    }
+    let { sourceURL } = await keysRepo();
+    let syncing = startRevocationSync(sourceURL);
+    if (syncing) {
+        let checkouts = await Promise.all([sourceRepoPath(sourceURL), revokeRepoPath(sourceURL)].map(repoPath =>
+            fs.access(path.join(repoPath, ".git")).then(() => true, () => false)
+        ));
+        let acceptedNow = false;
+        if (checkouts.every(exists => exists)) {
+            let listed = (await getTrustedMachines()).find(entry => entry.machineId === machineId);
+            acceptedNow = !!listed && (listed.ips.includes(ip) || LOOPBACK.includes(ip));
+        }
+        if (!acceptedNow) {
+            await syncing;
+        } else {
+            let synced = await Promise.race([
+                syncing,
+                new Promise<boolean>(resolve => setTimeout(() => resolve(false), ACCEPTED_SYNC_WAIT)),
+            ]);
+            if (!synced) {
+                return { accepted: true, reason: "" };
+            }
         }
-        let { sourceURL } = await keysRepo();
-        await syncRevocations(sourceURL);
+    }
+
+    let evaluate = async (): Promise<{ verdict: MachineVerdict; froze: boolean }> => {
         // The list has revocations already applied - readSignedRepo drops frozen machines - so a
         // machine in it is trusted, and a machine missing from it is either unknown or frozen.
         let machine = (await getTrustedMachines()).find(entry => entry.machineId === machineId);
@@ -456,7 +519,7 @@ export async function isMachineAccepted(config: {
     // final.
     if (Date.now() - lastMachinesInvalidation > MACHINES_INVALIDATION_INTERVAL) {
         lastMachinesInvalidation = Date.now();
-        let { sourceURL } = await keysRepo();
+        await revokeSyncInFlight;
         await syncRepo(sourceURL, { forceUpdate: true }).catch(e => console.log(`Could not force update ${sourceURL}. ${e}`));
         trustedMachines.reset();
     }

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -28,7 +28,7 @@ export type StorageFactory = (path: string) => Promise<FileStorage>;
 export type BulkDatabase2Config = {
     maxTriggerThrottleMs?: number;
 };
-export type MergeSkipReason = "mergeInFlight" | "tabLockHeld" | "fileLockHeld" | "nothingToMerge";
+export type MergeSkipReason = "mergeInFlight" | "tabLockHeld" | "fileLockHeld" | "fileLockUnwritable" | "nothingToMerge";
 export type MergeAttemptResult = {
     merged: boolean;
     skipReason?: MergeSkipReason;

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -111,6 +111,7 @@ export type MergeSkipReason =
     | "tabLockHeld"
     // Another process holds the cross-process .merge-lock file (or won the settle race for it).
     | "fileLockHeld"
+    | "fileLockUnwritable"
     // compact() only: the collection has no files on disk.
     | "nothingToMerge";
 
@@ -689,7 +690,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         try {
             await this.tryMergeThrottled();
         } catch (e) {
-            console.warn(`${this.name}: background merge failed: ${(e as Error).message}`);
+            console.warn(`${this.name}: background merge failed:`, (e as Error).stack ?? e);
         }
     }
 
@@ -714,7 +715,14 @@ export class BulkDatabaseBase<T extends { key: string }> {
         if (this.mergeInFlight) return await this.mergeSkip("mergeInFlight");
         if (!tryAcquireMergeLock(this.name, writerId)) return await this.mergeSkip("tabLockHeld");
         const storage = await this.storage();
-        const haveFileLock = await tryAcquireMergeFileLock(storage, writerId);
+        let haveFileLock: boolean;
+        try {
+            haveFileLock = await tryAcquireMergeFileLock(storage, writerId);
+        } catch (e) {
+            releaseMergeLock(this.name, writerId);
+            console.warn(`${blue(this.name)} ${magenta("merge")} skipped: fileLockUnwritable, writer ${writerId} could not write .merge-lock:`, (e as Error).stack ?? e);
+            return { merged: false, skipReason: "fileLockUnwritable" };
+        }
         if (!haveFileLock) {
             releaseMergeLock(this.name, writerId);
             return await this.mergeSkip("fileLockHeld");