toolroll 0.9.31 → 0.9.32

package/dist/operate.js

@@ -3,6 +3,8 @@ import { leadNameOf } from "./lead-identity.js";
 import { maybeTriggerRepair } from "./dispose.js";
 import { CHECK_LEVEL_HINTS, CHECK_LEVEL_WORDS, isCheckLevel, liveQuickCommand, projectCheckLevel, quickVerifyKey, setProjectCheckLevel, setTaskCheckLevel, suggestQuickCommand } from "./check-levels.js";
 import { fileAddTestsTask, followUpChecksOf, requestFollowUpChecks, runFollowUpCheck, runWaitingChecks } from "./result-follow-ups.js";
+import { runBatchChecks } from "./batch-checks.js";
+import { BATCH_HINT, BATCH_WINDOW_MS, projectBatchChecks, setProjectBatchChecks } from "./batch-policy.js";
 import { buildReviewPass } from "./build-review.js";
 import { buildReviewLines, buildReviewOf } from "./review-switch.js";
 import { parseProtectedPaths } from "./approval-policy.js";
@@ -11,6 +13,7 @@ import { evidencePack, exportDay, ledgerExportChunks, standaloneEvidenceHtml } f
 import { matchesOutsideCheckpoint } from "./ledger-chain.js";
 import { CLEANUP_CHOICES, bytesWords, cleanupWords, diskBytes, parseCleanup, storageReport } from "./storage.js";
 import { removeStaleTestTemp, tempRoots, testTempFolders } from "./test-temp.js";
+import { dailyStorageSweep, lastSweep, nextSweepAt, saveSweep, storageSweepOff as storageSweepOffNow, sweepDetails, sweepWords as storageSweepWords } from "./storage-sweep.js";
 import { removeShared, sharedDepsRoot, sharedUse, unusedShared } from "./shared-deps.js";
 import { checkoutPlan, cleanCheckouts, discardCheckout, finishedCheckouts, slimKeptCheckouts, whyWords } from "./checkout-cleanup.js";
 import { closeOutCheckouts, closeOutRuns, stoppedWords } from "./run-closeout.js";
@@ -996,6 +999,8 @@ async function dispatch(command, positional, flags, context) {
                 return projectDeleteCommand(positional, flags, context);
             if (positional[0] === "concurrency")
                 return projectConcurrencyCommand(positional, flags, context);
+            if (positional[0] === "checks")
+                return projectChecksCommand(positional, flags, context);
             if (positional[0] === "demo")
                 return projectDemoCommand(positional, flags, context);
             return runProjectCommand(positional, flags, context);
@@ -4541,7 +4546,18 @@ async function reconcileCommand(flags, context) {
         retention = null; /* the next pass tries again */
     }
     const retained = retention === null ? [] : retention.counts.filter(one => one.count > 0);
+    // The daily storage sweep (storage-sweep.ts): stale test temp folders, orphans Toolroll left running, finished
+    // checkouts, extra staged runtimes and dependency installs unused for a week; saved for Settings → Storage.
+    let storageSwept = null;
+    try {
+        storageSwept = await dailyStorageSweep(store, { databaseFile: context.databaseFile, pool: worktrees, tempRoots: context.tempRoots ?? tempRoots() }, clock);
+    }
+    catch {
+        storageSwept = null; /* the next pass tries again */
+    }
+    const swept = storageSwept === null ? [] : storageSwept.parts.filter(one => one.count > 0);
     const nothing = retained.length === 0 &&
+        swept.length === 0 &&
         recovered.length === 0 &&
         reaped.length === 0 &&
         adoption.adopted.length === 0 &&
@@ -4562,6 +4578,7 @@ async function reconcileCommand(flags, context) {
         checkoutsSlimmed: slimmed.slimmed.map(one => one.path),
         liveViewsSwept: liveSwept.removed.length,
         retention: retention === null ? null : { counts: retention.counts, freedBytes: retention.freed },
+        storageSweep: storageSwept,
     }, () => nothing
         ? ["Nothing to reconcile. Everything is where it should be."]
         : [
@@ -4575,6 +4592,7 @@ async function reconcileCommand(flags, context) {
             ...(slimmed.slimmed.length === 0 ? [] : [`Dropped dependencies and build output from ${slimmed.slimmed.length} kept checkout(s), about ${bytesWords(slimmed.slimmed.reduce((sum, one) => sum + one.bytes, 0))}.`]),
             ...(liveSwept.removed.length === 0 ? [] : [`Cleared ${liveSwept.removed.length} finished live view(s).`]),
             ...(retention === null || retained.length === 0 ? [] : [`Retention removed ${sweepWords(retention.counts, retention.freed)}.`]),
+            ...(storageSwept === null || swept.length === 0 ? [] : [`Storage sweep: ${storageSweepWords(storageSwept.parts)}`]),
         ]);
 }
 // ---- gaps -----------------------------------------------------------------
@@ -7658,11 +7676,15 @@ async function runWatchLoop(args) {
         run: async () => {
             if (store.isDemo())
                 return;
+            // Batch checks first: a cohort that is ready runs one check for all its results.
+            const batched = await runBatchChecks(store, context.evidenceRoot, { now: context.clock, shouldStop: stopping });
+            if (batched > 0)
+                progress(`watch: settled ${batched} batch-checked result(s)`);
             const ran = await runWaitingChecks(store, context.evidenceRoot, { now: context.clock, shouldStop: stopping });
             if (ran > 0)
                 progress(`watch: ran ${ran} follow-up check(s)`);
             // A finished check may send a build's card down its failure path: now, not on the next beat.
-            if (ran > 0)
+            if (ran + batched > 0)
                 moveCards(store, repo, context.clock(), context.evidenceRoot);
         },
     });
@@ -12231,6 +12253,45 @@ async function projectConcurrencyCommand(positional, flags, context) {
         ...(changed.before > changed.after ? ["Builds already running carry on; the new number applies to the next one."] : []),
     ]);
 }
+/** `project checks [--batch on|off] --repo <p>`: whether the project's results share one full check when they
+ * finish close together (batch-checks.ts). Off by default; changing it is an approver's act, in the ledger. */
+async function projectChecksCommand(positional, flags, context) {
+    const command = "project checks";
+    const allowed = new Set(["repo", "batch", "as", "token", "token-file", "token-env", "db", "json"]);
+    for (const name of flags.keys())
+        if (!allowed.has(name))
+            return fail(context.write, context.json, command, "usage", `--${name} is not a project checks option.`, EXIT.usage);
+    const repoFlag = text(flags, "repo");
+    const wanted = flags.get("batch");
+    if (positional.length > 1 || repoFlag === undefined || (wanted !== undefined && wanted !== "on" && wanted !== "off")) {
+        return fail(context.write, context.json, command, "usage", "Use project checks [--batch on|off] --repo <project path>.", EXIT.usage);
+    }
+    const registered = await loadRepos(registryPathOf(context)).catch(() => ({ error: "unreadable" }));
+    const known = [...new Set([...context.store.knownRepos(), ...context.store.listProjects().map(one => one.path), ...("error" in registered ? [] : registered.repos)])];
+    const repo = known.find(one => one === repoFlag || one === resolve(repoFlag) || one === canonicalProject(repoFlag));
+    if (repo === undefined)
+        return fail(context.write, context.json, command, "not-found", "That isn't a project Toolroll knows.", EXIT.refused);
+    const level = projectCheckLevel(context.store, repo).level;
+    const words = (on) => [
+        `${repo}: batch checks are ${on ? "on" : "off"}. ${on ? BATCH_HINT : "Each result runs its own full check."}`,
+        ...(on && level !== "full" ? [`Checks are ${CHECK_LEVEL_WORDS[level]} for this project, so nothing is batched until they are Full.`] : []),
+    ];
+    if (wanted === undefined) {
+        const current = projectBatchChecks(context.store, repo);
+        return succeed(context.write, context.json, command, { repo, batch: current.on, windowMinutes: current.windowMs / 60_000, level, setBy: current.setBy }, () => words(current.on));
+    }
+    const acting = await askCredentials(flags, context);
+    const verified = acting === null ? null : authenticateApprover(context.store, acting.name, acting.token, repo);
+    if (acting === null || verified === null || !verified.ok || !context.store.accountCanAccess(acting.name, repo)) {
+        return fail(context.write, context.json, command, "refused", "An approver for this project changes batch checks: pass --as and --token (or use the remembered login).", EXIT.refused);
+    }
+    const on = wanted === "on";
+    const changed = setProjectBatchChecks(context.store, repo, on, acting.name, context.clock());
+    return succeed(context.write, context.json, command, { repo, batch: on, before: changed.before, changed: changed.changed, windowMinutes: BATCH_WINDOW_MS / 60_000, level }, () => [
+        ...words(on),
+        ...(changed.changed && !on ? ["Results already waiting are checked now."] : []),
+    ]);
+}
 /** `project demo [<url>|off] --repo <p>` (review 827): the project's own demo or dev server, which its scouts may
  * screenshot. Changing it is an approver's act. */
 async function projectDemoCommand(positional, flags, context) {
@@ -12625,17 +12686,33 @@ async function storageCommand(positional, flags, context) {
         return copies.map(one => ({ key: one.key, repo: one.repo, path: one.dir, bytes: sizes.get(one.dir) ?? 0, checkouts: one.checkouts.length, usedBy: one.checkouts, retired: one.retired, node: one.node, createdAt: one.createdAt, usedAt: one.usedAt }));
     };
     const sharedLine = (one) => `  ${bytesWords(one.bytes).padStart(8)}  ${projectName(one.repo)} ${one.key.slice(0, 8)}: ${one.checkouts === 1 ? "1 checkout uses it" : `${one.checkouts} checkouts use it`}${one.retired ? " (changed after install; never linked again)" : ""}`;
+    // What the latest sweep (automatic, or a clean-up by hand) did and when (storage-sweep.ts).
+    const sweepFacts = () => {
+        const last = lastSweep(store);
+        const automatic = last?.source === "automatic" ? last : lastSweep(store, "automatic");
+        return { last, lastAutomatic: automatic, next: nextSweepAt(store) };
+    };
+    const sweepLines = (facts) => {
+        const when = (at) => new Date(at).toLocaleString("en-US", { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" });
+        const one = facts.last;
+        const head = one === null ? (facts.next === null && storageSweepOffNow() ? "Automatic sweep: off." : "Automatic sweep: not run yet; it runs once a day.")
+            : `Last swept ${when(one.at)}${one.source === "manual" ? ` by ${one.actor}` : " automatically"}: ${storageSweepWords(one.parts)}`;
+        const auto = one !== null && one.source === "manual" && facts.lastAutomatic !== null ? [`Last automatic sweep ${when(facts.lastAutomatic.at)}: ${storageSweepWords(facts.lastAutomatic.parts)}`] : [];
+        return [head, ...auto, ...(one === null ? [] : sweepDetails(one.parts).slice(0, 8).map(line => `  ${line}`))];
+    };
     if (action === undefined) {
         const plan = await checkoutPlan(store, pool, context.clock(), { manual: true });
         const report = storageReport(store, context.databaseFile, plan.go.length === 0 ? "nothing to clean up" : `about ${bytesWords(plan.freeBytes)} to clean up`);
         const { staleBytes: _, ...temp } = leftovers(false);
         const shared = sharedFacts(sharedUse(depsRoot, checkoutRows()));
-        return succeed(context.write, context.json, "storage", { ...report, checkouts: checkoutSummary(plan), testTemp: temp, shared }, () => [
+        const sweep = sweepFacts();
+        return succeed(context.write, context.json, "storage", { ...report, checkouts: checkoutSummary(plan), testTemp: temp, shared, sweep }, () => [
             `${report.folder}: ${bytesWords(report.total)}`,
             ...report.lines.map(one => `  ${one.what.padEnd(19)} ${bytesWords(one.bytes).padStart(8)}${one.count === undefined ? "" : `  (${one.count})`}${one.note === undefined ? "" : `  ${one.note}`}`),
             ...(shared.length === 0 ? [] : ["Shared dependencies:", ...shared.map(sharedLine)]),
             ...summary(plan),
             leftoverLine({ ...temp, staleBytes: 0 }),
+            ...sweepLines(sweep),
         ]);
     }
     if (action === "cleanup") {
@@ -12692,6 +12769,13 @@ async function storageCommand(positional, flags, context) {
         for (const debris of unused.debris)
             removeShared(debris);
         const sharedFreed = sharedRemoved.reduce((sum, one) => sum + one.bytes, 0);
+        // Saved like an automatic sweep, so Storage shows what this clean-up did and when.
+        const parts = [
+            { kind: "checkouts", count: done.removed.length, bytes: done.freed, failed: done.kept.filter(one => one.why === "git refused").length, items: done.removed.slice(0, 20).map(one => one.path) },
+            { kind: "test temp", count: swept.removed.length, bytes: temp.staleBytes, failed: swept.failed.length, items: swept.removed.slice(0, 20) },
+            { kind: "dependencies", count: sharedRemoved.length, bytes: sharedFreed, failed: unused.copies.length - sharedRemoved.length, items: sharedRemoved.slice(0, 20).map(one => one.path) },
+        ];
+        saveSweep(store, { at: context.clock().toISOString(), source: "manual", actor, parts });
         return succeed(context.write, context.json, command, { removed: done.removed, kept: done.kept, freedBytes: done.freed, testTempRemoved: swept.removed.length, testTempFailed: swept.failed, sharedRemoved, sharedFreedBytes: sharedFreed }, () => [
             done.removed.length === 0 ? (swept.removed.length === 0 && sharedRemoved.length === 0 ? "Nothing to clean up." : "No checkouts to clean up.") : `Removed ${done.removed.length} checkout${done.removed.length === 1 ? "" : "s"}, about ${bytesWords(done.freed)}; their branches stay.`,
             ...done.removed.map(itemLine),

package/dist/orphan-sweep.d.ts

@@ -0,0 +1,63 @@
+import { type ProcessRow, type StopOptions } from "./run-closeout.js";
+import type { Store } from "./store.js";
+import { type TempOwner } from "./test-temp.js";
+import type { WorktreePool } from "./worktree.js";
+/** Where a process runs from: its working folder and executable, when the OS says. */
+export type Located = {
+    cwd?: string;
+    exe?: string;
+};
+/** Working folder and executable of each pid: /proc on Linux, lsof on macOS. Throws when the OS can't be asked. */
+export declare function locateProcesses(pids: readonly number[]): Map<number, Located>;
+/** A folder an orphan may be stopped from: a let-go checkout (only what was born before it was let go) or a temp root. */
+export type SweepRoot = {
+    path: string;
+    kind: "checkout" | "test temp";
+    bornBefore?: number;
+    repo?: string | null;
+    taskId?: string | null;
+};
+/** The root `located` runs from inside of, if any (its working folder first, then its executable). */
+export declare function rootOf(located: Located | undefined, roots: readonly SweepRoot[]): SweepRoot | null;
+export type OrphanTarget = {
+    row: ProcessRow;
+    root: SweepRoot;
+    orphan: number;
+};
+/**
+ * Which processes go: each orphan (parent 1, no terminal, not protected) running from inside a root — for a checkout,
+ * born before it was let go — and every process descending from one that goes (none at a terminal or protected).
+ */
+export declare function orphanTargets(census: readonly ProcessRow[], located: ReadonlyMap<number, Located>, roots: readonly SweepRoot[], guard: {
+    pids: ReadonlySet<number>;
+    groups: ReadonlySet<number>;
+}): OrphanTarget[];
+/** Whether a temp root's owner (pid and start) is still the running process. */
+export declare function ownerRunning(census: readonly ProcessRow[]): (owner: TempOwner) => boolean;
+/**
+ * The roots orphans may be stopped from now: every checkout let go with no live run on its task and not held, and
+ * every test temp root whose owner is gone or that nothing touched for a day.
+ */
+export declare function sweepRoots(store: Store, pool: WorktreePool | null, census: readonly ProcessRow[], tempRoots: readonly string[], now: Date): SweepRoot[];
+export type OrphanStop = {
+    pid: number;
+    name: string;
+    where: string;
+    kind: SweepRoot["kind"];
+    ranMs: number;
+    killed: boolean;
+};
+export type OrphanSweep = {
+    stopped: OrphanStop[];
+    failed: string[];
+};
+export type OrphanOptions = StopOptions & {
+    actor?: string;
+    pool?: WorktreePool | null;
+    tempRoots?: readonly string[];
+    locate?: (pids: readonly number[]) => Map<number, Located>;
+    self?: number;
+};
+/** Stop the orphans Toolroll left (see the top of this file), each in the ledger. */
+export declare function sweepOrphans(store: Store, now: () => Date, options?: OrphanOptions): Promise<OrphanSweep>;
+export declare function durationWords(ms: number): string;

package/dist/orphan-sweep.js

@@ -0,0 +1,235 @@
+/**
+ * Orphans Toolroll left running (the daily storage sweep, storage-sweep.ts): a process whose parent is gone (parent
+ * pid 1) that runs from inside a task checkout whose run has ended, or from inside a test temp root whose suite is
+ * gone — a preview server a run started and never stopped (Oct 4: `astro preview --port 4399` from a toolroll-site
+ * checkout, 4 days on), a browser a killed journey left. It is stopped with whatever it started (SIGTERM, then
+ * SIGKILL after the grace), and each one is in the ledger.
+ *
+ * "Runs from inside" is its working folder or its executable, read from the OS (lsof on macOS, /proc on Linux) —
+ * never its arguments, which can carry private provider inputs. An observation that can't be read stops nothing.
+ *
+ * Never touched: anything whose parent is alive (a person's shell, the lead's deploy), anything at a terminal, the
+ * live service and its ancestry, anything in a checkout leased or with a live run, a process born after its
+ * checkout was let go (somebody started it since), anything in a temp root whose owner still runs or that something
+ * touched within the day, and anything outside Toolroll's checkouts and test temp roots. Each target is looked at
+ * again right before the signal: the same process (pid and birth), still an orphan, still inside.
+ */
+import { execFileSync } from "node:child_process";
+import { existsSync, readlinkSync, realpathSync } from "node:fs";
+import { basename, sep } from "node:path";
+import { processCensus, protectedProcesses, stopProcesses } from "./run-closeout.js";
+import { tempOwner, testTempFolders } from "./test-temp.js";
+const DAY_MS = 86_400_000;
+/** `ps` ages are whole seconds. */
+const BIRTH_SLACK_MS = 2_000;
+/** Working folder and executable of each pid: /proc on Linux, lsof on macOS. Throws when the OS can't be asked. */
+export function locateProcesses(pids) {
+    const found = new Map();
+    if (pids.length === 0 || process.platform === "win32")
+        return found;
+    if (process.platform === "linux") {
+        for (const pid of pids) {
+            const one = {};
+            try {
+                one.cwd = readlinkSync(`/proc/${pid}/cwd`);
+            }
+            catch { /* gone, or not ours */ }
+            try {
+                one.exe = readlinkSync(`/proc/${pid}/exe`);
+            }
+            catch { /* gone, or not ours */ }
+            if (one.cwd !== undefined || one.exe !== undefined)
+                found.set(pid, one);
+        }
+        return found;
+    }
+    for (let at = 0; at < pids.length; at += 400) {
+        let text;
+        try {
+            text = execFileSync("lsof", ["-nP", "-a", "-p", pids.slice(at, at + 400).join(","), "-d", "cwd,txt", "-Fpfn"], { encoding: "utf8", timeout: 30_000, maxBuffer: 64 * 1024 * 1024, stdio: ["ignore", "pipe", "ignore"] });
+        }
+        catch (error) {
+            // lsof exits 1 when some pid has gone or can't be read; what it printed still stands. No output at all: unreadable.
+            const out = error.stdout;
+            if (typeof out !== "string" || error.status !== 1)
+                throw new Error("lsof could not list working folders");
+            text = out;
+        }
+        let pid = 0;
+        let fd = "";
+        for (const line of text.split("\n")) {
+            if (line.startsWith("p")) {
+                pid = Number(line.slice(1));
+                fd = "";
+            }
+            else if (line.startsWith("f"))
+                fd = line.slice(1);
+            else if (line.startsWith("n") && pid > 0) {
+                const one = found.get(pid) ?? {};
+                // The first txt is the executable; the rest are libraries it loaded.
+                if (fd === "cwd")
+                    one.cwd = line.slice(1);
+                else if (fd === "txt" && one.exe === undefined)
+                    one.exe = line.slice(1);
+                found.set(pid, one);
+            }
+        }
+    }
+    return found;
+}
+function canonical(path) {
+    let real = path;
+    try {
+        real = realpathSync(path);
+    }
+    catch { /* gone */ }
+    return [...new Set([path, real])];
+}
+/** The root `located` runs from inside of, if any (its working folder first, then its executable). */
+export function rootOf(located, roots) {
+    if (located === undefined)
+        return null;
+    for (const at of [located.cwd, located.exe]) {
+        if (at === undefined || at === "")
+            continue;
+        for (const root of roots) {
+            if (canonical(root.path).some(base => base !== "/" && base !== "" && (at === base || at.startsWith(base + sep))))
+                return root;
+        }
+    }
+    return null;
+}
+/**
+ * Which processes go: each orphan (parent 1, no terminal, not protected) running from inside a root — for a checkout,
+ * born before it was let go — and every process descending from one that goes (none at a terminal or protected).
+ */
+export function orphanTargets(census, located, roots, guard) {
+    const safe = (row) => row.pid > 1 && !row.terminal && !guard.pids.has(row.pid) && !guard.groups.has(row.pgid);
+    const out = new Map();
+    for (const row of census) {
+        if (row.ppid !== 1 || !safe(row))
+            continue;
+        const root = rootOf(located.get(row.pid), roots);
+        if (root === null)
+            continue;
+        if (root.bornBefore !== undefined && row.bornAt > root.bornBefore + BIRTH_SLACK_MS)
+            continue;
+        out.set(row.pid, { row, root, orphan: row.pid });
+    }
+    // Whatever an orphan that goes started goes with it.
+    const children = new Map();
+    for (const row of census)
+        children.set(row.ppid, [...(children.get(row.ppid) ?? []), row]);
+    for (const top of [...out.values()]) {
+        const pending = [...(children.get(top.row.pid) ?? [])];
+        while (pending.length > 0) {
+            const row = pending.pop();
+            if (out.has(row.pid) || !safe(row))
+                continue;
+            out.set(row.pid, { row, root: top.root, orphan: top.orphan });
+            pending.push(...(children.get(row.pid) ?? []));
+        }
+    }
+    return [...out.values()];
+}
+/** Whether a temp root's owner (pid and start) is still the running process. */
+export function ownerRunning(census) {
+    const byPid = new Map(census.map(row => [row.pid, row]));
+    return owner => { const now = byPid.get(owner.pid); return now !== undefined && Math.abs(now.bornAt - owner.startedAt) <= BIRTH_SLACK_MS; };
+}
+/**
+ * The roots orphans may be stopped from now: every checkout let go with no live run on its task and not held, and
+ * every test temp root whose owner is gone or that nothing touched for a day.
+ */
+export function sweepRoots(store, pool, census, tempRoots, now) {
+    const roots = [];
+    const live = store.handle.prepare("SELECT 1 FROM run WHERE task_ref = ? AND finished_at IS NULL LIMIT 1");
+    for (const row of store.listWorktrees()) {
+        if (row.releasedAt === null || row.runner !== null || !existsSync(row.path))
+            continue;
+        if (pool !== null && pool.inUse(row.path).held)
+            continue;
+        if (row.taskRef !== null && live.get(row.taskRef) !== undefined)
+            continue;
+        const releasedAt = Date.parse(row.releasedAt);
+        if (!Number.isFinite(releasedAt))
+            continue;
+        roots.push({ path: row.path, kind: "checkout", bornBefore: releasedAt, repo: row.repo, taskId: row.taskRef === null ? null : store.externalIdFor(row.taskRef) });
+    }
+    const running = ownerRunning(census);
+    const found = testTempFolders(tempRoots, now);
+    for (const one of found.all) {
+        const owner = tempOwner(one.path);
+        const gone = owner !== null ? !running(owner) : now.getTime() - one.touchedAt > DAY_MS;
+        if (gone)
+            roots.push({ path: one.path, kind: "test temp" });
+    }
+    return roots;
+}
+/** Stop the orphans Toolroll left (see the top of this file), each in the ledger. */
+export async function sweepOrphans(store, now, options = {}) {
+    if (process.platform === "win32")
+        return { stopped: [], failed: [] };
+    const read = options.census ?? (() => processCensus());
+    const locate = options.locate ?? locateProcesses;
+    const failed = [];
+    let census;
+    try {
+        census = read();
+    }
+    catch {
+        return { stopped: [], failed: ["the process list couldn't be read"] };
+    }
+    const roots = sweepRoots(store, options.pool ?? null, census, options.tempRoots ?? [], now());
+    if (roots.length === 0)
+        return { stopped: [], failed };
+    const guard = protectedProcesses(census, options.self);
+    const orphans = census.filter(row => row.ppid === 1 && row.pid > 1 && !row.terminal && !guard.pids.has(row.pid) && !guard.groups.has(row.pgid)).map(row => row.pid);
+    let located;
+    try {
+        located = locate(orphans);
+    }
+    catch (error) {
+        return { stopped: [], failed: [`working folders couldn't be read: ${error.message}`] };
+    }
+    const first = orphanTargets(census, located, roots, guard);
+    if (first.length === 0)
+        return { stopped: [], failed };
+    // Right before the signal, everything again: the same process, still an orphan (or under one), still inside.
+    let again;
+    try {
+        const fresh = read();
+        const freshGuard = protectedProcesses(fresh, options.self);
+        const freshLocated = locate(fresh.filter(row => row.ppid === 1).map(row => row.pid));
+        const before = new Map(first.map(one => [one.row.pid, one]));
+        again = orphanTargets(fresh, freshLocated, roots, freshGuard).filter(one => {
+            const was = before.get(one.row.pid);
+            return was !== undefined && Math.abs(was.row.bornAt - one.row.bornAt) <= BIRTH_SLACK_MS && was.root.path === one.root.path;
+        });
+    }
+    catch (error) {
+        return { stopped: [], failed: [`a second look failed: ${error.message}`] };
+    }
+    if (again.length === 0)
+        return { stopped: [], failed };
+    const done = await stopProcesses(again.map(one => one.row), options);
+    const killed = new Set(done.killed.map(row => row.pid));
+    const at = now();
+    const stopped = [];
+    for (const one of again) {
+        const ranMs = Math.max(0, at.getTime() - one.row.bornAt);
+        const where = basename(one.root.path);
+        stopped.push({ pid: one.row.pid, name: one.row.name, where: one.root.path, kind: one.root.kind, ranMs, killed: killed.has(one.row.pid) });
+        store.recordAction({ at: at.toISOString(), actor: options.actor ?? "worker", repo: one.root.repo ?? null, taskId: one.root.taskId ?? null, runId: null,
+            action: "orphan process stopped", outcome: "stopped", source: "work",
+            detail: `${one.row.name} (${one.row.pid}) running ${durationWords(ranMs)} from ${one.root.kind === "checkout" ? "checkout" : "test temp folder"} ${where}${one.orphan === one.row.pid ? "; its parent had gone" : `; started by orphan ${one.orphan}`}${killed.has(one.row.pid) ? "; needed SIGKILL" : ""}` });
+    }
+    return { stopped, failed };
+}
+export function durationWords(ms) {
+    const minutes = Math.round(ms / 60_000);
+    if (minutes < 60)
+        return `${minutes} min`;
+    const hours = Math.round(minutes / 60);
+    return hours < 48 ? `${hours} h` : `${Math.round(hours / 24)} days`;
+}

package/dist/result-follow-ups.d.ts

@@ -8,6 +8,19 @@ export declare const FOLLOW_UP_LOG = "follow-up check log v1";
 export declare const FOLLOW_UP_RECEIPT = "follow-up check receipt v1";
 export declare const ADD_TESTS_ACTION = "tests task filed";
 export type RunLevel = Exclude<CheckLevel, "off">;
+/** How a batch check (batch-checks.ts) reached this result's outcome. `together`: one check on a temporary
+ * merge of `members`; `split`: its batch failed, so it was checked on its own; `conflict`: it couldn't be merged
+ * with the others, so it was checked on its own; `alone`: nothing joined it within the window. */
+export type BatchFacts = {
+    mode: "together" | "split" | "conflict" | "alone";
+    tested: string;
+    base: string | null;
+    members: {
+        task: string;
+        run: number;
+        head: string;
+    }[];
+};
 export type FollowUpCheck = {
     request: number;
     runId: number;
@@ -15,13 +28,19 @@ export type FollowUpCheck = {
     head: string;
     actor: string;
     at: string;
-    why: "person" | "pull-request";
+    why: "person" | "pull-request" | "batch";
     /** The approved command's digest when checks were asked for: only that exact command runs. */
     digest: string;
     state: "waiting" | "running" | "passed" | "failed" | "not-run";
     exitCode: number | null;
     logArtifactId: number | null;
     note: string | null;
+    /** A batch request: the project base it is merged onto. */
+    base?: string | null;
+    /** The commit the check actually ran on: the result's own, or a temporary batch commit. Null until it ran. */
+    tested?: string | null;
+    /** From the sealed receipt of a batch check. */
+    batch?: BatchFacts | null;
 };
 type Refusal = {
     ok: false;
@@ -43,8 +62,9 @@ export declare function requestFollowUpChecks(store: Store, input: {
     level: RunLevel;
     existing: boolean;
 } | Refusal;
-/** Requests nobody has started yet, oldest first. */
+/** Requests nobody has started yet, oldest first. Batch requests wait for the batch pass (batch-checks.ts). */
 export declare function waitingCheckRequests(store: Store, now: Date): FollowUpCheck[];
+export declare function waitingRequestsOfAnyKind(store: Store, now: Date): FollowUpCheck[];
 export type CheckRunDeps = {
     runner?: Runner;
     git?: Runner;

package/dist/result-follow-ups.js

@@ -47,28 +47,34 @@ export function followUpChecksOf(store, runId, now = new Date(), root) {
                 const body = JSON.parse(read.content.toString("utf8"));
                 if (typeof body.request !== "number")
                     continue;
+                const head = typeof body.head === "string" ? body.head : null;
                 receipts.set(body.request, { ran: body.result?.ran === true, exitCode: typeof body.result?.exitCode === "number" ? body.result.exitCode : null,
-                    log: typeof body.log?.artifactId === "number" ? body.log.artifactId : null });
+                    log: typeof body.log?.artifactId === "number" ? body.log.artifactId : null, head,
+                    tested: typeof body.tested === "string" ? body.tested : head, batch: batchFactsOf(body.batch, runId, head) });
             }
             catch { /* an unreadable receipt upgrades nothing */ }
         }
     }
     return rows.filter(row => row.action === CHECKS_REQUESTED && (row.outcome === "quick" || row.outcome === "full")).map(request => {
-        const [head = "", why = "person", digest = ""] = (request.detail ?? "").split(" · ");
+        const [head = "", why = "person", digest = "", recordedBase = ""] = (request.detail ?? "").split(" · ");
         const started = rows.find(row => row.action === CHECKS_STARTED && row.outcome === String(request.id)) ?? null;
         const finished = rows.find(row => row.action === CHECKS_FINISHED && row.outcome.startsWith(`${request.id}:`)) ?? null;
         const base = { request: request.id, runId, level: request.outcome, head, digest, actor: request.actor, at: request.at,
-            why: why === "pull-request" ? "pull-request" : "person", exitCode: null, logArtifactId: null, note: null };
+            why: why === "pull-request" ? "pull-request" : why === "batch" ? "batch" : "person", exitCode: null, logArtifactId: null, note: null,
+            ...(why === "batch" ? { base: /^[a-f0-9]{40}$/.test(recordedBase) ? recordedBase : null } : {}) };
         if (finished !== null) {
             const status = finished.outcome.slice(String(request.id).length + 1);
             const sealed = receipts.get(request.id) ?? null;
-            // With the evidence at hand, only a sealed receipt can say passed.
-            if (status === "passed" && root !== undefined && (sealed === null || !sealed.ran || sealed.exitCode !== 0)) {
+            // With the evidence at hand, only a sealed receipt can say passed: for this result's own commit, and,
+            // for a batch, a batch that names this result.
+            if (status === "passed" && root !== undefined && (sealed === null || !sealed.ran || sealed.exitCode !== 0 || (sealed.head !== null && sealed.head !== head)
+                || (base.why === "batch" && sealed.batch === null))) {
                 return { ...base, state: "not-run", note: "The saved check result could not be verified." };
             }
             const exit = /exit (\d+)/.exec(finished.detail ?? "");
             return { ...base, state: status === "passed" ? "passed" : status === "failed" ? "failed" : "not-run",
-                exitCode: exit === null ? sealed?.exitCode ?? null : Number(exit[1]), logArtifactId: sealed?.log ?? null, note: status === "not-run" ? finished.detail : null };
+                exitCode: exit === null ? sealed?.exitCode ?? null : Number(exit[1]), logArtifactId: sealed?.log ?? null, note: status === "not-run" ? finished.detail : null,
+                ...(sealed === null ? {} : { tested: sealed.ran ? sealed.tested : null, batch: sealed.batch }) };
         }
         if (started !== null) {
             const limit = Number(/limit (\d+)/.exec(started.detail ?? "")?.[1] ?? 3_600_000);
@@ -79,6 +85,26 @@ export function followUpChecksOf(store, runId, now = new Date(), root) {
         return { ...base, state: "waiting" };
     });
 }
+/** A receipt's batch facts, only when they are whole and name this result at its own commit. */
+function batchFactsOf(value, runId, head) {
+    if (value === null || typeof value !== "object")
+        return null;
+    const raw = value;
+    const modes = ["together", "split", "conflict", "alone"];
+    const mode = modes.find(one => one === raw.mode);
+    if (mode === undefined || typeof raw.tested !== "string" || !/^[a-f0-9]{40}$/.test(raw.tested) || !Array.isArray(raw.members))
+        return null;
+    const members = raw.members.flatMap(one => {
+        const m = one;
+        return typeof m?.task === "string" && typeof m.run === "number" && typeof m.head === "string" ? [{ task: m.task, run: m.run, head: m.head }] : [];
+    });
+    if (members.length !== raw.members.length || !members.some(one => one.run === runId && one.head === head))
+        return null;
+    // Checked on its own: the commit tested is the result's own.
+    if (mode !== "together" && (members.length !== 1 || raw.tested !== head))
+        return null;
+    return { mode, tested: raw.tested, base: typeof raw.base === "string" ? raw.base : null, members };
+}
 /** Whether a result can have checks run on it, and with which command. */
 function checkTarget(store, runId, level) {
     const run = store.getRun(runId);
@@ -110,8 +136,11 @@ export function requestFollowUpChecks(store, input, now) {
         return { ok: true, request, level: target.level, existing: false };
     });
 }
-/** Requests nobody has started yet, oldest first. */
+/** Requests nobody has started yet, oldest first. Batch requests wait for the batch pass (batch-checks.ts). */
 export function waitingCheckRequests(store, now) {
+    return waitingRequestsOfAnyKind(store, now).filter(one => one.why !== "batch");
+}
+export function waitingRequestsOfAnyKind(store, now) {
     const runs = store.handle.prepare(`SELECT DISTINCT r.run_id AS run FROM action_ledger r WHERE r.action = ? AND r.run_id IS NOT NULL
     AND NOT EXISTS (SELECT 1 FROM action_ledger s WHERE s.run_id = r.run_id AND s.action = ? AND s.outcome = CAST(r.id AS TEXT)) ORDER BY r.id`)
         .all(CHECKS_REQUESTED, CHECKS_STARTED).map(row => Number(row["run"]));
@@ -204,7 +233,7 @@ export async function runFollowUpCheck(store, root, request, deps = {}) {
     const text = log.length === 0 ? "No command started." : log.join("\n\n");
     const logId = storeEvidence(store, root, runId, "structured-output", `follow-up-check-${request}.log`, Buffer.from(text, "utf8"), FOLLOW_UP_LOG, at, { captureStatus: "ok" });
     const logArtifact = store.getArtifact(logId);
-    const receipt = JSON.stringify({ version: 1, request, run: runId, head: target.head, level: target.level,
+    const receipt = JSON.stringify({ version: 1, request, run: runId, head: target.head, tested: target.head, level: target.level,
         command: { repo: target.command.repo, command: target.command.command, digest: target.command.digest, approvedBy: target.command.approvedBy },
         result, log: { artifactId: logArtifact.id, sha256: logArtifact.sha256, bytesStored: logArtifact.bytesStored, truncated: logArtifact.truncated } }, null, 1);
     storeEvidence(store, root, runId, "structured-output", `follow-up-check-${request}.json`, Buffer.from(receipt, "utf8"), FOLLOW_UP_RECEIPT, at, { captureStatus: "ok" });
@@ -262,9 +291,14 @@ export function fullCheckGate(store, runId, now) {
     const level = runCheckLevel(store, runId);
     const repo = repoOfRun(store, runId);
     const run = store.getRun(runId);
-    if (level === null || level === "full" || repo === null || run === null || store.liveVerifyCommand(repo) === null)
+    if (level === null || repo === null || run === null)
+        return { state: "clear" };
+    const fulls = followUpChecksOf(store, runId, now).filter(one => one.level === "full" && one.head === run.headRevision);
+    // A Full build that waited for a batch check has had no check of its own: Merge waits for the batch, like Quick does.
+    const batched = level === "full" && fulls.some(one => one.why === "batch");
+    if ((level === "full" && !batched) || (!batched && store.liveVerifyCommand(repo) === null))
         return { state: "clear" };
-    const last = followUpChecksOf(store, runId, now).filter(one => one.level === "full" && one.head === run.headRevision).at(-1);
+    const last = fulls.at(-1);
     if (last === undefined)
         return { state: "not-requested", message: "The full check hasn't run on this commit yet. Merge once it passes, or merge anyway." };
     if (last.state === "passed")