toolroll 0.9.31 → 0.9.33

package/dist/storage-sweep.d.ts

@@ -0,0 +1,50 @@
+import { type Located } from "./orphan-sweep.js";
+import { type ProcessRow } from "./run-closeout.js";
+import type { Store } from "./store.js";
+import type { WorktreePool } from "./worktree.js";
+export declare const SWEEP_EVERY_MS = 86400000;
+/** A sweep that failed somewhere, or left work for later, runs again this soon. */
+export declare const RETRY_MS: number;
+export type SweepKind = "test temp" | "orphans" | "checkouts" | "runtimes" | "dependencies";
+export type SweepPart = {
+    kind: SweepKind;
+    count: number;
+    bytes: number;
+    failed: number;
+    more?: boolean;
+    problem?: string;
+    items: string[];
+};
+export type SweepRecord = {
+    at: string;
+    source: "automatic" | "manual";
+    actor: string;
+    parts: SweepPart[];
+};
+export type SweepDeps = {
+    databaseFile: string;
+    pool: WorktreePool | null;
+    tempRoots: readonly string[];
+    actor?: string;
+    census?: () => ProcessRow[];
+    locate?: (pids: readonly number[]) => Map<number, Located>;
+    inUse?: () => string[];
+    graceMs?: number;
+};
+export declare function storageSweepOff(env?: NodeJS.ProcessEnv): boolean;
+export declare function storageSweepDue(store: Store, now: Date): boolean;
+/** The daily sweep, when it's due and on: one worker takes it. Null when it wasn't this pass's to run. */
+export declare function dailyStorageSweep(store: Store, deps: SweepDeps, now: () => Date): Promise<SweepRecord | null>;
+/** One sweep, now, every part; saved and in the ledger. Each part's failure is its own: the others still run. */
+export declare function sweepStorage(store: Store, deps: SweepDeps, now: () => Date): Promise<SweepRecord>;
+/** Save what a sweep or a clean-up by hand did (the latest few are kept), with one ledger entry saying it. */
+export declare function saveSweep(store: Store, record: SweepRecord): SweepRecord;
+/** The latest sweep (of `source`, or either), or null when there has been none. */
+export declare function lastSweep(store: Store, source?: SweepRecord["source"]): SweepRecord | null;
+/** When the next automatic sweep is due, or null when it is off. */
+export declare function nextSweepAt(store: Store): string | null;
+export declare function partWords(one: Pick<SweepPart, "kind" | "count">): string;
+/** "Removed 120 test temp folders and 2 finished checkouts, about 1.2 GB. Stopped 1 leftover process. 3 couldn't be removed." */
+export declare function sweepWords(parts: readonly SweepPart[]): string;
+/** The details a disclosure shows: each part's paths, problems and what's left for later, one line each. */
+export declare function sweepDetails(parts: readonly SweepPart[]): string[];

package/dist/storage-sweep.js

@@ -0,0 +1,203 @@
+/**
+ * The daily storage sweep: the plane cleans up after itself without anyone running a command. Once a day (one worker
+ * takes the day: the cursor moves in the same transaction) it removes test temp folders nothing touched for a day
+ * (test-temp.ts), stops orphans Toolroll left running (orphan-sweep.ts), removes finished checkouts the cleanup
+ * setting lets go (checkout-cleanup.ts), keeps the current, previous and newest staged runtimes (staged-runtimes.ts)
+ * and removes shared dependency installs no checkout has used for a week (shared-deps.ts).
+ *
+ * The everyday work stays where it happens: suites remove their own temp roots and processes, a run's processes stop
+ * when it ends and a finished checkout goes on the worker's next pass. This is the net for what slipped through.
+ *
+ * What each sweep did is saved (the latest few) and in the ledger: `toolroll storage` and Settings → Storage show it.
+ * A clean-up by hand (`storage clean --yes`) is saved the same way. A part that failed, or had more than one pass
+ * takes, brings the next sweep forward to an hour later. `TOOLROLL_STORAGE_SWEEP=off` turns the automatic sweep off
+ * (the unit tests do, so a test never sweeps the machine it runs on).
+ */
+import { dirname } from "node:path";
+import { cleanCheckouts } from "./checkout-cleanup.js";
+import { durationWords, sweepOrphans, ownerRunning } from "./orphan-sweep.js";
+import { processCensus } from "./run-closeout.js";
+import { pruneUnusedShared, sharedDepsRoot, sharedUse, SHARED_UNUSED_MS } from "./shared-deps.js";
+import { pruneStagedRuntimes, runtimesInUse, stagePlan } from "./staged-runtimes.js";
+import { bytesWords, diskBytes } from "./storage.js";
+import { removeStaleTestTemp } from "./test-temp.js";
+const CURSOR = "storage-sweep";
+export const SWEEP_EVERY_MS = 86_400_000;
+/** A sweep that failed somewhere, or left work for later, runs again this soon. */
+export const RETRY_MS = 60 * 60_000;
+/** Sweeps kept. */
+const KEEP = 30;
+/** Test temp folders one sweep removes at most (Oct 4 found 107,418). */
+const TEMP_MAX = 20_000;
+/** Paths saved per part, for the disclosure. */
+const ITEMS_MAX = 20;
+export function storageSweepOff(env = process.env) {
+    return (env["TOOLROLL_STORAGE_SWEEP"] ?? "").trim().toLowerCase() === "off";
+}
+export function storageSweepDue(store, now) {
+    return now.getTime() - store.serviceCursor(CURSOR) >= SWEEP_EVERY_MS;
+}
+/** The daily sweep, when it's due and on: one worker takes it. Null when it wasn't this pass's to run. */
+export async function dailyStorageSweep(store, deps, now) {
+    if (storageSweepOff())
+        return null;
+    const at = now();
+    const mine = store.transact(() => {
+        if (!storageSweepDue(store, at))
+            return false;
+        store.setServiceCursor(CURSOR, at.getTime(), at);
+        return true;
+    });
+    if (!mine)
+        return null;
+    const record = await sweepStorage(store, deps, now);
+    // Something failed or was left for later: the next sweep comes an hour from now, not tomorrow.
+    if (record.parts.some(one => one.failed > 0 || one.more === true || one.problem !== undefined)) {
+        store.setServiceCursor(CURSOR, Math.max(0, at.getTime() - SWEEP_EVERY_MS + RETRY_MS), now());
+    }
+    return record;
+}
+const capped = (items) => items.slice(0, ITEMS_MAX);
+const failure = (error) => error instanceof Error ? error.message : String(error);
+/** One sweep, now, every part; saved and in the ledger. Each part's failure is its own: the others still run. */
+export async function sweepStorage(store, deps, now) {
+    const actor = deps.actor ?? "worker";
+    const read = deps.census ?? (() => processCensus());
+    const parts = [];
+    const part = async (kind, work) => {
+        try {
+            parts.push({ kind, ...await work() });
+        }
+        catch (error) {
+            parts.push({ kind, count: 0, bytes: 0, failed: 0, problem: failure(error), items: [] });
+        }
+    };
+    await part("test temp", async () => {
+        // A root whose owner still runs stays, however old it looks; with no process list, every owned root stays.
+        let alive = () => true;
+        try {
+            alive = ownerRunning(read());
+        }
+        catch { /* owned roots stay */ }
+        const done = removeStaleTestTemp(deps.tempRoots, now(), { ownerAlive: alive, max: TEMP_MAX });
+        return { count: done.removed.length, bytes: 0, failed: done.failed.length, ...(done.more ? { more: true } : {}), items: capped(done.removed) };
+    });
+    await part("orphans", async () => {
+        const done = await sweepOrphans(store, now, {
+            actor, pool: deps.pool, tempRoots: deps.tempRoots, census: read,
+            ...(deps.locate === undefined ? {} : { locate: deps.locate }), ...(deps.graceMs === undefined ? {} : { graceMs: deps.graceMs }),
+        });
+        return {
+            count: done.stopped.length, bytes: 0, failed: 0, ...(done.failed.length === 0 ? {} : { problem: done.failed.join("; ") }),
+            items: capped(done.stopped.map(one => `${one.name} (${one.pid}), ${durationWords(one.ranMs)}, in ${one.where}`)),
+        };
+    });
+    if (deps.pool !== null) {
+        const pool = deps.pool;
+        await part("checkouts", async () => {
+            const done = await cleanCheckouts(store, pool, now, { manual: false, actor });
+            return { count: done.removed.length, bytes: done.freed, failed: done.kept.filter(one => one.why === "git refused").length, items: capped(done.removed.map(one => one.path)) };
+        });
+    }
+    const stateDir = dirname(deps.databaseFile);
+    await part("runtimes", async () => {
+        const inUse = deps.inUse ?? (() => runtimesInUse());
+        const plan = stagePlan(stateDir, inUse(), now());
+        const sizes = diskBytes(plan.go.map(one => one.path));
+        const done = pruneStagedRuntimes(stateDir, now(), { inUse });
+        return {
+            count: done.removed.length, bytes: done.removed.reduce((sum, path) => sum + (sizes.get(path) ?? 0), 0), failed: done.failed.length,
+            ...(done.refused === null ? {} : { problem: `kept every staged runtime: ${done.refused}` }), items: capped(done.removed),
+        };
+    });
+    await part("dependencies", async () => {
+        const root = sharedDepsRoot(deps.databaseFile);
+        const checkouts = () => store.listWorktrees().map(row => ({ path: row.path, repo: row.repo }));
+        const candidates = sharedUse(root, checkouts()).filter(one => one.checkouts.length === 0 && now().getTime() - Date.parse(one.usedAt) > SHARED_UNUSED_MS);
+        const sizes = diskBytes(candidates.map(one => one.dir));
+        const done = pruneUnusedShared(root, checkouts, now());
+        return { count: done.removed.length, bytes: done.removed.reduce((sum, one) => sum + (sizes.get(one.dir) ?? 0), 0), failed: done.failed.length, items: capped(done.removed.map(one => one.dir)) };
+    });
+    return saveSweep(store, { at: now().toISOString(), source: "automatic", actor, parts });
+}
+/** Save what a sweep or a clean-up by hand did (the latest few are kept), with one ledger entry saying it. */
+export function saveSweep(store, record) {
+    store.transact(() => {
+        store.handle.prepare("INSERT INTO storage_sweep (at, source, actor, result) VALUES (?, ?, ?, ?)").run(record.at, record.source, record.actor, JSON.stringify(record.parts));
+        store.handle.prepare("DELETE FROM storage_sweep WHERE id NOT IN (SELECT id FROM storage_sweep ORDER BY id DESC LIMIT ?)").run(KEEP);
+    });
+    const did = record.parts.some(one => one.count > 0);
+    const failed = record.parts.some(one => one.failed > 0 || one.problem !== undefined);
+    store.recordAction({ at: record.at, actor: record.actor, repo: null, taskId: null, runId: null, action: "storage sweep",
+        outcome: failed ? "partial" : did ? "removed" : "nothing due", source: record.source === "automatic" ? "policy" : "request", detail: sweepWords(record.parts) });
+    return record;
+}
+function parseParts(text) {
+    try {
+        const parsed = JSON.parse(text);
+        return Array.isArray(parsed) ? parsed.filter((one) => typeof one === "object" && one !== null && typeof one.kind === "string") : [];
+    }
+    catch {
+        return [];
+    }
+}
+/** The latest sweep (of `source`, or either), or null when there has been none. */
+export function lastSweep(store, source) {
+    const row = store.handle.prepare(`SELECT at, source, actor, result FROM storage_sweep${source === undefined ? "" : " WHERE source = ?"} ORDER BY id DESC LIMIT 1`)
+        .get(...(source === undefined ? [] : [source]));
+    if (row === undefined)
+        return null;
+    return { at: String(row["at"]), source: row["source"] === "manual" ? "manual" : "automatic", actor: String(row["actor"]), parts: parseParts(String(row["result"])) };
+}
+/** When the next automatic sweep is due, or null when it is off. */
+export function nextSweepAt(store) {
+    if (storageSweepOff())
+        return null;
+    const last = store.serviceCursor(CURSOR);
+    return last === 0 ? null : new Date(last + SWEEP_EVERY_MS).toISOString();
+}
+const NOUNS = {
+    "test temp": ["test temp folder", "test temp folders"],
+    orphans: ["leftover process", "leftover processes"],
+    checkouts: ["finished checkout", "finished checkouts"],
+    runtimes: ["staged runtime", "staged runtimes"],
+    dependencies: ["unused dependency install", "unused dependency installs"],
+};
+/** What a part looks at, for "Couldn't check …". */
+const CHECKED = { "test temp": "test temp folders", orphans: "processes", checkouts: "checkouts", runtimes: "staged runtimes", dependencies: "dependency installs" };
+export function partWords(one) {
+    return `${one.count.toLocaleString("en-US")} ${NOUNS[one.kind][one.count === 1 ? 0 : 1]}`;
+}
+function list(items) {
+    return items.length <= 1 ? items.join("") : `${items.slice(0, -1).join(", ")} and ${items.at(-1)}`;
+}
+/** "Removed 120 test temp folders and 2 finished checkouts, about 1.2 GB. Stopped 1 leftover process. 3 couldn't be removed." */
+export function sweepWords(parts) {
+    const removed = parts.filter(one => one.kind !== "orphans" && one.count > 0);
+    const stopped = parts.find(one => one.kind === "orphans" && one.count > 0);
+    const bytes = removed.reduce((sum, one) => sum + one.bytes, 0);
+    const failed = parts.reduce((sum, one) => sum + one.failed, 0);
+    const problems = parts.filter(one => one.problem !== undefined).map(one => one.kind);
+    const said = [
+        ...(removed.length === 0 ? [] : [`Removed ${list(removed.map(partWords))}${bytes > 0 ? `, about ${bytesWords(bytes)}` : ""}.`]),
+        ...(stopped === undefined ? [] : [`Stopped ${partWords(stopped)}.`]),
+        ...(failed === 0 ? [] : [`${failed} couldn't be removed.`]),
+        ...(problems.length === 0 ? [] : [`Couldn't check ${list(problems.map(kind => CHECKED[kind]))}.`]),
+    ];
+    return said.length === 0 ? "Nothing was due." : said.join(" ");
+}
+/** The details a disclosure shows: each part's paths, problems and what's left for later, one line each. */
+export function sweepDetails(parts) {
+    const lines = [];
+    for (const one of parts) {
+        if (one.problem !== undefined)
+            lines.push(`${NOUNS[one.kind][1][0].toUpperCase()}${NOUNS[one.kind][1].slice(1)}: ${one.problem}`);
+        if (one.more === true)
+            lines.push(`More ${NOUNS[one.kind][1]} are left for the next sweep.`);
+        for (const item of one.items)
+            lines.push(item);
+        if (one.items.length > 0 && one.count > one.items.length)
+            lines.push(`… and ${(one.count - one.items.length).toLocaleString("en-US")} more ${NOUNS[one.kind][1]}`);
+    }
+    return lines;
+}

package/dist/storage-ui.d.ts

@@ -1,8 +1,13 @@
 import { type CheckoutPlan } from "./checkout-cleanup.js";
+import { type SweepRecord } from "./storage-sweep.js";
 export declare const STORAGE_CSS: string;
 export type StorageView = {
     plan: CheckoutPlan;
     csrf: string;
+    sweep?: {
+        last: SweepRecord | null;
+        off: boolean;
+    };
 };
 export declare function storageHtml(view: StorageView, notice: {
     said?: string | null;

package/dist/storage-ui.js

@@ -1,12 +1,14 @@
 /**
  * Settings → Storage: how much space task checkouts use, what a clean-up would free now, a Clean up button whose
- * preview (what goes, what stays and why, with paths) sits behind the password, and when a finished task's clean
- * checkout goes by itself. An instance operator's page.
+ * preview (what goes, what stays and why, with paths) sits behind the password, what the last storage sweep removed
+ * and when (its paths in a disclosure), and when a finished task's clean checkout goes by itself. An instance
+ * operator's page.
  */
 import { homedir } from "node:os";
 import { basename } from "node:path";
 import { previewDigest, whyWords } from "./checkout-cleanup.js";
 import { CLEANUP_CHOICES, bytesWords } from "./storage.js";
+import { sweepDetails, sweepWords } from "./storage-sweep.js";
 const e = (value) => String(value ?? "").replace(/[&<>"']/g, c => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[c]);
 const plural = (count, one, many = `${one}s`) => `${count} ${count === 1 ? one : many}`;
 export const STORAGE_CSS = `.storage{max-width:720px;min-width:0;overflow-wrap:anywhere}.storage h2{font-size:1.0625rem;margin:24px 0 4px}` +
@@ -25,7 +27,31 @@ export const STORAGE_CSS = `.storage{max-width:720px;min-width:0;overflow-wrap:a
     `.storage form{display:grid;gap:10px;margin:0}.storage label{display:grid;gap:4px;font-size:.8125rem;font-weight:500;max-width:20rem}` +
     `.storage select,.storage input[type=password]{width:100%;box-sizing:border-box;min-height:2.125rem}.storage form button{justify-self:start;white-space:nowrap}` +
     `.storage form.setting button[type=submit],.storage .preview form.remove button[type=submit]{background:var(--primary);color:var(--primary-foreground);border-color:var(--primary);font-weight:600}` +
-    `@media (max-width:900px){.storage details.clean-up>summary,.storage form button{min-height:44px}.storage label{max-width:none}.storage select,.storage input[type=password]{min-height:44px;font-size:16px}.storage .preview form.remove button{width:100%}}`;
+    `.storage .sweep{display:grid;gap:4px}.storage .sweep p{margin:0}.storage .sweep .problem{color:var(--destructive)}` +
+    `.storage details.sweep-details>summary{cursor:pointer;min-height:2.25rem;display:inline-flex;align-items:center;font-size:.8125rem;color:var(--muted-foreground)}` +
+    `.storage details.sweep-details ul{margin:4px 0 0;padding-left:18px;font-family:var(--font-mono, ui-monospace, monospace);font-size:.6875rem;color:var(--muted-foreground);overflow-wrap:anywhere}` +
+    `@media (max-width:900px){.storage details.sweep-details>summary{min-height:44px}.storage details.clean-up>summary,.storage form button{min-height:44px}.storage label{max-width:none}.storage select,.storage input[type=password]{min-height:44px;font-size:16px}.storage .preview form.remove button{width:100%}}`;
+/** Paths under the home folder as ~/…, as the checkout list shows them. */
+function homeAsTilde(line) {
+    const home = homedir();
+    return home === "" || home === "/" ? line : line.split(`${home}/`).join("~/");
+}
+/** "Oct 4, 3:12 AM" in this computer's time. */
+function whenWords(at) {
+    const date = new Date(at);
+    return Number.isNaN(date.getTime()) ? at : date.toLocaleString("en-US", { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" });
+}
+/** One line: when the last sweep ran and what it did; its paths and problems behind Details. */
+function sweepHtml(sweep) {
+    const last = sweep.last;
+    if (last === null)
+        return `<p class="meta" data-last-sweep="">${sweep.off ? "Automatic sweep is off." : "Toolroll sweeps leftovers once a day. It hasn't run yet."}</p>`;
+    const who = last.source === "manual" ? `Cleaned up by ${e(last.actor)}` : "Swept";
+    const details = sweepDetails(last.parts);
+    const problem = last.parts.some(one => one.failed > 0 || one.problem !== undefined);
+    return `<div class="sweep" data-last-sweep="${e(last.at)}"><p${problem ? ' class="problem"' : ""}>${who} <time datetime="${e(last.at)}">${e(whenWords(last.at))}</time>. ${e(sweepWords(last.parts))}</p>` +
+        (details.length === 0 ? "" : `<details class="sweep-details"><summary>Details</summary><ul>${details.map(line => `<li>${e(homeAsTilde(line))}</li>`).join("")}</ul></details>`) + `</div>`;
+}
 function item(one, csrf) {
     const discard = one.why === "has changes"
         ? `<details class="discard"><summary>Discard</summary><form method="post" action="/settings/storage/discard"><input type="hidden" name="csrf" value="${e(csrf)}"><input type="hidden" name="path" value="${e(one.path)}">` +
@@ -58,7 +84,7 @@ export function storageHtml(view, notice) {
     const options = CLEANUP_CHOICES.map(one => `<option value="${one.value}"${one.value === plan.cleanup ? " selected" : ""}>${e(one.label)}</option>`).join("");
     return `<section class="storage">${note}` +
         `<div class="card storage-state"><h2 data-checkout-bytes="${plan.totalBytes}">Checkouts use ${e(bytesWords(plan.totalBytes))}</h2><p class="meta">${e(facts)}</p>` +
-        `<p data-clean-bytes="${plan.freeBytes}">${e(frees)}</p>${preview}</div>` +
+        `<p data-clean-bytes="${plan.freeBytes}">${e(frees)}</p>${view.sweep === undefined ? "" : sweepHtml(view.sweep)}${preview}</div>` +
         `<h2>Automatic cleanup</h2>` +
         `<form method="post" action="/settings/storage" class="setting">${hidden}` +
         `<label>Remove a finished task's clean checkout<select name="cleanup">${options}</select></label>` +

package/dist/storage.d.ts

@@ -1,6 +1,8 @@
 import type { Store } from "./store.js";
 /** Settings → Storage: when a finished task's clean checkout is removed. One row, or none (the default). */
 export declare const CHECKOUT_CLEANUP_SCHEMA = "\nCREATE TABLE IF NOT EXISTS checkout_cleanup_setting (\n  id         INTEGER PRIMARY KEY CHECK (id = 1),\n  cleanup    TEXT NOT NULL CHECK (cleanup IN ('finished', '2d', '7d', 'never')),\n  updated_by TEXT NOT NULL,\n  updated_at TEXT NOT NULL\n);\n";
+/** What each storage sweep or clean-up by hand did (storage-sweep.ts): the latest few. */
+export declare const STORAGE_SWEEP_SCHEMA = "\nCREATE TABLE IF NOT EXISTS storage_sweep (\n  id     INTEGER PRIMARY KEY,\n  at     TEXT NOT NULL,\n  source TEXT NOT NULL CHECK (source IN ('automatic', 'manual')),\n  actor  TEXT NOT NULL,\n  result TEXT NOT NULL\n);\n";
 export type CheckoutCleanup = "finished" | "2d" | "7d" | "never";
 export declare const DEFAULT_CLEANUP: CheckoutCleanup;
 export declare const CLEANUP_CHOICES: readonly {

package/dist/storage.js

@@ -3,7 +3,9 @@
  * beside the database, by kind, and what storage retention takes care of.
  * Build checkouts and staged releases are the big ones; the worker removes a
  * finished task's clean checkout as the checkout cleanup setting says
- * (checkout-cleanup.ts), and a deploy keeps its last few releases.
+ * (checkout-cleanup.ts), and the daily storage sweep (storage-sweep.ts) takes
+ * what slips through: stale test temp folders, leftover processes, extra
+ * staged runtimes and dependency installs unused for a week.
  */
 import { spawnSync } from "node:child_process";
 import { lstatSync, readdirSync } from "node:fs";
@@ -18,6 +20,16 @@ CREATE TABLE IF NOT EXISTS checkout_cleanup_setting (
   updated_at TEXT NOT NULL
 );
 `;
+/** What each storage sweep or clean-up by hand did (storage-sweep.ts): the latest few. */
+export const STORAGE_SWEEP_SCHEMA = `
+CREATE TABLE IF NOT EXISTS storage_sweep (
+  id     INTEGER PRIMARY KEY,
+  at     TEXT NOT NULL,
+  source TEXT NOT NULL CHECK (source IN ('automatic', 'manual')),
+  actor  TEXT NOT NULL,
+  result TEXT NOT NULL
+);
+`;
 export const DEFAULT_CLEANUP = "finished";
 export const CLEANUP_CHOICES = [
     { value: "finished", label: "When its task is complete or cancelled", days: 0 },
@@ -124,8 +136,8 @@ export function storageReport(store, databaseFile, checkoutNote) {
     const lines = [
         { what: "Database", bytes: take(name => name.startsWith(base)) },
         { what: "Build checkouts", bytes: take(name => name === "worktrees"), count: checkouts.length, ...(checkoutNote === undefined ? {} : { note: checkoutNote }) },
-        { what: "Shared dependencies", bytes: take(name => name === DEPS_FOLDER), count: sharedCopies(join(folder, DEPS_FOLDER)).length, note: "one install per lockfile, linked into checkouts" },
-        { what: "Staged releases", bytes: take(name => name === "staged-upgrades"), count: staged, note: "a deploy keeps the release it installed, the one before and the newest few" },
+        { what: "Shared dependencies", bytes: take(name => name === DEPS_FOLDER), count: sharedCopies(join(folder, DEPS_FOLDER)).length, note: "one install per lockfile, linked into checkouts; one unused for a week goes" },
+        { what: "Staged releases", bytes: take(name => name === "staged-upgrades"), count: staged, note: "kept: the one running, the one before it and the newest" },
         { what: "Evidence", bytes: take(name => name === "evidence"), note: "the audit record of every run; kept" },
         { what: "Backups", bytes: take(name => name === "backups") },
     ];

package/dist/store.d.ts

@@ -21,6 +21,14 @@ import { type RetentionKind, type RetentionPeriods } from "./retention.js";
 import { type CheckoutCleanup } from "./storage.js";
 import { type LedgerChainReport } from "./ledger-chain.js";
 import { type ApprovalGate, type ApprovalRules, type ApproverKind, type Filer } from "./approval-policy.js";
+/** A lead reply stopped at its deadline, in the thread as plain words: what happened, what it means, one next step. The
+ * proposals it made before then are kept; the cost line only when an unknown cost paused chat. */
+export declare function mateTimeoutNotice(keptProposals: boolean, unknownCost?: boolean): string;
+/** Whether a turn's proposals stand: it answered, or it stopped at its deadline after its completed tool calls proposed them. */
+export declare function mateTurnKeepsProposals(turn: {
+    state: string;
+    failureReason: string | null;
+} | null): boolean;
 /** What the lead promised to follow up on (lead-commitments.ts; no version bump: additive only, so a build that
  * predates it still opens the store). `condition_json` is a task, attempt, check or time; `check_at` is when to look
  * next; `channel` is where it reports (`chat`: the conversation it was promised in); a promise lapses at `expires_at`. */
@@ -6916,12 +6924,17 @@ export declare class Store {
         tokensIn: number;
         tokensOut: number;
         failureReason?: string;
-        /** Answered turns: the assistant row, written in the same transaction as the promotion. */
+        /** Answered turns: the assistant row, written in the same transaction as the promotion. A timed-out turn's notice. */
         message?: {
             text: string;
             activity: string;
         };
+        /** A timed-out turn keeps what its completed, validated tool calls proposed (its drafts go pending with its notice). */
+        keepProposals?: boolean;
     }, now: Date): boolean;
+    /** A turn stopped at its deadline: the proposals its completed tool calls drafted (each validated when drafted) go pending, as an
+     * answered turn's do; anything the provider sent after them was never acted on. Every other failure still deletes its drafts. */
+    private keepTimedOutMateProposals;
     /** Revocation's companion for a loop in flight (finding 5): every live
      * turn of the approver fails NOW, charged its whole reservation, its
      * generation moved so the returning loop's terminal CAS loses. */
@@ -6929,7 +6942,9 @@ export declare class Store {
     /** The crash sweep for turns: past the deadline, failed. The session is
      * charged the settled sum of the steps that answered — unless any step
      * is unproven (dispatched and never settled, or latched), in which case
-     * the WHOLE reservation is charged (finding 1). Drafts are deleted. */
+     * the WHOLE reservation is charged (finding 1). Like a turn that stopped
+     * at its own deadline, it keeps the proposals its completed tool calls
+     * drafted and says, in the thread, that the reply took too long. */
     sweepStaleMateTurns(now: Date): number;
     /** A task_ref by its row id: the external id and repo — for decisions, which hang off runs. */
     refById(id: number): {

package/dist/store.js

@@ -71,7 +71,7 @@ import { PROVIDER_AUTH_SCHEMA } from "./provider-auth.js";
 import { REVIEW_SCHEMA, reviewSwitchWords } from "./review-switch.js";
 import { SPEND_SCHEMA, billingOf, budgetStates, canPrice, claudeMachineBilling, countsToward, filersOf, monthOf, priceWork, seenBilling as seenBillingOf, spendItems, teammateFilers, usd } from "./spend.js";
 import { DEFAULT_PERIODS, RETENTION_SCHEMA, periodWords, widenRetentionSchema } from "./retention.js";
-import { CHECKOUT_CLEANUP_SCHEMA, DEFAULT_CLEANUP, cleanupWords } from "./storage.js";
+import { CHECKOUT_CLEANUP_SCHEMA, STORAGE_SWEEP_SCHEMA, DEFAULT_CLEANUP, cleanupWords } from "./storage.js";
 import { IN_RANGE, LEDGER_CHAIN_SCHEMA, safeWhole, sealLedger, verifyLedgerChain } from "./ledger-chain.js";
 import { NO_RULES, filerFor, isProtectedWork, protectedChanges, rulesWords } from "./approval-policy.js";
 import { configBase, envValue, namedPath } from "./names.js";
@@ -253,6 +253,19 @@ CREATE TABLE IF NOT EXISTS project_mute (
   PRIMARY KEY (account, repo)
 );
 `;
+/** A lead reply stopped at its deadline, in the thread as plain words: what happened, what it means, one next step. The
+ * proposals it made before then are kept; the cost line only when an unknown cost paused chat. */
+export function mateTimeoutNotice(keptProposals, unknownCost = false) {
+    return [
+        unknownCost ? "The reply took too long and was stopped, and its cost isn't known yet, so chat is paused." : "The reply took too long and was stopped.",
+        keptProposals ? "What it proposed is below." : null,
+        unknownCost ? "Confirm the cost on the Chat page to turn chat back on." : "Send your message again, or ask for less at once.",
+    ].filter(part => part !== null).join(" ");
+}
+/** Whether a turn's proposals stand: it answered, or it stopped at its deadline after its completed tool calls proposed them. */
+export function mateTurnKeepsProposals(turn) {
+    return turn !== null && (turn.state === "answered" || turn.state === "failed" && (turn.failureReason === "timeout" || turn.failureReason === "crashed"));
+}
 /** What the lead promised to follow up on (lead-commitments.ts; no version bump: additive only, so a build that
  * predates it still opens the store). `condition_json` is a task, attempt, check or time; `check_at` is when to look
  * next; `channel` is where it reports (`chat`: the conversation it was promised in); a promise lapses at `expires_at`. */
@@ -3992,6 +4005,8 @@ function initializeStore(db, file) {
     // 1-day evidence: an older file's retention_setting only allowed 7 days or more (no version bump: its rows carry over).
     widenRetentionSchema(db);
     db.exec(CHECKOUT_CLEANUP_SCHEMA);
+    // The daily storage sweep's record (no version bump: a new table, nothing carried over).
+    db.exec(STORAGE_SWEEP_SCHEMA);
     db.exec(BACKUP_SCHEMA);
     // Sprint 8: the organisation policy (one row, or none: nothing restricted).
     db.exec(POLICY_SCHEMA);
@@ -20282,12 +20297,23 @@ export class Store {
                 }
             }
             else {
-                this.discardMateProposals(id);
+                if (outcome.keepProposals === true)
+                    this.keepTimedOutMateProposals(id);
+                else
+                    this.discardMateProposals(id);
                 this.discardMateTurnEvidence(id);
+                if (outcome.message !== undefined) {
+                    this.appendMateMessage({ thread: Number(before["thread"]), turn: id, role: "assistant", text: outcome.message.text, activity: outcome.message.activity }, now);
+                }
             }
             return true;
         });
     }
+    /** A turn stopped at its deadline: the proposals its completed tool calls drafted (each validated when drafted) go pending, as an
+     * answered turn's do; anything the provider sent after them was never acted on. Every other failure still deletes its drafts. */
+    keepTimedOutMateProposals(turn) {
+        return Number(this.db.prepare("UPDATE mate_proposal SET state = 'pending' WHERE turn = ? AND state = 'drafting'").run(turn).changes);
+    }
     /** Revocation's companion for a loop in flight (finding 5): every live
      * turn of the approver fails NOW, charged its whole reservation, its
      * generation moved so the returning loop's terminal CAS loses. */
@@ -20309,12 +20335,14 @@ export class Store {
     /** The crash sweep for turns: past the deadline, failed. The session is
      * charged the settled sum of the steps that answered — unless any step
      * is unproven (dispatched and never settled, or latched), in which case
-     * the WHOLE reservation is charged (finding 1). Drafts are deleted. */
+     * the WHOLE reservation is charged (finding 1). Like a turn that stopped
+     * at its own deadline, it keeps the proposals its completed tool calls
+     * drafted and says, in the thread, that the reply took too long. */
     sweepStaleMateTurns(now) {
         return this.transact(() => {
             this.sweepStaleChatTurns(now);
             const rows = this.db
-                .prepare("SELECT id, session, reserved_microusd AS reserved FROM mate_turn WHERE state IN ('queued','running') AND deadline_at < ?")
+                .prepare("SELECT id, session, thread, reserved_microusd AS reserved FROM mate_turn WHERE state IN ('queued','running') AND deadline_at < ?")
                 .all(now.toISOString());
             for (const row of rows) {
                 const id = Number(row["id"]);
@@ -20329,8 +20357,12 @@ export class Store {
                     .prepare("UPDATE mate_turn SET state = 'failed', failure_reason = 'crashed', generation = generation + 1, settled_microusd = ?, finished_at = ? WHERE id = ?")
                     .run(spent, now.toISOString(), id);
                 this.db.prepare("UPDATE mate_session SET spent_microusd = spent_microusd + ? WHERE id = ?").run(spent, Number(row["session"]));
-                this.discardMateProposals(id);
+                const kept = this.keepTimedOutMateProposals(id);
                 this.discardMateTurnEvidence(id);
+                const latched = this.db.prepare("SELECT 1 AS hit FROM chat_turn WHERE mate_turn = ? AND unknown_spend = 1 AND acknowledged_at IS NULL LIMIT 1").get(id) !== undefined;
+                const open = this.db.prepare("SELECT 1 AS hit FROM mate_thread WHERE id = ? AND closed_at IS NULL").get(Number(row["thread"])) !== undefined;
+                if (open)
+                    this.appendMateMessage({ thread: Number(row["thread"]), turn: id, role: "assistant", text: mateTimeoutNotice(kept > 0, latched), activity: null }, now);
             }
             return rows.length;
         });

package/dist/subscription-chat.d.ts

@@ -10,6 +10,8 @@ export type SubscriptionMateRequest = {
     history: readonly MateHistoryMessage[];
     tools: readonly MateToolSchema[];
     timeoutMs: number;
+    /** The turn's own deadline: aborting it ends the harness's whole process group. */
+    signal?: AbortSignal;
     /** The reply as it is written, cumulative, when the harness can stream it
      * (Claude). Display only: the finished answer is still parsed whole. */
     onText?: (text: string) => void;
@@ -23,7 +25,7 @@ export type SubscriptionMateRunner = (request: SubscriptionMateRequest) => Promi
 }>;
 type CommandRunner = (file: string, args: readonly string[], options: Parameters<typeof run>[2]) => Promise<ExecResult>;
 /** The model-visible request. Tool results are already redacted by mateView. */
-export declare function composeSubscriptionMatePrompt(request: Omit<SubscriptionMateRequest, "provider" | "model" | "timeoutMs">): string;
+export declare function composeSubscriptionMatePrompt(request: Omit<SubscriptionMateRequest, "provider" | "model" | "timeoutMs" | "signal">): string;
 /** Strictly turn the harness's structured output into the provider-neutral answer. */
 export declare function parseSubscriptionMateAnswer(text: string, usage?: {
     tokensIn?: unknown;

package/dist/subscription-chat.js

@@ -224,6 +224,7 @@ export async function performSubscriptionMateRequest(request, runner = run) {
             cwd: dir,
             stdin: prompt,
             timeoutMs: request.timeoutMs,
+            ...(request.signal === undefined ? {} : { signal: request.signal }),
             // Events repeat the answer (deltas, the message, the result line).
             maxBuffer: streaming ? RESPONSE_CAP_BYTES * 16 : RESPONSE_CAP_BYTES,
             omitEnv: ALL_CREDENTIAL_ENV,

package/dist/surface.js

@@ -114,7 +114,7 @@ export const COMMAND_GUIDE = [
         mutation: spec.mutation ? 'keyed' : 'none', inputSchema: spec.inputSchema,
         flags: Object.entries(sessionCliFlags(spec)).map(([name, arity]) => ({ name, takesValue: arity === 'value', meaning: name === 'key' ? 'keep the same key when reconciling; inspect after unknown delivery, never blind-retry' : `see session ${spec.operation} --help` })),
     })),
-    ...KNOWLEDGE_DESCRIPTORS.map(spec => ({ invocation: `knowledge ${spec.action}`, synopsis: spec.synopsis, audience: "agent", agentMayInvoke: true, mutation: spec.mutation, flags: spec.flags, ...(spec.takesQuery ? { positionals: [{ name: "query", required: true, meaning: "search text or source file for impact" }] } : {}) })),
+    ...KNOWLEDGE_DESCRIPTORS.map(spec => ({ invocation: `knowledge ${spec.action}`, synopsis: spec.synopsis, audience: "audience" in spec ? spec.audience : "agent", agentMayInvoke: !("audience" in spec), mutation: spec.mutation, flags: spec.flags, ...(spec.takesQuery ? { positionals: [{ name: "query", required: true, meaning: "search text or source file for impact" }] } : {}) })),
     ...MEMORY_DESCRIPTORS.map(spec => ({ invocation: `memory ${spec.action}`, synopsis: spec.synopsis, audience: "agent", agentMayInvoke: true, mutation: spec.mutation, flags: spec.flags, ...(spec.takesQuery ? { positionals: [{ name: "query", required: true, meaning: "search text, a decision id, or the decision sentence" }] } : {}) })),
     // Flow writes carry an approver's credential: an agent runs them only as the person asked, after showing the preview.
     ...FLOWS_DESCRIPTORS.map(spec => ({ invocation: `flows ${spec.action}`, synopsis: spec.synopsis, audience: "agent", agentMayInvoke: true, mutation: spec.mutation, flags: spec.flags,
@@ -338,6 +338,7 @@ export const COMMAND_GUIDE = [
     operator("runner retire", "retire a worker"),
     operator("runner bind", "replace which repositories a worker may build in"),
     operator("runner capacity", "set how many tasks a worker runs at once; running work carries on, the new number applies at the next claim"),
+    operator("project checks", "show or set a project's batch checks (--batch on|off, off by default): results that finish within 10 minutes share one full check on a temporary batch commit; nothing is merged into a real branch"),
     operator("project concurrency", "set how many of a project's tasks build at once (default 2, never past the worker's capacity); running work carries on"),
     operator("coordinator mint", "mint the MCP filing credential — repo-bound, rate-limited, token shown once"),
     operator("coordinator revoke", "revoke an MCP filing credential"),

package/dist/task-status.d.ts

@@ -53,7 +53,20 @@ export type ChecksFact = {
     level?: "quick" | "full" | "off" | null;
     /** A follow-up check (Run checks) waiting or running on this commit. */
     running?: "quick" | "full" | null;
+    /** A batch check (batch-checks.ts): waiting for its batch, or how it was checked. */
+    batch?: ChecksBatch | null;
 };
+/** `waiting`: the result waits to be checked with others. Otherwise how its check ran: `together` on the
+ * temporary batch commit `tested` with `peers`; `split` (its batch failed), `conflict` or `alone` on its own commit. */
+export type ChecksBatch = {
+    state: "waiting" | "together" | "split" | "conflict" | "alone";
+    tested: string | null;
+    peers: string[];
+};
+/** "T2", "T2 and T3", "T2, T3 and 2 more": short enough for one line. The full list is on the result. */
+export declare function peerWords(peers: readonly string[], shown?: number): string;
+/** "Checked together with T2 on 1a2b3c4": every surface says a shared check the same way, naming the commit it tested. */
+export declare function checkedTogetherWords(batch: ChecksBatch): string | null;
 export type PullRequestFact = {
     state: "none" | "opening" | "open" | "merged" | "closed" | "failed";
     number: number | null;