toolroll 0.9.61 → 0.9.63

package/dist/worktree.js

@@ -23,15 +23,16 @@
  * over `.git` internals corrupts both.
  */
 import { createHash, randomBytes } from "node:crypto";
-import { existsSync, readFileSync, writeFileSync, rmSync, realpathSync, mkdirSync, readdirSync, renameSync } from "node:fs";
+import { existsSync, lstatSync, readFileSync, writeFileSync, rmSync, realpathSync, mkdirSync, readdirSync, renameSync } from "node:fs";
 import { hostname } from "node:os";
-import { dirname, join } from "node:path";
+import { basename, dirname, join, relative, resolve, sep } from "node:path";
 import { run } from "./exec.js";
 import { currentBootId, provenDeadByBootChange } from "./boot-identity.js";
 import { HANDOFF_PREFIX, MAILBOX_SUFFIX, looksLikeProtocolFile, readMailbox } from "./evidence.js";
 import { parseHandoff } from "./decision.js";
 import { diskBytes } from "./storage.js";
 import { markNeverIndex } from "./never-index.js";
+import { teardownGit } from "./teardown-git.js";
 /**
  * The note a lease leaves in the checkout, naming the process holding it.
  *
@@ -40,6 +41,13 @@ import { markNeverIndex } from "./never-index.js";
  * something is in there.
  */
 export const MARKER = ".standing-orders-lease";
+/**
+ * The reflog message of a branch Toolroll created, naming it: its creation entry is the proof, later, that the branch
+ * is Toolroll's and how old it is. A branch copied or renamed from it names the old branch, so it proves nothing.
+ */
+export const branchCreatedMessage = (branch) => `Toolroll created branch ${branch}`;
+/** Scouts' disposable branches: never landed work. */
+export const SCOUT_PREFIX = "toolroll-scout/";
 /** Checkouts this process is leasing or removing right now: a lease and storage retention never overlap on one. */
 const busy = new Map();
 // Outside the checkout so removal cannot erase the reservation before git and the row update finish.
@@ -122,6 +130,31 @@ export function worktreePath(root, repo, branch) {
         .slice(0, 8);
     return join(root, safeSegment(basenameOf(repo)), `${safeSegment(branch)}-${digest}`);
 }
+/** The pool root a checkout at `path` of `repo` was made under (worktreePath's layout), or null for one it didn't make. */
+export function poolRootOf(path, repo) {
+    const project = dirname(path);
+    return basename(project) === safeSegment(basenameOf(repo)) && /-[0-9a-f]{8}$/.test(basename(path)) ? dirname(project) : null;
+}
+/** Inspect names, not their destinations, before the plane writes into a pool. */
+function safePoolPath(root, target) {
+    const base = resolve(root);
+    const rel = relative(base, resolve(target));
+    if (rel.startsWith("..") || rel.startsWith(sep))
+        return false;
+    let path = base;
+    for (const component of ["", ...rel.split(sep)]) {
+        path = join(path, component);
+        try {
+            if (!lstatSync(path).isDirectory())
+                return false;
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                return false;
+        }
+    }
+    return true;
+}
 function normalisePath(path) {
     return path.replace(/\\/g, "/").replace(/\/+$/, "");
 }
@@ -203,20 +236,21 @@ export function worktreeProcessOccupancy(path) {
  * reset or clean a tree saves this first.
  */
 export async function saveWorkPatch(runner, path, file, header) {
-    const status = await runner(GIT, [...READ_ONLY, "status", "--porcelain", "--untracked-files=all"], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+    const git = teardownGit(runner);
+    const status = await git(path, [...READ_ONLY, "status", "--porcelain", "--untracked-files=all", "--ignore-submodules=all"]);
     if (status.code !== 0)
         return { ok: false, message: `${path} could not be inspected before its work was saved` };
     const untracked = status.stdout
         .split("\n")
         .filter(line => line.startsWith("?? ") && !line.trimEnd().endsWith(MARKER))
         .map(line => line.slice(3).trim());
-    const tracked = await runner(GIT, [...READ_ONLY, "diff", "--binary", "HEAD"], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+    const tracked = await git(path, [...READ_ONLY, "diff", "--binary", "--ignore-submodules=all", "HEAD"]);
     if (tracked.code !== 0)
         return { ok: false, message: `${path}: git diff failed while saving its work (${firstLine(tracked.stderr)})` };
     const parts = [tracked.stdout];
     for (const one of untracked) {
         // --no-index exits 1 when the sides differ; that is the expected answer.
-        const diff = await runner(GIT, [...READ_ONLY, "diff", "--binary", "--no-index", "--", "/dev/null", one], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+        const diff = await git(path, [...READ_ONLY, "diff", "--binary", "--no-index", "--ignore-submodules=all", "--", "/dev/null", one]);
         if (diff.code > 1)
             return { ok: false, message: `${path}: ${one} could not be captured (${firstLine(diff.stderr)})` };
         parts.push(diff.stdout);
@@ -297,6 +331,8 @@ export class WorktreePool {
     }
     async lease(request) {
         const target = worktreePath(this.options.root, request.repo, request.branch);
+        if (!safePoolPath(this.options.root, target))
+            return { ok: false, reason: "git", message: "The checkout pool contains a symlink or an unreadable folder." };
         if (busy.has(target))
             return { ok: false, reason: "in-use", message: `${target} is busy; try again in a moment` };
         const release = reserveCheckout(this.store, target);
@@ -324,6 +360,9 @@ export class WorktreePool {
             };
         }
         const path = worktreePath(this.options.root, request.repo, request.branch);
+        const unsafe = () => ({ ok: false, reason: "git", message: "The checkout pool contains a symlink or an unreadable folder." });
+        if (!safePoolPath(this.options.root, path))
+            return unsafe();
         const existing = this.store.getWorktree(path);
         // The machine outranks the database here. A row can say "released" while a
         // process that never got to write its own ending is still working in the
@@ -401,23 +440,46 @@ export class WorktreePool {
             const fromBase = request.base !== undefined && !branchExists;
             // Spotlight leaves the checkouts alone: the marker sits in the pool and the project's folder, never inside a
             // checkout, where a build's `git add -A` would commit it (never-index.ts).
+            if (!safePoolPath(this.options.root, path))
+                return unsafe();
             try {
                 mkdirSync(dirname(path), { recursive: true });
             }
-            catch { /* git reports it */ }
+            catch {
+                return unsafe();
+            }
             markNeverIndex(this.options.root);
             markNeverIndex(dirname(path));
-            const add = await this.git(request.repo, [
-                "worktree",
-                "add",
-                ...(fromBase ? ["-b", request.branch] : []),
-                path,
-                ...(fromBase ? [request.base] : [request.branch]),
-            ]);
+            // A new branch is created first, on its own, so its reflog's first entry says Toolroll made it and when.
+            let made = null;
+            if (fromBase) {
+                const base = await this.git(request.repo, ["rev-parse", "--verify", "--quiet", `${request.base}^{commit}`], READ_ONLY);
+                const oid = base.stdout.trim();
+                if (base.code !== 0 || !/^[0-9a-f]{40,64}$/.test(oid))
+                    return { ok: false, reason: "git", message: `git couldn't find ${request.base}` };
+                const branch = await this.git(request.repo, ["update-ref", "--create-reflog", "-m", branchCreatedMessage(request.branch), `refs/heads/${request.branch}`, oid, "0".repeat(oid.length)]);
+                if (branch.code !== 0)
+                    return { ok: false, reason: "git", message: firstLine(branch.stderr) };
+                made = oid;
+            }
+            if (!safePoolPath(this.options.root, path))
+                return unsafe();
+            const add = await this.git(request.repo, ["worktree", "add", path, request.branch]);
             if (add.code !== 0) {
+                if (made !== null)
+                    await this.teardown(request.repo, ["update-ref", "-d", `refs/heads/${request.branch}`, made]);
                 return { ok: false, reason: "git", message: firstLine(add.stderr) };
             }
         }
+        // Fresh base, inside the same reservation as everything above: nothing else leases, prunes or slims this
+        // checkout between the inspection and the move.
+        let base;
+        if (request.freshBase !== undefined && request.taskRef !== undefined) {
+            const fresh = await this.freshen(path, request.branch, request.taskRef, request.freshBase, request.now);
+            if (!fresh.ok)
+                return { ok: false, reason: "git", message: fresh.message };
+            base = fresh.standing;
+        }
         const row = {
             path,
             repo: request.repo,
@@ -453,8 +515,50 @@ export class WorktreePool {
             ...(reclaimed === undefined ? {} : { reclaimed }),
             ...(resumedFromRun === undefined ? {} : { resumedFromRun }),
             ...(recoveryKind === undefined ? {} : { recoveryKind }),
+            ...(base === undefined ? {} : { base }),
         };
     }
+    /**
+     * Move an untouched checkout to current main, or say how far behind it is. Untouched means all three: on its own
+     * branch, HEAD still the recorded base (or, with none recorded, a commit main already has), and nothing changed or
+     * untracked. The move is `git reset --keep`, which refuses rather than overwrite anything; HEAD and the tree are read
+     * again right before and after it, and any surprise fails the lease with the checkout left for a person.
+     */
+    async freshen(path, branch, taskRef, fresh, now) {
+        const read = async (args) => {
+            const out = await this.runner(GIT, [...READ_ONLY, ...args], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+            return out.code === 0 ? out.stdout.trim() : null;
+        };
+        const main = await read(["rev-parse", "--verify", "--quiet", `${fresh.main}^{commit}`]);
+        const head = await read(["rev-parse", "--verify", "--quiet", "HEAD^{commit}"]);
+        if (main === null || head === null)
+            return { ok: true, standing: null };
+        const count = await read(["rev-list", "--count", `${head}..${main}`]);
+        const behind = count === null ? NaN : Number(count);
+        if (!Number.isSafeInteger(behind))
+            return { ok: true, standing: null };
+        const stay = (kept) => ({ ok: true, standing: { head, main, behind, moved: null, kept } });
+        if (behind === 0)
+            return stay(null);
+        if (await read(["symbolic-ref", "-q", "HEAD"]) !== `refs/heads/${branch}`)
+            return stay("not on its branch");
+        if (fresh.recordedBase !== null && fresh.recordedBase !== head)
+            return stay("has commits");
+        const contained = await this.runner(GIT, [...READ_ONLY, "merge-base", "--is-ancestor", head, main], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+        if (contained.code !== 0)
+            return stay("has commits");
+        if (await this.isDirty(path) !== false)
+            return stay("has changes");
+        // Recorded before git moves it: a move that then fails leaves the next attempt's own base as the branch's origin.
+        this.store.recordBranchBaseMove(taskRef, branch, head, main, now);
+        const moved = await this.runner(GIT, ["reset", "--keep", main], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+        const after = await read(["rev-parse", "--verify", "--quiet", "HEAD^{commit}"]);
+        const branchNow = await read(["rev-parse", "--verify", "--quiet", `refs/heads/${branch}^{commit}`]);
+        if (moved.code !== 0 || after !== main || branchNow !== main || await this.isDirty(path) !== false) {
+            return { ok: false, message: `${path} could not be moved to current main (${firstLine(moved.stderr) || "it changed during the move"}) — leaving it for a person` };
+        }
+        return { ok: true, standing: { head: main, main, behind: 0, moved: { from: head, behind }, kept: null } };
+    }
     /**
      * A crash can strand either a finished handoff or a partial source draft.
      * The expired lease can no longer authorize a commit, so this never ingests
@@ -526,10 +630,10 @@ export class WorktreePool {
     }
     /** Back to HEAD, untracked gone, our lease note kept; proven clean after. */
     async resetTree(path) {
-        const checkout = await this.runner(GIT, ["checkout", "--", "."], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+        const checkout = await this.teardown(path, ["checkout", "--", "."]);
         if (checkout.code !== 0)
             return { ok: false, message: `${path}: git checkout failed while resetting (${firstLine(checkout.stderr)})` };
-        const clean = await this.runner(GIT, ["clean", "-fd", "-e", MARKER], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+        const clean = await this.teardown(path, ["clean", "-fd", "-e", MARKER]);
         if (clean.code !== 0)
             return { ok: false, message: `${path}: git clean failed while resetting (${firstLine(clean.stderr)})` };
         const dirty = await this.isDirty(path);
@@ -617,17 +721,17 @@ export class WorktreePool {
         const occupied = this.inUse(path);
         if (occupied.held)
             return { ok: false, message: `${path} is being used by process ${occupied.by} — leaving it alone` };
-        const removed = await this.git(existing.repo, ["worktree", "remove", "--force", path]);
+        const removed = await this.teardown(existing.repo, ["worktree", "remove", "--force", path]);
         if (removed.code !== 0)
             return { ok: false, message: `git worktree remove: ${removed.stderr.trim() || `exit ${removed.code}`}` };
         // The branch is disposable too; a failure here is bookkeeping, not custody.
-        await this.git(existing.repo, ["branch", "-D", existing.branch]);
+        await this.teardown(existing.repo, ["branch", "-D", existing.branch]);
         this.store.forgetWorktree(path);
         void now;
         return { ok: true };
     }
     async orphans(repo) {
-        const listed = await this.git(repo, ["worktree", "list", "--porcelain"], READ_ONLY);
+        const listed = await this.teardown(repo, ["worktree", "list", "--porcelain"]);
         // A listing that failed is not a listing that came back empty. Treating
         // it as empty would report every stored row as missing — and anything
         // acting on that answer would erase real bookkeeping over a git hiccup.
@@ -724,14 +828,14 @@ export class WorktreePool {
     /**
      * Whether a let-go checkout could go without losing anything: clean (build output that .gitignore hides is not
      * work, and neither are the files Toolroll itself wrote in there), and everything its HEAD reaches on a branch, tag
-     * or remote. Reads only.
+     * or remote. Reads only, through `git` when given (teardown's hardened git).
      */
-    async inspect(path) {
-        const dirty = await this.isDirty(path, true);
+    async inspect(path, git = this.teardown) {
+        const dirty = await this.isDirty(path, true, git);
         if (dirty !== false)
             return dirty === null ? "unreadable" : "has changes";
         // Commits only this checkout's HEAD reaches (a detached HEAD moved on) exist nowhere else.
-        const own = await this.runner(GIT, [...READ_ONLY, "rev-list", "-1", "HEAD", "--not", "--branches", "--tags", "--remotes"], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+        const own = await git(path, [...READ_ONLY, "rev-list", "-1", "HEAD", "--not", "--branches", "--tags", "--remotes"]);
         if (own.code !== 0)
             return "unreadable";
         return own.stdout.trim() === "" ? "clean" : "has commits";
@@ -739,10 +843,12 @@ export class WorktreePool {
     /**
      * Keep storage in check: remove the working copies nobody needs any more. Of `rows`, a checkout goes when it is
      * let go, nothing holds it, `wanted` says it may go (asked again right before it goes, so a task coming back or a
-     * lease since keeps it) and `inspect` finds it clean. Only the working copy goes: its branch, and so every commit,
-     * stays, and a later lease of the same branch makes a new one. At most `max` go per pass.
+     * lease since keeps it) and `inspect` finds it clean, then clean still right before it goes (git refuses an edit
+     * made after that). Only the working copy goes: its branch, and so every commit,
+     * stays, and a later lease of the same branch makes a new one. At most `max` go per pass. Every git command goes
+     * through `git` when given: landed cleanup passes its hardened teardown git (teardown-git.ts).
      */
-    async prune(rows, wanted, max = Infinity) {
+    async prune(rows, wanted, max = Infinity, git = this.teardown) {
         const removed = [];
         const kept = [];
         const idle = (row, releasedAt) => row !== null && row.runner === null && row.releasedAt === releasedAt && !this.inUse(row.path).held;
@@ -756,7 +862,7 @@ export class WorktreePool {
                 continue;
             if (!existsSync(row.path) || !idle(row, row.releasedAt))
                 continue;
-            const found = await this.inspect(row.path);
+            const found = await this.inspect(row.path, git);
             if (found !== "clean") {
                 kept.push({ path: row.path, why: found });
                 continue;
@@ -776,9 +882,9 @@ export class WorktreePool {
                 const allowed = wanted(now, true);
                 if (!(typeof allowed === "boolean" ? allowed : await allowed))
                     continue;
-                const gone = await this.removeWorkingCopy(row);
-                if (!gone) {
-                    kept.push({ path: row.path, why: "git refused" });
+                const gone = await this.removeCleanCopy(row, git);
+                if (gone !== "removed") {
+                    kept.push({ path: row.path, why: gone });
                     continue;
                 }
                 removed.push(row);
@@ -800,7 +906,7 @@ export class WorktreePool {
         const idle = (now) => now !== null && now.runner === null && now.releasedAt !== null && now.releasedAt === row.releasedAt && !this.inUse(row.path).held;
         if (busy.has(row.path) || !existsSync(row.path) || !idle(this.store.getWorktree(row.path)))
             return { ok: false, message: `${row.path} is in use` };
-        const listed = await this.runner(GIT, [...READ_ONLY, "ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z"], { cwd: row.path, timeoutMs: WORKTREE_TIMEOUT_MS });
+        const listed = await this.teardown(row.path, [...READ_ONLY, "ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z"]);
         if (listed.code !== 0)
             return { ok: false, message: `git couldn't list ${row.path}` };
         const drop = listed.stdout.split("\0").filter(one => one.endsWith("/")).map(one => one.slice(0, -1))
@@ -871,7 +977,8 @@ export class WorktreePool {
             busy.delete(path);
         }
     }
-    /** Remove one idle checkout's working copy (the caller holds it in `busy`) and forget its row. */
+    /** Throw away one idle checkout's working copy, changes and all (the caller holds it in `busy`), and forget its row.
+     * Only `discardChanges`, where losing them is what a person asked for. */
     async removeWorkingCopy(row) {
         // Held while it goes: another process's lease sees this process in it (the busy map is this process's only).
         const note = join(row.path, MARKER);
@@ -881,17 +988,44 @@ export class WorktreePool {
         catch {
             return false;
         }
-        const gone = await this.git(row.repo, ["worktree", "remove", "--force", row.path]);
+        const gone = await this.teardown(row.repo, ["worktree", "remove", "--force", row.path]);
         if (gone.code !== 0) {
             rmSync(note, { force: true });
             return false;
         }
+        this.forgetRemoved(row);
+        return true;
+    }
+    /**
+     * Remove one idle checkout's working copy only while it is still clean (the caller holds it in `busy`, and its
+     * reservation outside the checkout holds it for other processes): inspected once more right before it goes, then
+     * removed without --force, so git itself refuses an edit made since. Only Toolroll's own files go first, because
+     * git's check would count them.
+     */
+    async removeCleanCopy(row, git) {
+        const found = await this.inspect(row.path, git);
+        if (found !== "clean")
+            return found;
+        const status = await git(row.path, [...READ_ONLY, "status", "--porcelain", "--untracked-files=all", "--ignore-submodules=all"]);
+        if (status.code !== 0)
+            return "unreadable";
+        for (const line of status.stdout.split("\n")) {
+            const name = line.startsWith("?? ") ? line.slice(3).trim() : "";
+            if (name === MARKER || (name !== "" && looksLikeProtocolFile(name)))
+                rmSync(join(row.path, name), { recursive: true, force: true });
+        }
+        const gone = await git(row.repo, ["worktree", "remove", row.path]);
+        if (gone.code !== 0)
+            return (await this.isDirty(row.path, false, git)) === false ? "git refused" : "has changes";
+        this.forgetRemoved(row);
+        return "removed";
+    }
+    forgetRemoved(row) {
         this.store.transact(() => {
             const still = this.store.getWorktree(row.path);
             if (still !== null && still.runner === null && still.releasedAt === row.releasedAt)
                 this.store.forgetWorktree(row.path);
         });
-        return true;
     }
     /**
      * Whether anybody's work is in there. null means we could not tell, which is
@@ -900,8 +1034,8 @@ export class WorktreePool {
      * `--untracked-files=all` and no `--ignored`: build output that .gitignore
      * hides is not work, but a file somebody dropped in and never staged is.
      */
-    async isDirty(path, ignoreOwnFiles = false) {
-        const status = await this.runner(GIT, [...READ_ONLY, "status", "--porcelain", "--untracked-files=all"], { cwd: path, timeoutMs: WORKTREE_TIMEOUT_MS });
+    async isDirty(path, ignoreOwnFiles = false, git = this.teardown) {
+        const status = await git(path, [...READ_ONLY, "status", "--porcelain", "--untracked-files=all", "--ignore-submodules=all"]);
         if (status.code !== 0)
             return null;
         // Our own lease note is not the operator's work. Without this the marker
@@ -915,6 +1049,8 @@ export class WorktreePool {
             .filter(line => line.trim() !== "")
             .some(line => !line.trimEnd().endsWith(MARKER) && !(ignoreOwnFiles && line.startsWith("?? ") && looksLikeProtocolFile(line.slice(3).trim())));
     }
+    /** All disposal and inspection paths share the hardened teardown boundary. */
+    teardown = (cwd, args) => teardownGit(this.runner)(cwd, args);
     git(cwd, args, prefix = []) {
         return this.runner(GIT, [...prefix, ...args], { cwd, timeoutMs: WORKTREE_TIMEOUT_MS });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "toolroll",
-  "version": "0.9.61",
+  "version": "0.9.63",
   "description": "A control plane for unattended coding agents — queue tasks, walk away, come back to pull requests. Agents build in leased worktrees, never touch your default branch, and interrupt you only for decisions that need a human.",
   "directories": {
     "doc": "docs"
@@ -8,6 +8,8 @@
   "scripts": {
     "build": "tsc -p tsconfig.build.json && node scripts/browser-build.mjs && node scripts/postbuild.mjs",
     "test": "vitest run",
+    "test:attacks": "vitest run --config test/attacks/vitest.config.ts",
+    "check:quick": "node scripts/quick-check.mjs",
     "dev": "tsx src/cli.ts",
     "typecheck": "tsc --noEmit && tsc -p tsconfig.browser.json",
     "certify:provider": "npm run build && node scripts/provider-canary.mjs",