toolroll 0.9.92 → 0.9.93

package/dist/stop-proof.js

@@ -0,0 +1,293 @@
+/**
+ * Toolroll's own proof that a finished build stopped. Nobody is ever asked to "confirm it stopped": a person can't see
+ * whether a process exited, and asking defeats unattended builds and releases.
+ *
+ * When a finished run's process record is missing or incomplete (a spawn whose process ID was never written, a lost
+ * process-tree snapshot, an agent start with no witness, a workspace note that can't be read), Toolroll looks at this
+ * computer itself, a grace period after the run finished:
+ *
+ *   - no process the run is known to have started is still that process (pid and birth time, never a pid alone);
+ *   - no process works in the run's folder (its working directory) or names it in its command line;
+ *   - this process still tracks no live child for the run.
+ *
+ * A leftover it finds is stopped (SIGTERM, then SIGKILL) only when it is proven the run's: in the run's folder, born
+ * while the run ran, not this service or its ancestors, and orphaned or descended from the run's
+ * own processes — or it descends from such a leftover. A pid now held by a process born after the run ended is never
+ * the run's. Anything else in the folder (a person's shell or editor) is left alone and holds the proof, in plain
+ * words. When this computer can't look (the process table or the folder listing can't be read, or the run's
+ * processes belong to another computer), the stop stays unproven and the reason is shown; there is still nothing to
+ * vouch for.
+ *
+ * Command lines are read only to match the folder's path in memory; they are never stored or shown.
+ */
+import { execFileSync } from "node:child_process";
+import { existsSync, readdirSync, readlinkSync, realpathSync } from "node:fs";
+import { hostname } from "node:os";
+import { sep } from "node:path";
+import { ownedProcessCount, runOwnerTag } from "./exec.js";
+import { closeoutTargets, processCensus, protectedProcesses, stopProcesses, stoppedWords } from "./run-closeout.js";
+import { recordStopProofBlocked, settleRunStopByObservation } from "./store.js";
+import { worktreeProcessOccupancy } from "./worktree.js";
+/** How long after a run finished its processes have to go on their own before Toolroll looks. */
+export const STOP_PROOF_GRACE_MS = 60_000;
+/** `ps` gives ages in whole seconds: a process born this close to the run's start or end still counts as during it. */
+const BIRTH_SLACK_MS = 2_000;
+/** One pass's shared look at this computer: the process table, folders and command lines are read once for every
+ * run the pass checks, and again after anything was stopped. */
+export function sharedLook(options = {}) {
+    let census = null, folders = null, lines = null;
+    return {
+        ...options,
+        census: () => (census ??= (options.census ?? (() => processCensus()))()),
+        cwds: paths => options.cwds !== undefined ? options.cwds(paths) : processesInStrict(paths, folders ??= processFolders()),
+        commandLines: () => (lines ??= (options.commandLines ?? processCommandLines)()),
+        refresh: () => { census = null; folders = null; lines = null; options.refresh?.(); },
+    };
+}
+/** Which processes work inside each path, from the OS: /proc on Linux, lsof on macOS. Unlike a best-effort listing,
+ * it throws when the listing can't be read: no answer is never "nobody". */
+export function processesInStrict(paths, listing = processFolders()) {
+    const found = new Map();
+    const roots = paths.map(path => { let real = path; try {
+        real = realpathSync(path);
+    }
+    catch { /* gone */ } return { path, real }; });
+    for (const { pid, cwd } of listing) {
+        for (const one of roots)
+            if ([one.path, one.real].some(root => cwd === root || cwd.startsWith(root + sep)))
+                found.set(one.path, [...(found.get(one.path) ?? []), pid]);
+    }
+    return found;
+}
+/** Every process's working directory, from the OS; throws when the listing can't be read. */
+export function processFolders() {
+    const found = [];
+    const note = (pid, cwd) => { found.push({ pid, cwd }); };
+    if (process.platform === "linux" && existsSync("/proc")) {
+        for (const name of readdirSync("/proc")) {
+            if (!/^\d+$/.test(name))
+                continue;
+            // Gone, or another user's (never a process this user's run started).
+            try {
+                note(Number(name), readlinkSync(`/proc/${name}/cwd`));
+            }
+            catch { /* not readable */ }
+        }
+        if (found.length === 0)
+            throw new Error("no process folder could be read");
+        return found;
+    }
+    if (process.platform !== "darwin")
+        throw new Error("this computer's process folders can't be listed");
+    let text;
+    try {
+        text = execFileSync("lsof", ["-nP", "-d", "cwd", "-Fpn"], { encoding: "utf8", timeout: 20_000, maxBuffer: 64 * 1024 * 1024, stdio: ["ignore", "pipe", "ignore"] });
+    }
+    catch (error) {
+        // lsof exits 1 when some process can't be inspected (another user's) yet still lists every one it can.
+        // A timeout, a signal or an overflow leaves a partial listing, which is no answer at all.
+        const failure = error;
+        if (failure.status !== 1 || failure.signal != null || typeof failure.code === "string")
+            throw new Error("the process folder listing could not be read");
+        text = String(failure.stdout ?? "");
+    }
+    if (!/^p\d+$/m.test(text))
+        throw new Error("the process folder listing was empty");
+    let pid = 0;
+    for (const line of text.split("\n")) {
+        if (line.startsWith("p"))
+            pid = Number(line.slice(1));
+        else if (line.startsWith("n") && pid > 0)
+            note(pid, line.slice(1));
+    }
+    return found;
+}
+/** Every process's command line, by pid. Read to match a folder's path in memory only; never stored or shown. */
+export function processCommandLines() {
+    if (process.platform === "win32")
+        throw new Error("command lines can't be read here");
+    const text = execFileSync("/bin/ps", ["-axww", "-o", "pid=,args="], { encoding: "utf8", timeout: 10_000, maxBuffer: 64 * 1024 * 1024, stdio: ["ignore", "pipe", "ignore"] });
+    const lines = new Map();
+    for (const line of text.split("\n")) {
+        const match = /^\s*(\d+)\s+(.*)$/.exec(line);
+        if (match !== null)
+            lines.set(Number(match[1]), match[2]);
+    }
+    if (lines.size === 0)
+        throw new Error("the process list was empty");
+    return lines;
+}
+/** Whether a command line names `path` or something inside it (a whole path, never a longer folder's prefix). */
+export function namesPath(line, path) {
+    for (let at = line.indexOf(path); at >= 0; at = line.indexOf(path, at + 1)) {
+        const before = at === 0 ? " " : line[at - 1], after = line[at + path.length];
+        if (/[\s"'=:]/.test(before) && (after === undefined || after === sep || /[\s"':]/.test(after)))
+            return true;
+    }
+    return false;
+}
+/**
+ * One look at this computer for one finished run: which processes are its leftovers (`targets`) and which are in its
+ * folder without being proven its (`blockers`). Throws when something can't be read.
+ */
+function look(store, runId, options) {
+    const run = store.getRun(runId);
+    const census = (options.census ?? (() => processCensus()))();
+    if (census.length === 0)
+        throw new Error("the process table was empty");
+    const guard = protectedProcesses(census, options.self);
+    const byPid = new Map(census.map(row => [row.pid, row]));
+    // Its recorded processes still open: still the same process (pid and birth), or a member of its open group.
+    const open = store.handle.prepare(`SELECT pid, process_group, observed_at FROM run_process
+    WHERE run = ? AND exited_at IS NULL AND pid IS NOT NULL AND containment IS NULL AND container IS NULL`).all(runId)
+        .map(row => ({ pid: Number(row["pid"]), group: Number(row["process_group"]) === 1, observedAt: Date.parse(String(row["observed_at"])) }))
+        .filter(one => Number.isFinite(one.observedAt));
+    const recorded = new Set(closeoutTargets(census, open, guard).map(row => row.pid));
+    // Its folder: working directory inside it, or named on the command line.
+    const folder = new Set();
+    if (run.worktree !== null) {
+        const paths = [run.worktree];
+        try {
+            const real = realpathSync(run.worktree);
+            if (real !== run.worktree)
+                paths.push(real);
+        }
+        catch { /* gone: only its name can match */ }
+        for (const pids of (options.cwds ?? processesInStrict)(paths).values())
+            for (const pid of pids)
+                folder.add(pid);
+        for (const [pid, line] of (options.commandLines ?? processCommandLines)())
+            if (paths.some(path => namesPath(line, path)))
+                folder.add(pid);
+    }
+    const started = Date.parse(run.startedAt) - BIRTH_SLACK_MS, ended = Date.parse(run.finishedAt) + BIRTH_SLACK_MS;
+    const during = (row) => row.bornAt >= started && row.bornAt <= ended;
+    const unguarded = (row) => !guard.pids.has(row.pid) && !guard.groups.has(row.pgid);
+    // A leftover in the folder: born while the run ran and orphaned (its starter gone). A terminal proves nothing either
+    // way here: a worker started from a terminal hands its controlling terminal to everything its runs start.
+    const orphans = new Set();
+    for (const pid of folder) {
+        const row = byPid.get(pid);
+        if (row !== undefined && during(row) && unguarded(row) && row.ppid === 1)
+            orphans.add(pid);
+    }
+    // And whatever descends from a leftover, wherever it works: from the run's own recorded processes (proven by pid
+    // and birth) or from an orphan found in its folder.
+    const descendsFrom = (roots, row) => {
+        for (let at = row, hops = 0; at !== undefined && at.pid > 1 && hops < 64; at = byPid.get(at.ppid), hops++)
+            if (roots.has(at.pid))
+                return true;
+        return false;
+    };
+    const targets = census.filter(row => unguarded(row) && (descendsFrom(recorded, row) || descendsFrom(orphans, row)));
+    const chosen = new Set(targets.map(row => row.pid));
+    // In the folder while the run ran but not proven its (a person's shell or editor: a live parent Toolroll didn't start):
+    // left alone, and the proof waits for it. Born before the run or after it ended (and not a leftover's child) can't
+    // be the run's, so it neither blocks nor is touched.
+    const blockers = [...folder].map(pid => byPid.get(pid)).filter((row) => row !== undefined && !chosen.has(row.pid) && during(row) && !guard.pids.has(row.pid));
+    // A run with no folder (a reviewer's): nothing names its processes but their groups. An orphan born while it ran
+    // in one of its recorded process groups may be its own, or a newer group that took the number: never signalled,
+    // it holds the proof.
+    if (run.worktree === null) {
+        const groups = new Set(store.handle.prepare("SELECT pid FROM run_process WHERE run = ? AND pid IS NOT NULL AND process_group = 1").all(runId).map(row => Number(row["pid"])));
+        for (const row of census)
+            if (!chosen.has(row.pid) && unguarded(row) && row.ppid === 1 && during(row) && groups.has(row.pgid))
+                blockers.push(row);
+    }
+    return { census, targets, blockers, inFolder: folder.size };
+}
+/**
+ * Prove one finished run stopped, on this computer, without asking anyone (see the file's comment). Walks the run's
+ * owned runs one unproven stop at a time. Never signals a process that isn't proven the run's.
+ */
+export async function proveRunStopped(store, runId, clock, options = {}) {
+    let stopped = 0;
+    for (let pass = 0; pass < 8; pass++) {
+        const fact = store.stopQuiescenceFact(runId);
+        if (fact === null)
+            return { run: runId, outcome: pass === 0 ? "skipped" : "settled", words: "stopped", ...(stopped > 0 ? { stopped } : {}) };
+        // Already proven by the record (an unrecorded spawn whose run's every process is proven gone): settle it now.
+        if (fact.kind === "settling" && store.settleUnspawnedWitnesses(clock(), fact.run) > 0) {
+            store.settleQuiescentStops(clock());
+            continue;
+        }
+        const cause = fact.cause;
+        if (cause === undefined || (fact.kind !== "unprovable" && fact.kind !== "unknown"))
+            return { run: fact.run, outcome: pass === 0 ? "skipped" : "waiting", words: fact.problem };
+        const subject = fact.run, run = store.getRun(subject);
+        const finished = Date.parse(run.finishedAt ?? "");
+        const now = clock();
+        if (!Number.isFinite(finished) || now.getTime() < finished + (options.afterFinishMs ?? STOP_PROOF_GRACE_MS))
+            return { run: subject, outcome: "waiting", words: "finishing" };
+        if (ownedProcessCount(runOwnerTag(store, subject)) > 0)
+            return { run: subject, outcome: "waiting", words: "a process of it is still tracked here" };
+        if (process.platform === "win32")
+            return block(store, subject, "can't be checked on this computer: Windows processes can't be listed by folder", "no folder listing on Windows", now);
+        const foreign = store.handle.prepare("SELECT host FROM run_process WHERE run = ? AND exited_at IS NULL AND host <> ? LIMIT 1").get(subject, hostname());
+        if (foreign !== undefined)
+            return block(store, subject, `ran on another computer (${String(foreign["host"])}); only that computer can check it stopped`, `an open witness belongs to ${String(foreign["host"])}`, now);
+        if (run.worktree !== null) {
+            try {
+                const held = (options.occupancy ?? worktreeProcessOccupancy)(run.worktree);
+                if (held.held)
+                    return { run: subject, outcome: "waiting", words: `its folder is still held by process ${held.by}` };
+            }
+            catch { /* an unreadable note: the look below decides */ }
+        }
+        let seen;
+        try {
+            seen = look(store, subject, options);
+        }
+        catch (error) {
+            return block(store, subject, "can't be checked on this computer: its process list could not be read", `look failed: ${error instanceof Error ? error.message : String(error)}`, now);
+        }
+        if (seen.targets.length > 0) {
+            const done = await stopProcesses(seen.targets, options);
+            options.refresh?.();
+            stopped += done.stopped.length;
+            const ref = store.refById(run.taskRef);
+            store.recordAction({ at: clock().toISOString(), actor: "system", repo: ref?.repo ?? null, taskId: ref?.externalId ?? null, runId: subject,
+                action: "run processes stopped", outcome: "stopped", source: "work", detail: `run #${subject} ended (${run.outcome ?? "?"}); its stop check found and stopped ${stoppedWords(done, options.graceMs)}` });
+            try {
+                seen = look(store, subject, options);
+            }
+            catch {
+                return { run: subject, outcome: "waiting", words: "stopped what it left running; checking again", stopped };
+            }
+            if (seen.targets.length > 0)
+                return { run: subject, outcome: "waiting", words: "stopped what it left running; checking again", stopped };
+        }
+        if (seen.blockers.length > 0) {
+            const one = seen.blockers[0];
+            const others = seen.blockers.length - 1;
+            return block(store, subject, `can't be proven stopped: ${one.name} (process ${one.pid})${others > 0 ? ` and ${others} more` : ""} is ${run.worktree === null ? "in one of its process groups" : "using its folder"}, and Toolroll didn't start it, so it left it alone`, `in its folder: ${seen.blockers.slice(0, 8).map(row => `${row.name} ${row.pid}`).join(", ")}`, now);
+        }
+        const evidence = `${hostname()}: ${seen.census.length} processes checked ${Math.round((now.getTime() - finished) / 1000)} s after it finished; none in ${run.worktree ?? "no folder"} or naming it (${seen.inFolder} seen there, none from the run), none of its recorded processes running${stopped > 0 ? `; stopped ${stopped} it left running` : ""}`;
+        if (!settleRunStopByObservation(store, { runId: subject, cause, evidence }, clock()))
+            return { run: subject, outcome: "waiting", words: "its record changed while checking" };
+    }
+    return { run: runId, outcome: "waiting", words: "more to check" };
+}
+function block(store, runId, words, detail, now) {
+    recordStopProofBlocked(store, runId, words, detail, now);
+    return { run: runId, outcome: "blocked", words };
+}
+/** The worker's pass (operate.ts reconcile): every finished run of this computer whose stop is unproven where
+ * Toolroll's own look can prove it. A batch a pass, oldest first; a blocked run never holds newer ones back. */
+export async function proveFinishedRunsStopped(store, clock, options = {}) {
+    const grace = options.afterFinishMs ?? STOP_PROOF_GRACE_MS;
+    const before = new Date(clock().getTime() - grace).toISOString();
+    const runs = store.handle.prepare(`SELECT id FROM run r WHERE r.outcome IS NOT NULL AND r.finished_at IS NOT NULL AND r.finished_at <= ? AND (
+      EXISTS (SELECT 1 FROM run_process p WHERE p.run = r.id AND p.exited_at IS NULL AND p.pid IS NULL AND p.containment IS NULL AND p.container IS NULL AND p.host = ?)
+      OR (r.provider_started_at IS NOT NULL AND NOT EXISTS (SELECT 1 FROM run_process p WHERE p.run = r.id) AND NOT EXISTS (SELECT 1 FROM run_process_summary s WHERE s.run = r.id)))
+    ORDER BY r.id LIMIT ?`).all(before, hostname(), options.batch ?? 50).map(row => Number(row["id"]));
+    const out = [];
+    const shared = sharedLook(options);
+    for (const runId of runs) {
+        try {
+            out.push(await proveRunStopped(store, runId, clock, shared));
+        }
+        catch { /* one run's failure never stops the pass; the next pass looks again */ }
+    }
+    return out;
+}

package/dist/store.d.ts

@@ -59,6 +59,15 @@ export declare const LEAD_SWEEP_SCHEMA = "\nCREATE TABLE IF NOT EXISTS lead_swee
 export declare const LEAD_COMMITMENT_SCHEMA = "\nCREATE TABLE IF NOT EXISTS lead_commitment (\n  id             INTEGER PRIMARY KEY AUTOINCREMENT,\n  owner          TEXT NOT NULL,\n  repo           TEXT,\n  thread         INTEGER NOT NULL,\n  turn           INTEGER,\n  channel        TEXT NOT NULL DEFAULT 'chat',\n  what           TEXT NOT NULL,\n  condition_json TEXT NOT NULL,\n  check_at       TEXT NOT NULL,\n  expires_at     TEXT NOT NULL,\n  state          TEXT NOT NULL DEFAULT 'open' CHECK (state IN ('open', 'done', 'cancelled', 'expired')),\n  created_at     TEXT NOT NULL,\n  checked_at     TEXT,\n  closed_at      TEXT,\n  closed_by      TEXT,\n  outcome        TEXT\n);\nCREATE INDEX IF NOT EXISTS lead_commitment_due ON lead_commitment (state, check_at);\nCREATE INDEX IF NOT EXISTS lead_commitment_owner ON lead_commitment (owner, state, id);\n";
 /** What stops a finished run reading stopped (Store.stopQuiescenceFact). */
 export type StopFactKind = "open" | "alive" | "elsewhere" | "unprovable" | "unknown" | "settling";
+/** Why a finished run's stop is unproven where Toolroll's own process check (stop-proof.ts) can prove it: a run that
+ * started its agent with no process witness, a spawn witness with no pid, or a workspace note that can't be read. */
+export type StopProofCause = "no-witness" | "incomplete-spawn" | "workspace-unreadable";
+export type StopFact = {
+    run: number;
+    kind: StopFactKind;
+    problem: string;
+    cause?: StopProofCause;
+};
 export type TaskAct = "filed" | "approved" | "cancelled" | "completed" | "asked";
 /** Store.stopQuiescenceFact, read safely: a deploy proves completion with this code over the INSTALLED runtime's store,
  * which may predate the method. There the older stopQuiescenceProblem decides, and its words give the kind. */
@@ -234,7 +243,7 @@ export type SealedLedgerEntry = LedgerEntry & {
         hash: string;
     } | null;
 };
-export declare const SCHEMA_VERSION = 148;
+export declare const SCHEMA_VERSION = 149;
 /** v118: the tables merged into `credential`, in the order they move (a person's API tokens keep their ids first). */
 export declare const V118_MERGED_TABLES: Readonly<Record<string, CredentialKind>>;
 /**
@@ -426,7 +435,7 @@ export declare const V147_ADDED_COLUMNS: Readonly<Record<string, Readonly<Record
 }>>>>;
 /** The v147 migration's steps after the epoch, in order: a test seam (OpenOptions.migrationStep) may stop after each. */
 export declare const V147_STEPS: readonly ["epoch", "local-mode", "password-set", "web-session", "console-instance"];
-export type MigrationStep = (typeof V147_STEPS)[number] | (typeof V148_STEPS)[number];
+export type MigrationStep = (typeof V147_STEPS)[number] | (typeof V148_STEPS)[number] | (typeof V149_STEPS)[number];
 /**
  * v148 (docs/plans/results-routing.md): every table the results-routing arc uses, landed at once because a current
  * store runs no DDL. Slice 1 writes run_cohort, route_record and models_generation; route_rule, route_exposure* and
@@ -458,6 +467,33 @@ export declare const V148_ADDED_COLUMNS: Readonly<Record<string, Readonly<Record
 }>>>>;
 /** The v148 migration's steps after v147's, in order (OpenOptions.migrationStep may stop after each). */
 export declare const V148_STEPS: readonly ["results-tables", "exec-class", "revision-origin", "suggest-models"];
+/**
+ * v149 (docs/plans/learning-loops.md): every table the learning-loops arc uses, landed at once because a current store
+ * runs no DDL. Slice 1 writes check_fact, check_result and learning_watch; the rest are written from later slices.
+ * Each table has one writer module (the migration aside), which release-auto's learning scan (eval-hooks.ts
+ * learningWriteProblem) holds a release to: check_fact and check_result → check-facts.ts; local_observation → flaky.ts;
+ * ci_flaky_observation and ci_pairing_rejected → flaky-ci.ts; learning_watch → flaky.ts and lesson-notes.ts;
+ * lesson_note and lesson_note_tally → lesson-notes.ts; memory_use → memory-use.ts; memory_tidy_proposal and
+ * decision_merge → memory-tidy.ts.
+ *
+ * Nothing here is ever read for authority: a check's outcome, a verdict and an approval never change because of it.
+ *
+ * Deletion follows today's semantics (runs are never deleted by ordinary work; removing a project keeps its results).
+ * The tables are keyed by repo path with no foreign key to `project`; their run keys are plain REFERENCES with no ON
+ * DELETE action. check_fact, learning_watch, lesson_note, lesson_note_tally, memory_use, memory_tidy_proposal and
+ * decision_merge are kept and never deleted (their triggers allow only the columns their one writer sets: a watch's
+ * `until`, a tally's counters, a proposal's decision, a merge's undo). check_result, local_observation,
+ * ci_flaky_observation and ci_pairing_rejected are pruned after 180 days by the storage sweep's own step (a later
+ * slice), the only DELETE on them; their rows never change. The whole-project deletion (project-delete.ts), which
+ * lifts every guard for its one transaction, takes them with the project as it takes everything else.
+ */
+export declare const V149_SCHEMA = "\nCREATE TABLE IF NOT EXISTS check_fact (\n  run               INTEGER PRIMARY KEY REFERENCES run(id),\n  repo              TEXT,\n  task_ref          INTEGER NOT NULL,\n  head              TEXT,\n  tree              TEXT,\n  command_digest    TEXT,\n  access            TEXT,\n  harness_digest    TEXT,\n  lockfile          TEXT,\n  conditions_digest TEXT,\n  conditions_class  TEXT,\n  purpose           TEXT NOT NULL CHECK (purpose IN ('ordinary','release','nightly')),\n  learn_state       TEXT NOT NULL CHECK (learn_state IN ('recorded','not-recorded','not-asked')),\n  learn_why         TEXT,\n  recorded_at       TEXT NOT NULL,\n  CHECK ((learn_state = 'not-recorded') = (learn_why IS NOT NULL)),\n  CHECK ((conditions_digest IS NULL) = (conditions_class IS NULL)),\n  CHECK (conditions_digest IS NULL OR tree IS NOT NULL)\n);\nCREATE INDEX IF NOT EXISTS check_fact_by_repo ON check_fact (repo, recorded_at);\nCREATE INDEX IF NOT EXISTS check_fact_by_tree ON check_fact (tree, conditions_digest);\nCREATE TABLE IF NOT EXISTS check_result (\n  run         INTEGER NOT NULL REFERENCES check_fact(run),\n  identity    TEXT NOT NULL CHECK (length(identity) BETWEEN 1 AND 2600),\n  state       TEXT NOT NULL CHECK (state IN ('failed','passed','skipped','passed-on-retry')),\n  source      TEXT NOT NULL CHECK (source IN ('receipt','report','log','learn')),\n  recorded_at TEXT NOT NULL,\n  PRIMARY KEY (run, identity, source)\n);\nCREATE INDEX IF NOT EXISTS check_result_by_identity ON check_result (identity, run);\nCREATE TABLE IF NOT EXISTS local_observation (\n  id                INTEGER PRIMARY KEY AUTOINCREMENT,\n  repo              TEXT NOT NULL,\n  identity          TEXT NOT NULL,\n  failed_run        INTEGER NOT NULL REFERENCES run(id),\n  passed_run        INTEGER NOT NULL REFERENCES run(id),\n  tree              TEXT NOT NULL,\n  conditions_digest TEXT NOT NULL,\n  kind              TEXT NOT NULL CHECK (kind IN ('regate','same-tree','in-check-retry')),\n  observed_at       TEXT NOT NULL,\n  UNIQUE (identity, failed_run, passed_run),\n  CHECK ((kind = 'in-check-retry') = (failed_run = passed_run))\n);\nCREATE INDEX IF NOT EXISTS local_observation_by_repo ON local_observation (repo, identity, observed_at);\nCREATE TABLE IF NOT EXISTS ci_flaky_observation (\n  id                INTEGER PRIMARY KEY AUTOINCREMENT,\n  repo              TEXT NOT NULL,\n  provider          TEXT NOT NULL CHECK (provider = 'github'),\n  repository        TEXT NOT NULL,\n  workflow          TEXT NOT NULL,\n  lane_id           TEXT NOT NULL,\n  lane_name         TEXT NOT NULL,\n  workflow_run_id   INTEGER NOT NULL CHECK (workflow_run_id >= 1),\n  head_sha          TEXT NOT NULL,\n  failed_attempt    INTEGER NOT NULL CHECK (failed_attempt >= 1),\n  failed_job_id     INTEGER NOT NULL,\n  failed_conclusion TEXT NOT NULL,\n  passed_attempt    INTEGER NOT NULL,\n  passed_job_id     INTEGER NOT NULL,\n  passed_conclusion TEXT NOT NULL,\n  observed_at       TEXT NOT NULL,\n  UNIQUE (provider, repository, workflow_run_id, lane_id, failed_attempt, passed_attempt),\n  CHECK (passed_attempt > failed_attempt)\n);\nCREATE INDEX IF NOT EXISTS ci_flaky_observation_by_repo ON ci_flaky_observation (repo, lane_id, observed_at);\nCREATE TABLE IF NOT EXISTS ci_pairing_rejected (\n  provider        TEXT NOT NULL CHECK (provider = 'github'),\n  repository      TEXT NOT NULL,\n  workflow_run_id INTEGER NOT NULL CHECK (workflow_run_id >= 1),\n  attempt         INTEGER NOT NULL CHECK (attempt >= 1),\n  lane_id         TEXT,\n  why             TEXT NOT NULL,\n  at              TEXT NOT NULL,\n  PRIMARY KEY (provider, repository, workflow_run_id, attempt, why)\n);\nCREATE TABLE IF NOT EXISTS learning_watch (\n  repo       TEXT NOT NULL,\n  identity   TEXT NOT NULL CHECK (length(identity) BETWEEN 1 AND 2600),\n  reason     TEXT NOT NULL CHECK (reason IN ('note','observation')),\n  source_ref TEXT NOT NULL,\n  since      TEXT NOT NULL,\n  until      TEXT,\n  PRIMARY KEY (repo, identity, reason, source_ref),\n  CHECK (until IS NULL OR until >= since)\n);\nCREATE INDEX IF NOT EXISTS learning_watch_open ON learning_watch (repo, since) WHERE until IS NULL;\nCREATE TABLE IF NOT EXISTS lesson_note (\n  proposal                INTEGER PRIMARY KEY REFERENCES memory_proposal(id),\n  repo                    TEXT NOT NULL,\n  failed_run              INTEGER NOT NULL REFERENCES run(id),\n  passing_run             INTEGER NOT NULL REFERENCES run(id),\n  repair_chain            INTEGER REFERENCES repair_chain(id),\n  revision_origin         TEXT CHECK (revision_origin IS NULL OR revision_origin IN ('person','automatic-review','flow','lead','machine')),\n  identities_json         TEXT NOT NULL,\n  changed_paths_json      TEXT NOT NULL,\n  test_files_changed_json TEXT NOT NULL,\n  facts_digest            TEXT NOT NULL,\n  created_at              TEXT NOT NULL,\n  CHECK (failed_run <> passing_run)\n);\nCREATE INDEX IF NOT EXISTS lesson_note_by_repo ON lesson_note (repo, created_at);\nCREATE TABLE IF NOT EXISTS lesson_note_tally (\n  note                INTEGER NOT NULL REFERENCES lesson_note(proposal),\n  month               TEXT NOT NULL CHECK (month GLOB '[0-9][0-9][0-9][0-9]-[0-1][0-9]'),\n  executions          INTEGER NOT NULL DEFAULT 0 CHECK (executions >= 0),\n  failures            INTEGER NOT NULL DEFAULT 0 CHECK (failures >= 0 AND failures <= executions),\n  mixed_failures      INTEGER NOT NULL DEFAULT 0 CHECK (mixed_failures >= 0),\n  received_executions INTEGER NOT NULL DEFAULT 0 CHECK (received_executions >= 0 AND received_executions <= executions),\n  received_failures   INTEGER NOT NULL DEFAULT 0 CHECK (received_failures >= 0 AND received_failures <= received_executions AND received_failures <= failures),\n  unreported_checks   INTEGER NOT NULL DEFAULT 0 CHECK (unreported_checks >= 0),\n  sources_json        TEXT NOT NULL DEFAULT '[]',\n  partial             INTEGER NOT NULL DEFAULT 0 CHECK (partial IN (0,1)),\n  PRIMARY KEY (note, month)\n);\nCREATE TABLE IF NOT EXISTS memory_use (\n  run         INTEGER NOT NULL REFERENCES run(id),\n  repo        TEXT,\n  kind        TEXT NOT NULL CHECK (kind IN ('lesson','reference','decision')),\n  item        TEXT NOT NULL,\n  version     INTEGER,\n  phase       TEXT NOT NULL,\n  included_at TEXT,\n  PRIMARY KEY (run, kind, item)\n);\nCREATE INDEX IF NOT EXISTS memory_use_by_item ON memory_use (repo, kind, item, run);\nCREATE TABLE IF NOT EXISTS memory_tidy_proposal (\n  id            INTEGER PRIMARY KEY AUTOINCREMENT,\n  repo          TEXT NOT NULL,\n  kind          TEXT NOT NULL CHECK (kind IN ('retire','merge','contradiction','note-failing','restore','unmerge')),\n  items_json    TEXT NOT NULL,\n  after_text    TEXT,\n  rationale     TEXT NOT NULL,\n  evidence_json TEXT NOT NULL,\n  source        TEXT NOT NULL CHECK (source IN ('usage','duplicate','model','note','person')),\n  status        TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','accepted','rejected','stale')),\n  created_at    TEXT NOT NULL,\n  decided_by    TEXT,\n  decided_proof TEXT,\n  decided_at    TEXT,\n  CHECK ((status = 'pending') = (decided_at IS NULL)),\n  CHECK (status IN ('accepted','rejected') OR (decided_by IS NULL AND decided_proof IS NULL)),\n  CHECK (status NOT IN ('accepted','rejected') OR decided_by IS NOT NULL)\n);\nCREATE INDEX IF NOT EXISTS memory_tidy_proposal_pending ON memory_tidy_proposal (repo, status, kind);\nCREATE TABLE IF NOT EXISTS decision_merge (\n  merge                INTEGER NOT NULL,\n  successor            INTEGER NOT NULL REFERENCES project_decision(id),\n  successor_revision   INTEGER NOT NULL CHECK (successor_revision >= 1),\n  predecessor          INTEGER NOT NULL REFERENCES project_decision(id),\n  predecessor_revision INTEGER NOT NULL CHECK (predecessor_revision >= 1),\n  at                   TEXT NOT NULL,\n  by                   TEXT NOT NULL,\n  undone_at            TEXT,\n  undone_by            TEXT,\n  PRIMARY KEY (merge, predecessor),\n  CHECK (successor <> predecessor),\n  CHECK ((undone_at IS NULL) = (undone_by IS NULL))\n);\nCREATE INDEX IF NOT EXISTS decision_merge_by_successor ON decision_merge (successor);\nCREATE INDEX IF NOT EXISTS decision_merge_by_predecessor ON decision_merge (predecessor);\nCREATE TRIGGER IF NOT EXISTS check_fact_no_update BEFORE UPDATE ON check_fact BEGIN SELECT RAISE(ABORT,'check facts are recorded once'); END;\nCREATE TRIGGER IF NOT EXISTS check_fact_no_delete BEFORE DELETE ON check_fact BEGIN SELECT RAISE(ABORT,'check facts are kept'); END;\nCREATE TRIGGER IF NOT EXISTS check_result_no_update BEFORE UPDATE ON check_result BEGIN SELECT RAISE(ABORT,'check results are recorded once'); END;\nCREATE TRIGGER IF NOT EXISTS local_observation_no_update BEFORE UPDATE ON local_observation BEGIN SELECT RAISE(ABORT,'observations are recorded once'); END;\nCREATE TRIGGER IF NOT EXISTS ci_flaky_observation_no_update BEFORE UPDATE ON ci_flaky_observation BEGIN SELECT RAISE(ABORT,'observations are recorded once'); END;\nCREATE TRIGGER IF NOT EXISTS ci_pairing_rejected_no_update BEFORE UPDATE ON ci_pairing_rejected BEGIN SELECT RAISE(ABORT,'rejected pairings are recorded once'); END;\nCREATE TRIGGER IF NOT EXISTS learning_watch_only_ends BEFORE UPDATE ON learning_watch\n  WHEN NEW.repo IS NOT OLD.repo OR NEW.identity IS NOT OLD.identity OR NEW.reason IS NOT OLD.reason OR NEW.source_ref IS NOT OLD.source_ref\n    OR NEW.since IS NOT OLD.since OR OLD.until IS NOT NULL OR NEW.until IS NULL\n  BEGIN SELECT RAISE(ABORT,'a tracked identity only ends, once'); END;\nCREATE TRIGGER IF NOT EXISTS learning_watch_no_delete BEFORE DELETE ON learning_watch BEGIN SELECT RAISE(ABORT,'tracked identities are kept'); END;\nCREATE TRIGGER IF NOT EXISTS lesson_note_no_update BEFORE UPDATE ON lesson_note BEGIN SELECT RAISE(ABORT,'lesson note facts are recorded once'); END;\nCREATE TRIGGER IF NOT EXISTS lesson_note_no_delete BEFORE DELETE ON lesson_note BEGIN SELECT RAISE(ABORT,'lesson notes are kept'); END;\nCREATE TRIGGER IF NOT EXISTS lesson_note_tally_only_counts BEFORE UPDATE ON lesson_note_tally\n  WHEN NEW.note IS NOT OLD.note OR NEW.month IS NOT OLD.month OR NEW.executions < OLD.executions OR NEW.failures < OLD.failures\n    OR NEW.mixed_failures < OLD.mixed_failures OR NEW.received_executions < OLD.received_executions OR NEW.received_failures < OLD.received_failures\n    OR NEW.unreported_checks < OLD.unreported_checks OR NEW.partial < OLD.partial\n  BEGIN SELECT RAISE(ABORT,'a tally only counts up'); END;\nCREATE TRIGGER IF NOT EXISTS lesson_note_tally_no_delete BEFORE DELETE ON lesson_note_tally BEGIN SELECT RAISE(ABORT,'tallies are never pruned'); END;\nCREATE TRIGGER IF NOT EXISTS memory_use_no_update BEFORE UPDATE ON memory_use BEGIN SELECT RAISE(ABORT,'memory use is recorded once'); END;\nCREATE TRIGGER IF NOT EXISTS memory_use_no_delete BEFORE DELETE ON memory_use BEGIN SELECT RAISE(ABORT,'memory use is kept'); END;\nCREATE TRIGGER IF NOT EXISTS memory_tidy_proposal_only_decides BEFORE UPDATE ON memory_tidy_proposal\n  WHEN NEW.id IS NOT OLD.id OR NEW.repo IS NOT OLD.repo OR NEW.kind IS NOT OLD.kind OR NEW.items_json IS NOT OLD.items_json\n    OR NEW.after_text IS NOT OLD.after_text OR NEW.rationale IS NOT OLD.rationale OR NEW.evidence_json IS NOT OLD.evidence_json\n    OR NEW.source IS NOT OLD.source OR NEW.created_at IS NOT OLD.created_at OR OLD.status <> 'pending'\n  BEGIN SELECT RAISE(ABORT,'a tidy-up proposal is decided once'); END;\nCREATE TRIGGER IF NOT EXISTS memory_tidy_proposal_no_delete BEFORE DELETE ON memory_tidy_proposal BEGIN SELECT RAISE(ABORT,'tidy-up proposals are kept'); END;\nCREATE TRIGGER IF NOT EXISTS decision_merge_only_undone BEFORE UPDATE ON decision_merge\n  WHEN NEW.merge IS NOT OLD.merge OR NEW.successor IS NOT OLD.successor OR NEW.successor_revision IS NOT OLD.successor_revision\n    OR NEW.predecessor IS NOT OLD.predecessor OR NEW.predecessor_revision IS NOT OLD.predecessor_revision OR NEW.at IS NOT OLD.at\n    OR NEW.by IS NOT OLD.by OR OLD.undone_at IS NOT NULL\n  BEGIN SELECT RAISE(ABORT,'a merge is undone once, never rewritten'); END;\nCREATE TRIGGER IF NOT EXISTS decision_merge_no_delete BEFORE DELETE ON decision_merge BEGIN SELECT RAISE(ABORT,'merges are kept'); END;\n";
+/** The tables V149_SCHEMA adds, in the order a test drops them to make the v148 shape (dependents first). */
+export declare const V149_TABLES: readonly ["check_result", "check_fact", "local_observation", "ci_flaky_observation", "ci_pairing_rejected", "learning_watch", "lesson_note_tally", "lesson_note", "memory_use", "memory_tidy_proposal", "decision_merge"];
+/** The v149 tables a whole-project deletion clears by repo (the rest hang off these or a deleted run or decision). */
+export declare const V149_REPO_TABLES: readonly ["check_fact", "local_observation", "ci_flaky_observation", "learning_watch", "lesson_note", "memory_use", "memory_tidy_proposal"];
+/** The v149 migration's steps after v148's, in order (OpenOptions.migrationStep may stop after each). */
+export declare const V149_STEPS: readonly ["learning-tables"];
 /**
  * v135: one approval per release (docs/plans/release-once.md). `release_authority` is the single source of truth for a
  * release a person approved: one row per generation, changed only by compare-and-swap on `rev`, with a lease whose
@@ -6873,17 +6909,14 @@ export declare class Store {
      * what kind of answer it is (needs-you.ts reads it): `alive` while
      * something of the run may still be running, `elsewhere` when only
      * another computer can tell, `unprovable` when nothing is known to run
-     * but its exit was never recorded, `unknown` when this computer can't
-     * check at all (both are what `run settle` resolves; only the first may
-     * be called not running). `settling` already meets the automatic
-     * reconciliation proof, so no confirmation is needed. */
-    stopQuiescenceFact(runId: number): {
-        run: number;
-        kind: StopFactKind;
-        problem: string;
-    } | null;
-    /** `run settle` recorded an approver's word that a run whose workspace
-     * note couldn't be read has stopped. */
+     * but its exit was never recorded (Toolroll's own look, stop-proof.ts,
+     * settles it), `unknown` when this computer can't check, or that look
+     * was blocked. `settling` already meets the automatic reconciliation
+     * proof. Nobody is ever asked to confirm a build stopped. `cause` names
+     * what stop-proof.ts can prove. */
+    stopQuiescenceFact(runId: number): StopFact | null;
+    /** A run whose workspace note couldn't be read was proven free: by Toolroll's own look (stop-proof.ts), or, in
+     * ledgers from before it, by an approver's recorded word (still read, never written now). */
     private workspaceConfirmedByApprover;
     reserveRunProcess(runId: number, now: Date, group?: boolean): number;
     /** The run's process witnesses not yet proved exited — after first saving whatever exit evidence is available. */
@@ -6915,29 +6948,12 @@ export declare class Store {
      * here, every other witness of the run has exited and none of the run's
      * process groups is alive, it settles as never started, with a ledger
      * entry. A sole pid-less witness has no group to prove gone and stays for
-     * `run settle`. Native witnesses keep their OS-only road. */
+     * Toolroll's own look (stop-proof.ts). Native witnesses keep their OS-only road. */
     settleUnspawnedWitnesses(now: Date, onlyRun?: number): number;
     /** The existing automatic settlement proof, shared with its read-only
      * status. Unknown-only, foreign, native and still-running custody keep
      * their fences; only an observed single-process exit avoids a new probe. */
     private settleableUnspawnedWitnesses;
-    /** `run settle` (an approver's last resort): every witness of a finished
-     * run that cannot be proven either way is recorded settled under the
-     * approver's reason, in the ledger. Refuses while anything of the run is
-     * alive or could be checked only from another host. Never signals. */
-    settleRunWitnessesByApprover(args: {
-        runId: number;
-        by: string;
-        why: string;
-    }, now: Date): {
-        ok: true;
-        witnesses: number[];
-        repeated: boolean;
-    } | {
-        ok: false;
-        reason: "unknown-run" | "still-running" | "alive";
-        detail: string;
-    };
     recordRunProcess(runId: number, pid: number, now: Date, group?: boolean, witness?: number): void;
     /** The native OS object a witnessed spawn ran inside (v53) — recorded
      * with the spawn, before the target can execute. */
@@ -8400,3 +8416,26 @@ export declare function recordLaptopPort(store: Store, port: number, now: Date):
  * results-routing rows go (route-store.ts forgetProjectRoutes). One of the routing scan's named phase_* writers
  * (route-hooks.ts). How many rows went. */
 export declare function forgetProjectModels(store: Store, repo: string, now: Date): number;
+/** The ledger words of Toolroll's own stop proof. Nobody is asked to vouch that a build stopped: Toolroll looks at
+ * this computer's processes itself, and a person's older recorded word (`run settle` before this) is still read. */
+export declare const STOP_PROOF_SETTLED = "run stop proven automatically";
+export declare const STOP_PROOF_BLOCKED = "automatic stop check blocked";
+export declare const STOP_PROOF_WORKSPACE_FREE = "workspace proven free automatically";
+/** Why Toolroll's own stop check is blocked for a run, in words that follow "run #N " (the stop fact's problem), or
+ * null when its newest word is not a block (never checked, or proven since). */
+export declare function stopProofVerdictOf(store: Store, runId: number): string | null;
+/** Record that Toolroll's own stop check could not prove a run stopped (`words` follow "run #N "; `detail` is the
+ * evidence). The same block twice in a row is recorded once. True when recorded. */
+export declare function recordStopProofBlocked(store: Store, runId: number, words: string, detail: string, now: Date): boolean;
+/**
+ * Settle a finished run's unproven stop on Toolroll's own observation of this computer (stop-proof.ts): the caller
+ * looked at every process here and found none of the run's. Applies only while the run's stop is still unproven for
+ * exactly `cause` (re-read in this transaction): a pid-less spawn witness is recorded ended, a run with no witness gets
+ * one ended witness, an unreadable workspace note is recorded proven free. The evidence goes in the ledger. Never
+ * signals. True when this call settled it.
+ */
+export declare function settleRunStopByObservation(store: Store, args: {
+    runId: number;
+    cause: StopProofCause;
+    evidence: string;
+}, now: Date): boolean;