toolroll 0.9.77 → 0.9.78

package/dist/runner.js

@@ -26,6 +26,7 @@ import { inspectAcpAgent, inspectionWords } from "./acp-inspect.js";
 import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
 import { resolve } from "node:path";
 import { canonicalProject } from "./project.js";
+import { lifecycleOf, reportLifecycle } from "./store.js";
 import { ALL_CREDENTIAL_ENV, PROVIDER_IDS, inspectionOf } from "./provider.js";
 import { attestationOf, versionInRange } from "./attest.js";
 /** A runner's repo binding, in the same canonical form task filing uses —
@@ -39,11 +40,18 @@ export const RUNNER_TOKEN_MS = 365 * 86_400_000;
 /**
  * How long a runner may go quiet before it is presumed gone.
  *
- * Comfortably more than any sensible heartbeat interval: declaring a busy
- * machine dead because one check-in was slow costs more than waiting another
- * minute for a machine that really is.
+ * Four heartbeats (RUNNER_HEARTBEAT_MS): one slow check-in never declares a busy
+ * machine dead, and a machine that really is gone holds its name for a minute,
+ * not three (docs/plans/actors-lessons.md). A clean stop doesn't wait even that
+ * long: a draining or stopped holder never blocks the next start
+ * (registerRunnerIfIdle).
  */
-export const DEFAULT_LIVENESS_MS = 3 * 60_000;
+export const DEFAULT_LIVENESS_MS = 60_000;
+/** How often a live runner says so: `up`'s runner heartbeat and every watch lease renewal. */
+export const RUNNER_HEARTBEAT_MS = 15_000;
+/** The longest a watch lease may be taken or renewed for (durable-actors' MAX_HOST_LEASE_DURATION_MS): a crashed
+ * holder's projects are free within a minute. The store clamps every ttl to it. */
+export { MAX_WATCH_LEASE_MS } from "./store.js";
 export function register(store, options) {
     const { name, host, capacity = 1, repos = [], agents = [], now, newToken = mintToken, mutation = {}, } = options;
     const token = newToken();
@@ -78,6 +86,9 @@ export function register(store, options) {
     // Doing it from the `previous` snapshot above is racy: two registrars can
     // read the same predecessor and otherwise mint the same generation.
     const runner = store.saveRunner(proposed, hashToken(token), mutation);
+    // v137: this registration is started; its lifecycle row is bound to it by registered_at, so an earlier holder's
+    // drain or stop never describes it.
+    reportLifecycle(store, workerComponent(name), "started", runner.registeredAt, now);
     return {
         runner,
         token,
@@ -204,9 +215,30 @@ export function normalizeRunnerName(raw) {
  * the other gets the typed refusal.
  */
 export function registerRunnerIfIdle(store, options) {
-    const { name, now } = options;
+    const { name, now, previousGone, ...registration } = options;
     return store.transact(() => {
-        const existing = store.getRunner(name);
+        let existing = store.getRunner(name);
+        // The protection against two live holders stands: only a holder whose every process is proved gone, or whose own
+        // drain has ended (its stop bound passed, or it reported stopped), is signed off.
+        let signedOff = null;
+        const blocked = existing !== null && ((existing.runner.retiredAt === null && isAlive(existing.runner, now)) || store.liveWatchLeaseOf(name, now) !== null);
+        const drain = existing === null ? null : drainOf(store, existing.runner, now);
+        // A scheduled drain deadline is not proof that its timer ran or its children exited.
+        if (blocked && existing !== null && previousGone !== undefined) {
+            const proof = previousGone(existing);
+            if (proof.gone) {
+                store.retireRunner(name, now);
+                store.expireWatchLeasesOf(name, now);
+                store.recordAction({ at: now.toISOString(), actor: "toolroll up", repo: null, taskId: null, runId: null,
+                    action: "worker signed off", outcome: "processes proved gone", source: "policy", detail: proof.detail });
+                signedOff = proof.detail;
+                existing = store.getRunner(name);
+            }
+        }
+        // A live draining holder remains exclusive even if its advertised deadline has passed.
+        if (existing !== null && drain !== null && (isAlive(existing.runner, now) || store.liveWatchLeaseOf(name, now) !== null)) {
+            return { ok: false, reason: "runner-draining", until: drain.until, detail: `Toolroll is restarting (draining): ${name} is still stopping (scheduled by ${drain.until})` };
+        }
         if (existing !== null && existing.runner.retiredAt === null && isAlive(existing.runner, now)) {
             return {
                 ok: false,
@@ -227,8 +259,8 @@ export function registerRunnerIfIdle(store, options) {
         // otherwise hide it from every later recovery.
         const recoveredRuns = existing === null ? 0 : store.recoverRunnerWork(name, now).runs.length;
         // A restart keeps the capacity an operator set; only an explicit one changes it.
-        const registration = register(store, { ...options, capacity: options.capacity ?? existing?.runner.capacity ?? 1 });
-        return { ok: true, ...registration, recoveredRuns };
+        const registered = register(store, { ...registration, name, now, capacity: options.capacity ?? existing?.runner.capacity ?? 1 });
+        return { ok: true, ...registered, recoveredRuns, signedOff };
     });
 }
 /**
@@ -282,6 +314,10 @@ export function retireRunnerIfCurrent(store, name, token, now) {
         if (!auth.ok)
             return { ok: false, reason: auth.reason };
         store.retireRunner(name, now);
+        // Stopped is retired: its leases lapse now, so the next start or watch takes its projects over (recovering whatever
+        // this incarnation left) without waiting for them to run out.
+        store.expireWatchLeasesOf(name, now);
+        reportLifecycle(store, workerComponent(name), "stopped", auth.runner.registeredAt, now);
         return { ok: true };
     });
 }
@@ -435,3 +471,30 @@ export function reportProviderReadinessAuthed(store, args, now) {
         return auth;
     });
 }
+// ---- v137 lifecycle (docs/plans/actors-lessons.md): last in the file, so the reviewed function positions stay ----
+/** A worker's lifecycle row (service_lifecycle). */
+export const workerComponent = (name) => `worker:${name}`;
+/**
+ * The stop begins (durable-actors' draining report), fenced by this holder's credential: from now this runner takes
+ * no new work (claim.ts acquire refuses). `until` is a schedule, never proof that the holder exited.
+ */
+export function drainRunnerIfCurrent(store, name, token, now, until) {
+    return store.transact(() => {
+        const auth = authenticate(store, name, token);
+        if (!auth.ok)
+            return false;
+        return reportLifecycle(store, workerComponent(name), "draining", auth.runner.registeredAt, now, { drainUntil: until });
+    });
+}
+/** A runner's drain as its own lifecycle row reports it (only the current registration's row counts): null when it
+ * isn't draining; ended once its bound passed or it reported stopped. */
+export function drainOf(store, runner, now) {
+    const row = lifecycleOf(store, workerComponent(runner.name));
+    if (row === null || row.holder !== runner.registeredAt)
+        return null;
+    if (row.state === "stopped")
+        return { until: row.changedAt, ended: true };
+    if (row.state !== "draining" || row.drainUntil === null)
+        return null;
+    return { until: row.drainUntil, ended: Date.parse(row.drainUntil) <= now.getTime() };
+}

package/dist/scope.d.ts

@@ -412,6 +412,9 @@ export type Scope = SavedScopeTerms & {
      * carry policy, grant, check access, expiry. Part of the digest when present; a person's seal of them opens the
      * release's authority record. */
     releaseTerms?: ReleaseTerms | null;
+    /** v138 (docs/plans/pipeline-speed.md, P0): a release check's purpose when the release can't carry (no terms) — this
+     * exact candidate is a release's check. Part of the digest when present; never present beside terms. */
+    releaseCheck?: ReleaseCheckPurpose | null;
 };
 /** Who wrote a scope's text, when not a person (approval-policy plan §1.4). */
 export type ScopeAuthor = "mate" | "coordinator" | "scout" | "lead";
@@ -464,6 +467,9 @@ export type ScopeInput = {
     /** v135: release terms (release-carry.ts releaseTermsFor), from the plane's own records. Undefined keeps any terms
      * the scope already carries; null removes them. */
     releaseTerms?: ReleaseTerms | null;
+    /** v138: a release check's purpose (release-carry.ts releaseCheckPurposeFor), from the plane's own records. Undefined
+     * keeps any purpose the scope already carries; null removes it. */
+    releaseCheck?: ReleaseCheckPurpose | null;
     now: Date;
     mutation?: Mutation;
 };
@@ -488,6 +494,8 @@ export declare function digestOf(scope: Pick<Scope, "goal" | "outOfScope" | "tou
     candidate?: string | null;
     /** v135: release terms, folded in only when present. */
     releaseTerms?: ReleaseTerms | null;
+    /** v138: a release check's purpose, folded in only when present. */
+    releaseCheck?: ReleaseCheckPurpose | null;
 }, target?: ExecutionProfile | null | {
     chain: readonly ChainEntry[];
 }, route?: PhaseRoute | null): string;
@@ -864,10 +872,12 @@ export declare function scopeRouteWords(scope: Pick<Scope, "proposedRouteJson">,
 export declare function describeScope(scope: Scope, readiness?: ReadinessLookup): string[];
 /** The approval sheet's release lines (release-once.md): the release in one sentence, then what asks again. */
 export declare function releaseTermsSentence(terms: ReleaseTerms): string;
+/** The approval sheet's line for a release check purpose (pipeline-speed.md, P0): a release that can't carry. */
+export declare function releaseCheckSentence(purpose: ReleaseCheckPurpose): string;
 /** What asks again, in full: the terms behind the sentence. */
 export declare function releaseAsksAgainWords(terms: ReleaseTerms): string[];
 export { personProof } from "./person-proof.js";
-import { type ReleaseTerms } from "./contracts/release-terms.js";
+import { type ReleaseCheckPurpose, type ReleaseTerms } from "./contracts/release-terms.js";
 import { type AutoEvidence, type ReleaseAuthority, type ReleaseCarry } from "./release-authority.js";
 /** Whether the command running now is a machine's: the lead (its token) or a remote API run. Trusted context only. */
 export declare const machineActing: () => boolean;
@@ -955,6 +965,14 @@ export declare function mainDebtProblem(store: Store, repo: string, taskId: stri
  * chat it is approved only through the release card.
  */
 export declare function deployGateScope(scope: Pick<Scope, "candidate" | "releaseTerms">): boolean;
+/**
+ * Whether a scope's check is a release's check (docs/plans/pipeline-speed.md, P0): it carries signed release terms, or a
+ * release check purpose sealed with it. Read from the scope's signed fields only, independent of whether the release can
+ * carry. It makes the check stricter (a complete, valid report; CI never stands in for its units) and, through
+ * check-failure.ts recheckReleaseTask, lets the plane rerun the identical candidate after confirmed infrastructure; it
+ * never carries a later commit or approves anything.
+ */
+export declare function releaseCheckScope(scope: Pick<Scope, "releaseTerms" | "releaseCheck"> | null): boolean;
 /**
  * The plane's own automatic approval of a release task (docs/plans/auto-approve-green.md), read the way the dispatch
  * belts read it (release-authority.ts autoSealSql): sealed `kind` on the current digest, its evidence row naming exactly

package/dist/scope.js

@@ -619,6 +619,7 @@ route) {
         ...(scope.qualityMode === "strict" ? { qualityMode: "strict" } : {}),
         ...(scope.candidate ? { candidate: scope.candidate } : {}),
         ...(scope.releaseTerms == null ? {} : { releaseTerms: releaseTermsDigest(scope.releaseTerms) }),
+        ...(scope.releaseCheck == null ? {} : { releaseCheck: releaseCheckPurposeDigest(scope.releaseCheck) }),
         // The single outer key, whichever target produced its inner value:
         // absent => the golden and every profileless approval are untouched;
         // a legacy profile => its exact profileDigestOf; an explicit chain
@@ -672,6 +673,7 @@ function proposeLocked(store, input) {
         ...(input.qualityMode === undefined ? {} : { qualityMode: input.qualityMode }),
         ...(posture === undefined ? {} : { posture }),
         ...(input.releaseTerms === undefined ? {} : { releaseTerms: input.releaseTerms }),
+        ...(input.releaseCheck === undefined ? {} : { releaseCheck: input.releaseCheck }),
         proposedVia,
     });
     // The store may have RECOMPUTED the digest to bind the resolved profile
@@ -1265,7 +1267,7 @@ export function scopeAuthorityOf(scope, env = {}) {
     const parity = routeParityProblem(route, profile, { riskLevel: scope.riskLevel ?? "routine", qualityMode: scope.qualityMode ?? "default" });
     if (parity !== null)
         return { ok: false, reason: "parity", problem: parity };
-    const digest = digestOf({ goal: scope.goal, outOfScope: scope.outOfScope, touches: scope.touches, budgetMicrousd: scope.budgetMicrousd, acceptance: scope.acceptance, candidate: scope.candidate ?? null, qualityMode: scope.qualityMode ?? "default", releaseTerms: scope.releaseTerms ?? null }, profile, route);
+    const digest = digestOf({ goal: scope.goal, outOfScope: scope.outOfScope, touches: scope.touches, budgetMicrousd: scope.budgetMicrousd, acceptance: scope.acceptance, candidate: scope.candidate ?? null, qualityMode: scope.qualityMode ?? "default", releaseTerms: scope.releaseTerms ?? null, releaseCheck: scope.releaseCheck ?? null }, profile, route);
     if (digest !== scope.digest) {
         return { ok: false, reason: "digest", problem: "the scope's reference does not re-derive from its own terms, agents, and route — file it again" };
     }
@@ -1325,6 +1327,7 @@ export function describeScope(scope, readiness = NO_READINESS) {
         // Its sandbox terms (project-sandbox.ts), part of what the yes agrees to: what it reads, and any widening.
         ...(scope.profile === null || scope.profile === undefined ? [] : ((terms) => [`  reads        ${terms.reads}`, ...(terms.mach === null ? [] : [`  services     ${terms.mach}`])])(sandboxTermsWords(scope.profile))),
         ...releaseTermsWords(scope.releaseTerms ?? null),
+        ...(scope.releaseCheck == null ? [] : [`  release      ${releaseCheckSentence(scope.releaseCheck)}`]),
         `  reference    ${scope.digest}`,
         `  approved     ${describeApproval(approval)}`,
     ];
@@ -1336,6 +1339,10 @@ export function releaseTermsSentence(terms) {
     const outside = terms.access === "sandboxed" ? "" : " This project's checks don't run in the sandbox: those later test-only commits will run outside the sandbox without being shown to you first.";
     return `Release ${terms.packageVersion} from pull request #${terms.pr} at ${terms.approvedSha.slice(0, 7)}. Later commits on this pull request that only change tests, docs or the changelog ship without asking you again, until it merges or for 3 days. You'll see each one in the release summary.${outside}`;
 }
+/** The approval sheet's line for a release check purpose (pipeline-speed.md, P0): a release that can't carry. */
+export function releaseCheckSentence(purpose) {
+    return `Release ${purpose.packageVersion} from pull request #${purpose.pr} at ${purpose.approvedSha.slice(0, 7)}. Every later commit asks for your approval again. If Toolroll's own machine fails during the check, it reruns this same commit's check (at most twice).`;
+}
 /** What asks again, in full: the terms behind the sentence. */
 export function releaseAsksAgainWords(terms) {
     const p = terms.policy;
@@ -1364,7 +1371,7 @@ function describeApproval(approval) {
 // ---- Only a person approves (docs/plans/approval-policy.md, phase 1). Kept last: the freshness tripwire pins places.
 export { personProof } from "./person-proof.js";
 import { personProof } from "./person-proof.js";
-import { releaseTermsDigest } from "./contracts/release-terms.js";
+import { releaseCheckPurposeDigest, releaseTermsDigest } from "./contracts/release-terms.js";
 import { CHECK_ACCESS_WIDTH, autoEvidenceOf, carriesUnsandboxed, newestReleaseGeneration, releaseAuthorityOfTask, releaseAutoApproves, releaseSealProblem } from "./release-authority.js";
 /** Whether the command running now is a machine's: the lead (its token) or a remote API run. Trusted context only. */
 export const machineActing = () => currentActor()?.lead === true || activeRemote() !== null;
@@ -1548,6 +1555,16 @@ export function mainDebtProblem(store, repo, taskId) {
 export function deployGateScope(scope) {
     return (scope.candidate ?? null) !== null || (scope.releaseTerms ?? null) !== null;
 }
+/**
+ * Whether a scope's check is a release's check (docs/plans/pipeline-speed.md, P0): it carries signed release terms, or a
+ * release check purpose sealed with it. Read from the scope's signed fields only, independent of whether the release can
+ * carry. It makes the check stricter (a complete, valid report; CI never stands in for its units) and, through
+ * check-failure.ts recheckReleaseTask, lets the plane rerun the identical candidate after confirmed infrastructure; it
+ * never carries a later commit or approves anything.
+ */
+export function releaseCheckScope(scope) {
+    return scope !== null && ((scope.releaseTerms ?? null) !== null || (scope.releaseCheck ?? null) !== null);
+}
 /**
  * The plane's own automatic approval of a release task (docs/plans/auto-approve-green.md), read the way the dispatch
  * belts read it (release-authority.ts autoSealSql): sealed `kind` on the current digest, its evidence row naming exactly

package/dist/scout.js

@@ -21,7 +21,7 @@ import { run } from "./exec.js";
 import { checkoutGit } from "./teardown-git.js";
 import { auditOf } from "./provider.js";
 import { currentClaim, heartbeat } from "./claim.js";
-import { heartbeat as runnerHeartbeat } from "./runner.js";
+import { heartbeat as runnerHeartbeat, RUNNER_HEARTBEAT_MS } from "./runner.js";
 import { parseDecision } from "./decision.js";
 import { scoutHandbackSchema } from "./contracts/scout-report.js";
 import { parseReport, REPORT_IMAGE_FILE, REPORT_LIMITS, REPORT_VERSION, SCOUT_OUTPUT_JSON_SCHEMA } from "./scout-report.js";
@@ -43,7 +43,7 @@ const GIT = "git";
 const AGENT_ENV_DENYLIST = [...TELEGRAM_TOKEN_ENVS];
 const DEFAULT_SCOUT_TIMEOUT_MS = 20 * 60_000;
 const DEFAULT_SCOUT_TURNS = CLAUDE_LIMITS.maxTurns;
-const DEFAULT_PULSE_MS = 60_000;
+const DEFAULT_PULSE_MS = RUNNER_HEARTBEAT_MS;
 /** The scout's own headless browser: the pinned Playwright tool, saving its
  * screenshots straight into the run's image folder. */
 export const SCOUT_BROWSER = "toolroll-browser";

package/dist/serve.js

@@ -100,6 +100,7 @@ import { prepareWorkspaceRevision, WorkspaceValidatorCache } from "./workspace-r
  * repo, only runs whose task belongs to that repo (or to no repo yet).
  */
 import { createHash, randomBytes } from "node:crypto";
+import { keepSlowRequest, Timeline } from "./latency-timeline.js";
 import { readFileSync } from "node:fs";
 import { createServer } from "node:http";
 import { homedir, hostname } from "node:os";
@@ -775,6 +776,9 @@ export function createDecisionServer(options) {
         // uses the address it reached), reads one row, and says nothing else about the installation.
         const method = request.method ?? "GET";
         const matchedRoute = matchRoute(method, hook.pathname);
+        // Where a slow answer's time went (latency-timeline.ts): milestones by route id only, kept when it took a second.
+        const timeline = new Timeline();
+        timeline.mark("route matched");
         const edgeRoute = matchedRoute?.stage === 'edge' ? matchedRoute : null;
         const wrongMethodEdge = edgeRoute === null ? ROUTES.find(row => row.stage === 'edge' && new RegExp(row.pattern).test(hook.pathname)) ?? null : null;
         // D7: a password on a request still signs in this release, announced on every answer and named in every refusal;
@@ -837,6 +841,7 @@ export function createDecisionServer(options) {
             return;
         const refusedToken = {};
         const who = identify(request, touch, edgeRoute?.domain === 'live', refusedToken);
+        timeline.mark("signed in");
         if (refusedToken.principal !== undefined) {
             if (refuseBudget(requestBudget.admit(refusedToken.principal.tokenId, "api")))
                 return;
@@ -883,7 +888,9 @@ export function createDecisionServer(options) {
             freshWork: ["request", "task", "result"].some(name => url.searchParams.has(name)),
             ...(workspaceRead && typeof request.headers['if-none-match'] === 'string' ? { workspaceIfNoneMatch: request.headers['if-none-match'] } : {}),
             workspaceRequest: url.searchParams.get('request'),
+            timeline,
         };
+        response.once("finish", () => keepSlowRequest(store, timeline, matchedRoute?.id ?? "unmatched", response.statusCode, clock()));
         if (method === "GET" || method === "POST")
             return freshFacts.run(fresh, () => provenPasswordAccount.run(passwordProof, () => requestContext.run(requestFacts, async () => {
                 if (edgeRoute?.domain === 'live')
@@ -937,6 +944,7 @@ export function createDecisionServer(options) {
                                 etag: '"' + createHash('sha256').update(`${key}:${revision}:${expiresAt}`).digest('hex') + '"' };
                         }
                         await prefetchChrome(route, url, who);
+                        timeline.mark("chrome and worker reads");
                         // The worker wait lets a sign-out, credential/access change or console ceiling change land. Prove the
                         // cookie and route again before rendering protected content or answering with a content validator.
                         if (who.via === 'cookie') {
@@ -1454,6 +1462,7 @@ export function createDecisionServer(options) {
      * are what sensitivity strips.
      */
     function sendScreen(response, status, s) {
+        requestContext.getStore()?.timeline?.mark("handler done; rendering");
         // Sensitivity is the page as authored (cockpit §2, R14): its own explicit word, else a password field it was written
         // with — judged before an identity-provider sign-in replaces that field, so an SSO step-up page stays sensitive.
         const authored = s.sensitive ?? (s.forceSensitive === true ||
@@ -1486,6 +1495,8 @@ export function createDecisionServer(options) {
                 notices.push(s.chrome.modeBanner.words);
             if (s.chrome.updateWaiting)
                 notices.push(s.chrome.updateWaiting.words);
+            if (options.draining?.() === true)
+                notices.push('Toolroll is restarting (draining). Saved work is kept; new builds start once it is back.');
             let crew = { crew: [], crewTruncated: false };
             try {
                 const project = s.chrome.active === 'chat' ? null : s.chrome.project;

package/dist/server/http.d.ts

@@ -148,6 +148,8 @@ export type ServeOptions = {
     ssoFetch?: typeof fetch;
     /** Cockpit B1: this `up` console's instance secret (null: none — the instance route answers 404). */
     instanceSecret?: () => string | null;
+    /** v137: whether this console is draining (its `up` is stopping): /healthz says so at once, and pages say Toolroll is restarting. */
+    draining?: () => boolean;
     /** Cockpit E: how Finder is asked to reveal a proved checkout (tests pass their own; never a launch). True: shown. */
     revealPath?: (path: string) => Promise<boolean> | boolean;
     chatEnv?: Record<string, string | undefined>;

package/dist/server/pages.js

@@ -43,6 +43,7 @@ import { healthWords } from "../server-telemetry.js";
 import { tally } from "../summary.js";
 import { serveInstallAsset } from "./pages-assets.js";
 import { healthz } from "./remote-hooks.js";
+import { slowRequestLines, SLOW_REQUEST_MS } from "../latency-timeline.js";
 import { leadsStillAdmitted, leadTilesFor } from './agents-live-lead.js';
 import { refuse, regionScript, screen, settingsRows, WORKSPACE_STYLE, workToolRows } from "./chrome.js";
 import { redirect, respond, safeReturn, SAFETY, taskHref } from "./http.js";
@@ -627,9 +628,12 @@ export function createPagesHandlers(runtime) {
     async function health(ctx) {
         const { request, response } = ctx;
         const health = store.telemetry.snapshot();
+        // Beside /healthz, for an instance operator: where the slowest recent console answers spent their time (route ids
+        // and milestones only).
+        const slow = slowRequestLines(store);
         return request.headers.accept?.includes('application/json')
-            ? respond(response, 200, 'application/json; charset=utf-8', JSON.stringify(health))
-            : respond(response, 200, 'text/plain; charset=utf-8', `${healthWords(health)}\n`);
+            ? respond(response, 200, 'application/json; charset=utf-8', JSON.stringify({ ...health, slowRequests: slow }))
+            : respond(response, 200, 'text/plain; charset=utf-8', `${healthWords(health)}\n${slow.length === 0 ? "" : `\nSlow console requests (over ${SLOW_REQUEST_MS} ms), newest first:\n${slow.join("\n")}\n`}`);
     }
     // v104: Prometheus metrics, for an instance operator (a scraper sends one's API token).
     async function metrics(ctx) {
@@ -984,7 +988,7 @@ export function createPagesHandlers(runtime) {
     }
     async function healthzEdge(ctx) {
         const { response, method } = ctx;
-        return healthz(store, response, method === 'HEAD');
+        return healthz(store, response, method === 'HEAD', options.draining?.() === true);
     }
     async function browserAsset(ctx) {
         const { request, response, url } = ctx;

package/dist/server/remote-hooks.d.ts

@@ -26,4 +26,4 @@ export declare function telegramHook(ctx: RemoteHookContext, request: IncomingMe
 export declare function flowHook(ctx: RemoteHookContext, request: IncomingMessage, response: ServerResponse, url: URL): Promise<void>;
 /** A shared button as a public form (v84): its questions, and one card per submission. No sign-in, no session, nothing about the flow shown. */
 export declare function flowForm(ctx: RemoteHookContext, request: IncomingMessage, response: ServerResponse, url: URL): Promise<void>;
-export declare function healthz(store: Store, response: ServerResponse, head: boolean): void;
+export declare function healthz(store: Store, response: ServerResponse, head: boolean, draining?: boolean): void;

package/dist/server/remote-hooks.js

@@ -122,7 +122,7 @@ export async function flowForm(ctx, request, response, url) {
         return page({ status: 500, html: "" });
     }
 }
-export function healthz(store, response, head) {
+export function healthz(store, response, head, draining = false) {
     let healthy = true;
     try {
         store.handle.prepare("SELECT 1 FROM schema_version").get();
@@ -130,7 +130,9 @@ export function healthz(store, response, head) {
     catch {
         healthy = false;
     }
-    const body = JSON.stringify({ status: healthy ? "ok" : "unavailable" });
-    response.writeHead(healthy ? 200 : 503, { "content-type": "application/json", "cache-control": "no-store", "x-content-type-options": "nosniff" });
+    // A draining console (its `up` is stopping) still answers pages, but says so here at once: a probe moves on instead
+    // of waiting for it to disappear.
+    const body = JSON.stringify(draining && healthy ? { status: "draining", message: "Toolroll is restarting (draining)" } : { status: healthy ? "ok" : "unavailable" });
+    response.writeHead(healthy && !draining ? 200 : 503, { "content-type": "application/json", "cache-control": "no-store", "x-content-type-options": "nosniff" });
     response.end(head ? undefined : body);
 }

package/dist/server/render-results.d.ts

@@ -185,6 +185,8 @@ export type ReviewCockpitView = {
     efficiency?: RunEfficiency | null;
     /** What the run's network was and what its proxy saw (egress.ts): display only. */
     network?: RunNetworkRecord | null;
+    /** Where the time went (latency-timeline.ts): its release check's parts and the task's deploys, display only. */
+    timing?: import("../latency-timeline.js").TimingView | null;
     /** How far its checks reached (check-access.ts): shown only when wider than sandboxed. */
     checkAccess?: CheckAccess;
     taskId: string;

package/dist/server/render-results.js

@@ -625,6 +625,7 @@ export function reviewCockpitDetailParts(view, csrf, noted, canRetryReview, tab,
         notes: view.notes.map(one => ({ author: one.author, at: one.createdAt, note: one.note })),
         efficiency: view.efficiency ?? null,
         network: view.network ?? null,
+        timing: view.timing ?? null,
         acts: { primary: null, secondary: null, line: null }, runChecks: null, mismatch: null,
         // The raw run record lives under Details now (2026-10-02): /r/<id> for this result redirects here.
         record: run === null ? null : { build: run.id, href: `/r/${run.id}?record=1`, facts: [

package/dist/server/request-context.d.ts

@@ -18,6 +18,8 @@ export type RequestFacts = {
     returnTo: string;
     actor?: string;
     createdTask?: string;
+    /** This request's latency timeline (latency-timeline.ts): kept when the answer was slow. */
+    timeline?: import("../latency-timeline.js").Timeline;
     browser?: boolean;
     workspaceRead?: boolean;
     /** Receipt reconciliation, task and result reads (`request`, `task`, `result`): work is read afresh, never cached. */

package/dist/server/settings.js

@@ -117,7 +117,9 @@ import { sweepSchedule } from "../storage-sweep-schedule.js";
 import { storageHtml } from "../storage-ui.js";
 import { landedSummary } from "../landed-cleanup.js";
 import { bytesWords, parseCleanup } from "../storage.js";
-import { isDigestTime, RESULT_SCREENSHOTS } from "../store.js";
+import { isDigestTime, liveBackupState, RESULT_SCREENSHOTS, setLiveBackupSettings } from "../store.js";
+import { defaultLiveFolder, liveFolderOf, liveFolderProblem } from "../live-backup.js";
+import { LIVE_EVERY } from "../backup-ui.js";
 import { checkTeamsCredentials, clearTeamsCredentials, loadTeamsCredentials, saveTeamsCredentials, TeamsError } from "../teams-api.js";
 import { teamsSettingsHtml } from "../teams-settings.js";
 import { telegramSettingsHtml } from "../telegram-settings.js";
@@ -497,7 +499,9 @@ ${connectionHtml(asked, connection, task)}${command === null ? "" : html `<p>On
         if (databaseFile === null)
             return refuse(response, who, 403, "An instance operator looks after backups.", "/settings");
         const settings = store.backupSettings();
-        const view = { settings, folder: backupFolderOf(settings, databaseFile), defaultFolder: defaultBackupFolder(databaseFile), runs: store.backupRuns(20), csrf: who.session.csrf, now };
+        const live = liveBackupState(store);
+        const view = { settings, folder: backupFolderOf(settings, databaseFile), defaultFolder: defaultBackupFolder(databaseFile), runs: store.backupRuns(20), csrf: who.session.csrf, now,
+            live: { state: live, folder: liveFolderOf(live.settings, databaseFile), defaultFolder: defaultLiveFolder(databaseFile) } };
         return sendScreen(response, 200, screen("Backups", html `<p><a href="/settings">Settings</a></p><h1>Backups</h1>${(backupHtml(view, { said: url.searchParams.get("said"), problem: url.searchParams.get("problem") }))}`, { chrome: chromeFor(project, "settings") }));
     }
     // v105: Settings → Data, where an instance operator downloads everything. The download is a POST with the password.
@@ -1325,8 +1329,24 @@ ${repositoryContextHtml(chosen, query, result, who.role === 'approver' ? csrf :
             if (problem !== null)
                 return back("problem", problem);
         }
+        // The continuous copy (live-backup.ts): how often, and a folder that is a plain, private folder of this user's
+        // outside every project and checkout. A form without these fields leaves it as it is.
+        const liveBefore = liveBackupState(store).settings;
+        const liveEvery = body.get("live_every") ?? null;
+        const liveMinutes = Number(liveEvery);
+        if (liveEvery !== null && liveEvery !== "off" && !LIVE_EVERY.includes(liveMinutes))
+            return back("problem", "Choose how often to copy continuously.");
+        const liveTyped = body.get("live_folder") === null ? liveBefore.folder ?? "" : (body.get("live_folder") ?? "").trim();
+        const liveFolder = liveTyped === "" || liveTyped === defaultLiveFolder(databaseFile) ? null : liveTyped;
+        if (liveFolder !== null) {
+            const problem = liveFolderProblem(liveFolder, [...new Set([...managedRepos(), ...store.knownRepos(), ...(options.poolRoot === undefined ? [] : [options.poolRoot])])]);
+            if (problem !== null)
+                return back("problem", problem);
+        }
         const before = store.backupSettings();
         store.setBackupSettings({ enabled: every !== "off", everyHours: every === "off" ? before.everyHours : hours, keep, folder }, who.name, now);
+        setLiveBackupSettings(store, { enabled: liveEvery === null ? liveBefore.enabled : liveEvery !== "off", everyMinutes: liveEvery === null || liveEvery === "off" ? liveBefore.everyMinutes : liveMinutes,
+            keep: liveBefore.keep, folder: liveFolder }, who.name, now);
         return back("said", every === "off" ? "Saved. Scheduled backups are off." : "Saved.");
     }
     // v105: download everything as a .zip. An instance operator, with a step-up; the ledger records it.

package/dist/server/tasks.js

@@ -66,6 +66,7 @@ import { boardTickScript, oneLineOf, QUEUE_VIEW, refuse, regionScript, screen, t
 import { chatReturnWithSaid, matchTaskPath, redirect, respond, reviewHref, safeChatReturn, safeReturn, SAFETY, taskChatHref, taskHref } from "./http.js";
 import { boardBody, boardWorkers, donePage, inboxFingerprints, inboxPage, newTaskPage, nextPage, parseInboxTab, QUEUE_FRONT, queueBody, queueScript, TASK_STATES, workPage } from "./render-pages.js";
 import { runEfficiencyOf } from "../run-efficiency.js";
+import { timingFor } from "../latency-timeline.js";
 import { ACCEPT_ANYWAY_NEEDS_REASON, editorFileHref, proofBundleView, rankReviewQueue, RESULT_REFUSALS, REVIEW_QUEUE_CAP, reviewCockpitDetailParts, reviewCockpitPage, reviewPriorityOf, runFactsFragment, runOutcomeBadge, runPage, RUNS_PAGE, runsPage, withRefusal } from "./render-results.js";
 import { approvalFormDigest, approveRefusalWords, checkProgressHtml, consentDoorOf, decisionPage, nonceHashOf, PermissionsWouldChange, requirementsFromEditor, resumeDigestOf, taskOf } from "./render-tasks.js";
 import {} from "./session.js";
@@ -2837,7 +2838,7 @@ export function createTasksHandlers(runtime) {
             ...(store.isDemo() ? { demo: true } : {}),
         };
         if (run === null) {
-            return { ...base, run: null, detail: null, reviewRetry: null, review: null, notes: [] };
+            return { ...base, run: null, detail: null, reviewRetry: null, review: null, notes: [], timing: timingFor(store, null, row.taskId) };
         }
         const detail = resultDetailOf(run, who, now);
         return {
@@ -2866,6 +2867,8 @@ export function createTasksHandlers(runtime) {
             } })(),
             // What its network was and what its proxy saw (egress.ts), display only.
             network: store.runNetwork(run.id),
+            // Where the time went: its release check's parts and the task's deploys (latency-timeline.ts), display only.
+            timing: timingFor(store, run.id, row.taskId),
             // How far its checks reached (check-access.ts), display only.
             checkAccess: store.runCheckAccess(run.id),
             reviewRetry: store.reviewRetryStateOf(run.id),

package/dist/storage-sweep.js

@@ -8,7 +8,7 @@
  * Nothing live work uses goes (`liveWork`): while a run is open, a lease current or a checkout in use, each removal
  * and each signal first asks again whether that work uses the folder, and when its processes can't be read nothing
  * goes. Every scan and removal runs in a separate child: synchronous disk work must never delay the worker's lease
- * heartbeats (90 s). On Oct 12 the sweep starved those heartbeats and three builds ended interrupted half a second
+ * heartbeats (a lease lasts a minute). On Oct 12 the sweep starved those heartbeats and three builds ended interrupted half a second
  * after it finished. Once a run has ended, the next sweep removes what
  * it no longer uses.
  *