toolroll 0.9.77 → 0.9.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/dist/agent-fence.js +18 -2
- package/dist/backup-ui.d.ts +9 -1
- package/dist/backup-ui.js +28 -3
- package/dist/backup.js +4 -3
- package/dist/browser/workspace.css +1 -1
- package/dist/browser/workspace.js +18 -18
- package/dist/browser-workspace.d.ts +2 -0
- package/dist/builder.d.ts +4 -3
- package/dist/builder.js +14 -8
- package/dist/check-failure.js +31 -3
- package/dist/check-report.d.ts +18 -3
- package/dist/check-report.js +67 -7
- package/dist/claim.d.ts +9 -0
- package/dist/claim.js +9 -1
- package/dist/contracts/console-api.d.ts +1 -1
- package/dist/contracts/console-api.js +1 -1
- package/dist/contracts/release-terms.d.ts +41 -0
- package/dist/contracts/release-terms.js +53 -0
- package/dist/controller-service.js +4 -1
- package/dist/deploy-rescue.d.ts +41 -0
- package/dist/deploy-rescue.js +94 -0
- package/dist/desktop-update.d.ts +1 -1
- package/dist/dispose.js +4 -3
- package/dist/evidence.d.ts +9 -0
- package/dist/evidence.js +15 -3
- package/dist/exec.d.ts +2 -0
- package/dist/exec.js +15 -1
- package/dist/guides.js +18 -8
- package/dist/latency-timeline.d.ts +46 -0
- package/dist/latency-timeline.js +120 -0
- package/dist/live-backup.d.ts +39 -0
- package/dist/live-backup.js +271 -0
- package/dist/operate.js +203 -16
- package/dist/planner.js +2 -2
- package/dist/release-adapters.d.ts +32 -1
- package/dist/release-adapters.js +99 -7
- package/dist/release-carry.d.ts +21 -1
- package/dist/release-carry.js +22 -0
- package/dist/release-cli.d.ts +3 -0
- package/dist/release-cli.js +74 -5
- package/dist/release-gate.d.ts +5 -0
- package/dist/release-gate.js +1 -1
- package/dist/release-recheck.js +3 -2
- package/dist/release.d.ts +37 -4
- package/dist/release.js +128 -43
- package/dist/runner-process.d.ts +67 -0
- package/dist/runner-process.js +219 -0
- package/dist/runner.d.ts +48 -6
- package/dist/runner.js +71 -8
- package/dist/scope.d.ts +19 -1
- package/dist/scope.js +19 -2
- package/dist/scout.js +2 -2
- package/dist/serve.js +11 -0
- package/dist/server/http.d.ts +2 -0
- package/dist/server/pages.js +7 -3
- package/dist/server/remote-hooks.d.ts +1 -1
- package/dist/server/remote-hooks.js +5 -3
- package/dist/server/render-results.d.ts +2 -0
- package/dist/server/render-results.js +1 -0
- package/dist/server/request-context.d.ts +2 -0
- package/dist/server/settings.js +22 -2
- package/dist/server/tasks.js +4 -1
- package/dist/storage-sweep.js +1 -1
- package/dist/store.d.ts +125 -2
- package/dist/store.js +250 -12
- package/dist/surface.js +3 -1
- package/dist/teardown-git.js +5 -5
- package/dist/toolroll-update.js +2 -2
- package/dist/up-role-child.js +6 -2
- package/dist/workspace-revision.js +3 -1
- package/package.json +1 -1
package/dist/operate.js
CHANGED
|
@@ -44,6 +44,7 @@ import { createConnectionChecker } from "./provider-connection.js";
|
|
|
44
44
|
import { checkIntegrations, integrationsBrokenLine, integrationsNow, renderIntegrations } from "./integrations.js";
|
|
45
45
|
import { buildHandoff, handoffLines, loginAccount, runOnboard } from "./agent-onboard.js";
|
|
46
46
|
import { backupFiles, backupFolderOf, backupOwner, backupNow, restoreDatabase, startBackups } from "./backup.js";
|
|
47
|
+
import { startLiveBackups } from "./live-backup.js";
|
|
47
48
|
import { pushLimitSink } from "./provider-limits.js";
|
|
48
49
|
import { heuristicSizing, installFilingSizer, ownerSizer, refineFiledSizing, settleSizings } from "./task-sizing.js";
|
|
49
50
|
import { tierLines, tierReport } from "./tier-report.js";
|
|
@@ -57,7 +58,7 @@ import { readHooksBase, scheduleFromWords } from "./flow-triggers.js";
|
|
|
57
58
|
import { sendSubagentSummaries } from "./subagent-admin.js";
|
|
58
59
|
import { runRequestedUndos, sendSubagentWeeklies } from "./subagent-week.js";
|
|
59
60
|
import { refreshConnections } from "./mcp-connect.js";
|
|
60
|
-
import { releaseTermsFor } from "./release-carry.js";
|
|
61
|
+
import { releaseCheckPurposeFor, releaseTermsFor } from "./release-carry.js";
|
|
61
62
|
import { refreshTrustedMains, settleReleaseChecks } from "./nightly-check.js";
|
|
62
63
|
import { SECRETS_FIRST, startToolSecrets, toolSecretStorage } from "./project-tools.js";
|
|
63
64
|
import { followDiscord } from "./discord.js";
|
|
@@ -100,7 +101,7 @@ import { taskReviewBrief, renderReviewBrief } from "./task-review-brief.js";
|
|
|
100
101
|
import { homedir, hostname } from "node:os";
|
|
101
102
|
import { basename, dirname, join, resolve } from "node:path";
|
|
102
103
|
import { releaseStalledUpdate, waitingUpdate } from "./toolroll-update.js";
|
|
103
|
-
import { openStore, openStoreNoMigrate, CONCURRENT_WRITER_WAIT_MS, isDatabaseBusy, databasePath, BUILT_IN, parseCapabilityKey, verifiedAuthor, isDigestTime, predecessorWatches, RESULT_SCREENSHOTS, } from "./store.js";
|
|
104
|
+
import { openStore, openStoreNoMigrate, CONCURRENT_WRITER_WAIT_MS, isDatabaseBusy, databasePath, BUILT_IN, parseCapabilityKey, verifiedAuthor, isDigestTime, predecessorWatches, RESULT_SCREENSHOTS, lifecycleOf, reportLifecycle, beatLifecycle, } from "./store.js";
|
|
104
105
|
import { createHash, randomBytes, randomInt, randomUUID } from "node:crypto";
|
|
105
106
|
import { claimInstance, clearInstance, openInstanceSocket } from "./console-instance.js";
|
|
106
107
|
import { authorizePlanUnderMode } from "./plan-auto.js";
|
|
@@ -194,7 +195,9 @@ import { acquire, acquireIfReady, heartbeat, release, reap, currentClaim, DEFAUL
|
|
|
194
195
|
import { disposeBuildOutcome, holdStaleApproval, regateTask } from "./dispose.js";
|
|
195
196
|
import { proposeGrant, describeGrant, describeWithheld, permits, MUTATION_CLASSES, DEFAULT_MUTATIONS, } from "./grant.js";
|
|
196
197
|
import { builtIn, guarded } from "./backend.js";
|
|
197
|
-
import { register, authenticate, heartbeat as heartbeatRunner, isAlive, recoverDead, recoveredAnything, acquireWatchLeaseAuthed, heartbeatWatchLeaseAuthed, addRunnerReposAuthed, removeRunnerRepoAuthed, registerRunnerIfIdle, retireRunnerIfCurrent, normalizeRunnerName, validRunnerName, canonicalRepos, RUNNER_NAME_MAX, RUNNER_CAPACITY_MAX, parseRunnerCapacity, } from "./runner.js";
|
|
198
|
+
import { register, authenticate, heartbeat as heartbeatRunner, isAlive, recoverDead, recoveredAnything, acquireWatchLeaseAuthed, heartbeatWatchLeaseAuthed, addRunnerReposAuthed, removeRunnerRepoAuthed, registerRunnerIfIdle, retireRunnerIfCurrent, drainRunnerIfCurrent, RUNNER_HEARTBEAT_MS, MAX_WATCH_LEASE_MS, normalizeRunnerName, validRunnerName, canonicalRepos, RUNNER_NAME_MAX, RUNNER_CAPACITY_MAX, parseRunnerCapacity, hashToken as hashRunnerToken, } from "./runner.js";
|
|
199
|
+
import { credentialDigest, recordedHolderProvedGone, processGone, processStart, readRunnerProcessRecord, runnerProcessFile, sealRunnerProcessRecord, writeRunnerProcessRecord } from "./runner-process.js";
|
|
200
|
+
import { stopProcessTree } from "./process-tree.js";
|
|
198
201
|
import { mintCoordinator, revokeCoordinator, listCoordinators } from "./coordinator.js";
|
|
199
202
|
import { serveMcp } from "./mcp.js";
|
|
200
203
|
import { createInterface } from "node:readline";
|
|
@@ -1542,6 +1545,10 @@ function claimCommand(positional, flags, context) {
|
|
|
1542
1545
|
return fail(write, json, "claim", "coordinator-filed", "a coordinator filed this — nothing runs until the operator signs its scope", EXIT.refused);
|
|
1543
1546
|
if (result.reason === "sweep-limit")
|
|
1544
1547
|
return fail(write, json, "claim", "sweep-limit", `its sweep already has ${result.limit} running at once — it starts when one finishes`, EXIT.refused, { sweep: result.sweep, limit: result.limit });
|
|
1548
|
+
if (result.reason === "watch-custody")
|
|
1549
|
+
return fail(write, json, "claim", "watch-custody", "This worker no longer holds the project; restart the watch", EXIT.refused);
|
|
1550
|
+
if (result.reason === "draining")
|
|
1551
|
+
return fail(write, json, "claim", "draining", `Toolroll is restarting (draining): ${runner} takes no new work until it is back (by ${result.until})`, EXIT.refused, { until: result.until });
|
|
1545
1552
|
return fail(write, json, "claim", result.reason, `held by ${result.by} until ${result.until}`, EXIT.refused, { holder: result.by, until: result.until });
|
|
1546
1553
|
}
|
|
1547
1554
|
// Taking a task is what makes it running; leaving that to the caller would
|
|
@@ -3948,6 +3955,7 @@ async function startConsole(options) {
|
|
|
3948
3955
|
...(options.upConsole === undefined ? {} : { upConsole: options.upConsole }),
|
|
3949
3956
|
...(options.editorLinks === undefined ? {} : { editorLinks: options.editorLinks }),
|
|
3950
3957
|
...(options.instanceSecret === undefined ? {} : { instanceSecret: options.instanceSecret }),
|
|
3958
|
+
...(options.draining === undefined ? {} : { draining: options.draining }),
|
|
3951
3959
|
});
|
|
3952
3960
|
await new Promise((resolve, reject) => {
|
|
3953
3961
|
server.once("error", reject);
|
|
@@ -6004,8 +6012,10 @@ async function daemonCommand(positional, flags, context) {
|
|
|
6004
6012
|
return fail(write, json, "daemon", "usage", "`toolroll daemon [install|status|uninstall|logs]`", EXIT.usage);
|
|
6005
6013
|
}
|
|
6006
6014
|
// ---- the watch loop --------------------------------------------------------
|
|
6007
|
-
|
|
6008
|
-
|
|
6015
|
+
// A watch lease lasts at most a minute and is renewed every 15 s (docs/plans/actors-lessons.md): a crashed worker's
|
|
6016
|
+
// projects are free within a minute, and a clean stop hands them back at once (retireRunnerIfCurrent).
|
|
6017
|
+
const WATCH_LEASE_MS = MAX_WATCH_LEASE_MS;
|
|
6018
|
+
const WATCH_HEARTBEAT_MS = RUNNER_HEARTBEAT_MS;
|
|
6009
6019
|
/** The start repair runs once per process, not once per project loop. */
|
|
6010
6020
|
/**
|
|
6011
6021
|
* The Telegram follower: one actor on the wire that sends notifications and
|
|
@@ -7110,8 +7120,13 @@ async function upCommand(flags, context) {
|
|
|
7110
7120
|
probe.once("error", () => resolveProbe(false));
|
|
7111
7121
|
probe.listen(port, "127.0.0.1", () => resolveProbe(true));
|
|
7112
7122
|
});
|
|
7113
|
-
if (!reserved)
|
|
7123
|
+
if (!reserved) {
|
|
7124
|
+
// The console that holds it said it is stopping: a restart is under way, so say that instead of "another one runs".
|
|
7125
|
+
const holder = lifecycleOf(store, `console:${port}`);
|
|
7126
|
+
if (holder?.state === "draining")
|
|
7127
|
+
return fail(write, json, "up", "port-busy", `Toolroll is restarting (draining): the console on port ${port} is stopping — try again in a few seconds`, EXIT.refused);
|
|
7114
7128
|
return fail(write, json, "up", "port-busy", `port ${port} is taken — another \`up\` or \`serve\` may already be running; stop it, or pick a different --port`, EXIT.refused);
|
|
7129
|
+
}
|
|
7115
7130
|
// Under a coding agent nothing is asked and no secret is printed, even in a real terminal: what is shown lands in
|
|
7116
7131
|
// its transcript.
|
|
7117
7132
|
const agentRunning = underAgent(context.upSeams?.env ?? process.env);
|
|
@@ -7143,21 +7158,47 @@ async function upCommand(flags, context) {
|
|
|
7143
7158
|
// The runner's authority is BOUND to the canonical roots this up
|
|
7144
7159
|
// serves (MCP spec v6): the claim gate enforces membership, so an
|
|
7145
7160
|
// unbound registration would deny-all its own dispatches.
|
|
7146
|
-
|
|
7161
|
+
// A holder whose every process is proved gone (its record beside the database) no longer blocks the name: an
|
|
7162
|
+
// `up` killed outright can't leave a fresh heartbeat in the way of the next start (the 2026-10-29 deploy).
|
|
7163
|
+
const tryRegister = () => registerRunnerIfIdle(store, { name, host: hostname(), repos, now: clock(),
|
|
7164
|
+
previousGone: holder => recordedHolderProvedGone(runnerProcessFile(context.databaseFile, name), holder, hostname()) });
|
|
7165
|
+
let answer = tryRegister();
|
|
7166
|
+
// Wait for a draining holder to sign off or be proved gone, never just for its deadline to elapse.
|
|
7167
|
+
const drainWaitUntil = Date.now() + 2 * 60_000;
|
|
7168
|
+
let saidDraining = false;
|
|
7169
|
+
while (!answer.ok && answer.reason === "runner-draining" && Date.now() < drainWaitUntil) {
|
|
7170
|
+
if (!saidDraining)
|
|
7171
|
+
progress(`${answer.detail}; waiting for it to stop`);
|
|
7172
|
+
saidDraining = true;
|
|
7173
|
+
await new Promise(resolveWait => setTimeout(resolveWait, 1_000));
|
|
7174
|
+
answer = tryRegister();
|
|
7175
|
+
}
|
|
7147
7176
|
if (answer.ok) {
|
|
7148
7177
|
runnerName = name;
|
|
7149
7178
|
runnerToken = answer.token;
|
|
7179
|
+
if (answer.signedOff !== null)
|
|
7180
|
+
progress(`signed off the previous ${name}: its processes had already exited`);
|
|
7150
7181
|
if (answer.recoveredRuns > 0)
|
|
7151
7182
|
progress(`recovered ${answer.recoveredRuns} interrupted attempt(s) left by the previous ${name}`);
|
|
7183
|
+
// Every process given this credential is recorded (pid and start time) before it gets it: this one now, each
|
|
7184
|
+
// role child as it is spawned. A later start proves them gone by this record.
|
|
7185
|
+
if (!recordRunnerProcess(context.databaseFile, { runner: answer.runner, credentialHash: hashRunnerToken(answer.token) }, "up", process.pid)) {
|
|
7186
|
+
retireRunnerIfCurrent(store, runnerName, runnerToken, clock());
|
|
7187
|
+
return fail(write, json, "up", "runner-alive", "The worker process record could not be saved; nothing started", EXIT.refused);
|
|
7188
|
+
}
|
|
7152
7189
|
doorAnswer = answer;
|
|
7153
7190
|
break;
|
|
7154
7191
|
}
|
|
7155
7192
|
doorAnswer = answer;
|
|
7156
7193
|
if (runnerFlag !== undefined)
|
|
7157
7194
|
break; // an explicit name is never suffixed around
|
|
7195
|
+
if (answer.reason === "runner-draining")
|
|
7196
|
+
break; // a restart under way keeps its name
|
|
7158
7197
|
}
|
|
7159
7198
|
if (runnerName === "") {
|
|
7160
7199
|
const detail = doorAnswer !== null && !doorAnswer.ok ? doorAnswer.detail : "no worker name could be taken";
|
|
7200
|
+
if (doorAnswer !== null && !doorAnswer.ok && doorAnswer.reason === "runner-draining")
|
|
7201
|
+
return fail(write, json, "up", "runner-alive", `${detail}, and it hasn't yet — try again shortly`, EXIT.refused);
|
|
7161
7202
|
return fail(write, json, "up", "runner-alive", `${detail} — that worker looks alive; stop the other \`up\` or watch, or name a different worker with --runner`, EXIT.refused);
|
|
7162
7203
|
}
|
|
7163
7204
|
// 6. Enrollment: repositories and project roots are one atomic machine
|
|
@@ -7203,6 +7244,13 @@ async function upCommand(flags, context) {
|
|
|
7203
7244
|
let currentRepos = [...repos];
|
|
7204
7245
|
let consoleRole = null;
|
|
7205
7246
|
let workerRole = null;
|
|
7247
|
+
// A stop that can't finish (a role that ignores its stop, a drain that hangs) still signs off: past the worker's own
|
|
7248
|
+
// drain bound and a margin, both roles' process trees are stopped outright, the runner is retired, and `up` ends. A
|
|
7249
|
+
// service's launchd window (ExitTimeOut 60 s, controller-service's 55 s) is longer than this at the default grace.
|
|
7250
|
+
let resolveForced = () => { };
|
|
7251
|
+
const forced = new Promise(resolveForce => (resolveForced = resolveForce));
|
|
7252
|
+
let forceTimer;
|
|
7253
|
+
const stopDeadlineMs = Math.max(1, stopGraceMs) + 20_000;
|
|
7206
7254
|
const stop = () => {
|
|
7207
7255
|
if (stopping) {
|
|
7208
7256
|
workerRole?.hardStop();
|
|
@@ -7210,6 +7258,20 @@ async function upCommand(flags, context) {
|
|
|
7210
7258
|
}
|
|
7211
7259
|
stopping = true;
|
|
7212
7260
|
resolveStopped();
|
|
7261
|
+
// Draining, durably: no new work is admitted for this runner from now, and a restart waits for this drain's end
|
|
7262
|
+
// (now + the forced-stop bound) instead of a heartbeat window.
|
|
7263
|
+
try {
|
|
7264
|
+
if (runnerName !== "")
|
|
7265
|
+
drainRunnerIfCurrent(store, runnerName, runnerToken, clock(), new Date(clock().getTime() + stopDeadlineMs));
|
|
7266
|
+
}
|
|
7267
|
+
catch { /* the heartbeat bound still holds */ }
|
|
7268
|
+
forceTimer = setTimeout(() => {
|
|
7269
|
+
fatal ??= `stopping took longer than ${Math.round(stopDeadlineMs / 1000)} s; its console and worker were stopped outright`;
|
|
7270
|
+
workerRole?.kill();
|
|
7271
|
+
consoleRole?.kill();
|
|
7272
|
+
resolveForced();
|
|
7273
|
+
}, stopDeadlineMs);
|
|
7274
|
+
forceTimer.unref?.();
|
|
7213
7275
|
// The coding owner record is released first, whatever else is still finishing: launchd kills a service that
|
|
7214
7276
|
// outlasts its exit window, and a killed process cannot release it. The console keeps serving until the end.
|
|
7215
7277
|
consoleRole?.send({ type: "stopping" });
|
|
@@ -7217,12 +7279,34 @@ async function upCommand(flags, context) {
|
|
|
7217
7279
|
};
|
|
7218
7280
|
process.on("SIGINT", stop);
|
|
7219
7281
|
process.on("SIGTERM", stop);
|
|
7282
|
+
// The process that started `up` gone (its service supervisor killed, or the IPC parent that ran it): nobody is left to
|
|
7283
|
+
// forward a stop, so `up` stops itself — it drains, retires its runner and ends instead of running on as an orphan.
|
|
7284
|
+
let parentGone = false;
|
|
7285
|
+
const parentAtStart = process.ppid;
|
|
7286
|
+
const parentWatch = context.upSeams === undefined && parentAtStart > 1 ? setInterval(() => {
|
|
7287
|
+
if (process.ppid === parentAtStart || parentGone)
|
|
7288
|
+
return;
|
|
7289
|
+
parentGone = true;
|
|
7290
|
+
progress("the process that started Toolroll is gone; stopping");
|
|
7291
|
+
stop();
|
|
7292
|
+
}, 1_000) : undefined;
|
|
7293
|
+
parentWatch?.unref?.();
|
|
7294
|
+
const onDisconnect = () => { if (!parentGone) {
|
|
7295
|
+
parentGone = true;
|
|
7296
|
+
progress("the process that started Toolroll closed its channel; stopping");
|
|
7297
|
+
stop();
|
|
7298
|
+
} };
|
|
7299
|
+
if (context.upSeams === undefined && process.channel !== undefined && process.channel !== null)
|
|
7300
|
+
process.once("disconnect", onDisconnect);
|
|
7220
7301
|
// A log (a service, an agent, --json or --verbose) says whose each line is; a person at a terminal reads plain lines.
|
|
7221
7302
|
const labelled = split && (json || flags.has("verbose") || !(context.upSeams?.terminal ?? process.stdout.isTTY === true));
|
|
7222
7303
|
const roleLine = (role) => (line) => progress(labelled ? `[${role}] ${line}` : line);
|
|
7223
7304
|
const dropSignals = () => {
|
|
7224
7305
|
process.removeListener("SIGINT", stop);
|
|
7225
7306
|
process.removeListener("SIGTERM", stop);
|
|
7307
|
+
process.removeListener("disconnect", onDisconnect);
|
|
7308
|
+
if (parentWatch !== undefined)
|
|
7309
|
+
clearInterval(parentWatch);
|
|
7226
7310
|
};
|
|
7227
7311
|
const consoleConfig = {
|
|
7228
7312
|
host: hostFlag, port, allowedHosts: allowFlag === undefined ? null : allowFlag.split(",").map(one => one.trim()).filter(one => one !== ""),
|
|
@@ -7230,8 +7314,12 @@ async function upCommand(flags, context) {
|
|
|
7230
7314
|
};
|
|
7231
7315
|
const pendingLinks = new Map();
|
|
7232
7316
|
let linkRequests = 0;
|
|
7317
|
+
const roleSpawned = (role, pid) => {
|
|
7318
|
+
const held = store.getRunner(runnerName);
|
|
7319
|
+
return held !== null && held.credentialHash === hashRunnerToken(runnerToken) && recordRunnerProcess(context.databaseFile, held, role, pid);
|
|
7320
|
+
};
|
|
7233
7321
|
consoleRole = startUpRole({
|
|
7234
|
-
role: "console", split, context, build, shutdownMs: 15_000,
|
|
7322
|
+
role: "console", split, context, build, shutdownMs: 15_000, onSpawned: pid => roleSpawned("console", pid),
|
|
7235
7323
|
start: () => ({ type: "start", role: "console", databaseFile: context.databaseFile, containment: containmentFlag, build, config: { ...consoleConfig, repos: [...currentRepos] } }),
|
|
7236
7324
|
onMessage: message => {
|
|
7237
7325
|
if (message.type === "log")
|
|
@@ -7266,7 +7354,7 @@ async function upCommand(flags, context) {
|
|
|
7266
7354
|
const url = consoleReady.url;
|
|
7267
7355
|
if (!stopping) {
|
|
7268
7356
|
workerRole = startUpRole({
|
|
7269
|
-
role: "worker", split, context, build,
|
|
7357
|
+
role: "worker", split, context, build, onSpawned: pid => roleSpawned("worker", pid),
|
|
7270
7358
|
// The worker drains its builds for the stop grace, then stops them; past that it is stopped outright.
|
|
7271
7359
|
shutdownMs: Math.max(1, stopGraceMs) + 15_000,
|
|
7272
7360
|
start: generation => ({
|
|
@@ -7275,7 +7363,7 @@ async function upCommand(flags, context) {
|
|
|
7275
7363
|
runner: runnerName, token: runnerToken, repos: generation === 1 ? [...repos] : [...currentRepos], pinnedRepos: [...pinnedRepos], projectRoots, registryPath, pool,
|
|
7276
7364
|
forMs: forGiven === undefined ? null : Number(forGiven), publicUrl: publicUrlFlag ?? null, stopGraceMs, verbose: flags.has("verbose"),
|
|
7277
7365
|
terminal: context.upSeams?.terminal ?? process.stdout.isTTY === true,
|
|
7278
|
-
// A restart waits out a crashed predecessor's lease (
|
|
7366
|
+
// A restart waits out a crashed predecessor's lease (at most a minute), then recovers it: as a fresh `up` after a crash.
|
|
7279
7367
|
leaseWaitMs: split ? WATCH_LEASE_MS + 15_000 : 0,
|
|
7280
7368
|
},
|
|
7281
7369
|
}),
|
|
@@ -7319,7 +7407,7 @@ async function upCommand(flags, context) {
|
|
|
7319
7407
|
fatal = `the builder identity changed (${beat.reason})`;
|
|
7320
7408
|
stop();
|
|
7321
7409
|
}
|
|
7322
|
-
},
|
|
7410
|
+
}, RUNNER_HEARTBEAT_MS);
|
|
7323
7411
|
runnerHeartbeat.unref?.();
|
|
7324
7412
|
// 9. Readiness, then the ONE startup envelope / greeting (finding 11/20).
|
|
7325
7413
|
const approverPlan2 = approver;
|
|
@@ -7391,14 +7479,19 @@ async function upCommand(flags, context) {
|
|
|
7391
7479
|
// only then does the console stop serving.
|
|
7392
7480
|
await stopped;
|
|
7393
7481
|
if (workerRole !== null)
|
|
7394
|
-
await workerRole.done;
|
|
7482
|
+
await Promise.race([workerRole.done, forced]);
|
|
7395
7483
|
clearInterval(runnerHeartbeat);
|
|
7396
7484
|
if (runTimer !== undefined)
|
|
7397
7485
|
clearTimeout(runTimer);
|
|
7398
7486
|
dropSignals();
|
|
7399
7487
|
retireRunnerIfCurrent(store, runnerName, runnerToken, clock());
|
|
7400
7488
|
consoleRole.stop();
|
|
7401
|
-
const closed = await consoleRole.done;
|
|
7489
|
+
const closed = await Promise.race([consoleRole.done, forced.then(() => ({ problem: null }))]);
|
|
7490
|
+
if (forceTimer !== undefined)
|
|
7491
|
+
clearTimeout(forceTimer);
|
|
7492
|
+
// With nobody left to wait for it, `up` ends once it has signed off, whatever handle lingers.
|
|
7493
|
+
if (parentGone)
|
|
7494
|
+
setTimeout(() => process.exit(fatal === null ? EXIT.ok : EXIT.failed), 1_000).unref?.();
|
|
7402
7495
|
// A live terminal that couldn't be proved stopped (or any other close failure) is said, never "stopped".
|
|
7403
7496
|
if (closed.problem !== null && fatal === null)
|
|
7404
7497
|
fatal = closed.problem;
|
|
@@ -9596,6 +9689,7 @@ async function scopeTask(positional, flags, context) {
|
|
|
9596
9689
|
// sandbox, say) file the scope without them: every later commit then asks again, as before.
|
|
9597
9690
|
const releaseGiven = text(flags, "release");
|
|
9598
9691
|
let releaseTerms;
|
|
9692
|
+
let releaseCheck;
|
|
9599
9693
|
let releaseNote = null;
|
|
9600
9694
|
if (releaseGiven !== undefined) {
|
|
9601
9695
|
const spec = /^(\d+\.\d+\.\d+):([1-9][0-9]*):([A-Za-z0-9._/-]+)$/.exec(releaseGiven);
|
|
@@ -9607,8 +9701,15 @@ async function scopeTask(positional, flags, context) {
|
|
|
9607
9701
|
const made = releaseTermsFor(store, { repo, version: spec[1], pr: Number(spec[2]), branch: spec[3], candidate: candidateGiven, now, noCarry: flags.get("no-carry") === true });
|
|
9608
9702
|
if (made.ok)
|
|
9609
9703
|
releaseTerms = made.terms;
|
|
9610
|
-
else
|
|
9704
|
+
else {
|
|
9611
9705
|
releaseNote = `Later commits on this release will each ask for approval: ${made.why}.`;
|
|
9706
|
+
// Without terms the candidate is still a release's check (pipeline-speed.md, P0): its purpose is sealed with the
|
|
9707
|
+
// scope, from the plane's own records, so the check runs under the release's rules and the plane may rerun it on
|
|
9708
|
+
// this same candidate after confirmed infrastructure. It never carries a later commit.
|
|
9709
|
+
const purpose = releaseCheckPurposeFor(store, { repo, version: spec[1], pr: Number(spec[2]), branch: spec[3], candidate: candidateGiven });
|
|
9710
|
+
if (purpose.ok)
|
|
9711
|
+
releaseCheck = purpose.purpose;
|
|
9712
|
+
}
|
|
9612
9713
|
}
|
|
9613
9714
|
if (acceptanceParse.criteria.length === 0)
|
|
9614
9715
|
return fail(write, json, "task scope", "acceptance-required", "--acceptance named no valid criteria", EXIT.usage);
|
|
@@ -9676,7 +9777,8 @@ async function scopeTask(positional, flags, context) {
|
|
|
9676
9777
|
: null;
|
|
9677
9778
|
const proposed = propose(store, {
|
|
9678
9779
|
...(candidateGiven === undefined ? {} : { candidate: candidateGiven }),
|
|
9679
|
-
...(releaseTerms === undefined ? {} : { releaseTerms }),
|
|
9780
|
+
...(releaseTerms === undefined ? {} : { releaseTerms, releaseCheck: null }),
|
|
9781
|
+
...(releaseCheck === undefined ? {} : { releaseCheck, releaseTerms: null }),
|
|
9680
9782
|
taskId: id,
|
|
9681
9783
|
goal,
|
|
9682
9784
|
outOfScope: text(flags, "not") ?? null,
|
|
@@ -11604,6 +11706,35 @@ function upNeedsOneProcess(options) {
|
|
|
11604
11706
|
options.now !== undefined;
|
|
11605
11707
|
}
|
|
11606
11708
|
/** The entry a role child runs: the built sibling when installed, the TypeScript source through tsx in a checkout. */
|
|
11709
|
+
/**
|
|
11710
|
+
* Add one process to the record of who holds this runner's credential (runner-process.ts), replacing that role's
|
|
11711
|
+
* earlier processes once they are gone. A complete authenticated record is required before delivering credentials.
|
|
11712
|
+
*/
|
|
11713
|
+
function recordRunnerProcess(databaseFile, holder, role, pid) {
|
|
11714
|
+
try {
|
|
11715
|
+
const started = processStart(pid);
|
|
11716
|
+
if (started === null)
|
|
11717
|
+
return false;
|
|
11718
|
+
const file = runnerProcessFile(databaseFile, holder.runner.name);
|
|
11719
|
+
const credential = credentialDigest(holder.credentialHash);
|
|
11720
|
+
const current = readRunnerProcessRecord(file);
|
|
11721
|
+
const same = current !== null && current.runner === holder.runner.name && current.registeredAt === holder.runner.registeredAt && current.credential === credential;
|
|
11722
|
+
// Never bless a tampered or incomplete earlier list while appending a credential holder.
|
|
11723
|
+
if (role !== "up" && (!same || current.signature !== sealRunnerProcessRecord(current, holder.credentialHash).signature))
|
|
11724
|
+
return false;
|
|
11725
|
+
// Keep every possibly live generation; truncating this list would erase custody.
|
|
11726
|
+
const kept = same ? current.processes.filter(one => one.role !== role || !processGone(one)) : [];
|
|
11727
|
+
const record = { v: 1, runner: holder.runner.name, host: holder.runner.host, registeredAt: holder.runner.registeredAt, credential,
|
|
11728
|
+
processes: [...kept.filter(one => one.pid !== pid), { pid, role, started }] };
|
|
11729
|
+
if (record.processes.length > 64)
|
|
11730
|
+
return false;
|
|
11731
|
+
writeRunnerProcessRecord(file, sealRunnerProcessRecord(record, holder.credentialHash));
|
|
11732
|
+
return true;
|
|
11733
|
+
}
|
|
11734
|
+
catch {
|
|
11735
|
+
return false;
|
|
11736
|
+
}
|
|
11737
|
+
}
|
|
11607
11738
|
function upRoleChildCommand(role) {
|
|
11608
11739
|
const source = import.meta.url.endsWith(".ts");
|
|
11609
11740
|
const entry = fileURLToPath(new URL(source ? "./up-role-child.ts" : "./up-role-child.js", import.meta.url));
|
|
@@ -11693,6 +11824,7 @@ function startUpRole(args) {
|
|
|
11693
11824
|
send: message => channel.parent.send(message),
|
|
11694
11825
|
stop: () => { stopping = true; channel.parent.stop(false); },
|
|
11695
11826
|
hardStop: () => { channel.parent.stop(true); },
|
|
11827
|
+
kill: () => { stopping = true; channel.parent.stop(true); },
|
|
11696
11828
|
pid: () => livePid,
|
|
11697
11829
|
};
|
|
11698
11830
|
}
|
|
@@ -11710,6 +11842,20 @@ function startUpRole(args) {
|
|
|
11710
11842
|
generations.spawned();
|
|
11711
11843
|
owned.on("message", raw => receive(raw, () => controller.abort()));
|
|
11712
11844
|
owned.on("error", () => { });
|
|
11845
|
+
let recorded = false;
|
|
11846
|
+
try {
|
|
11847
|
+
recorded = owned.pid !== undefined && (args.onSpawned?.(owned.pid) ?? true);
|
|
11848
|
+
}
|
|
11849
|
+
catch { /* refuse credential delivery */ }
|
|
11850
|
+
if (!recorded) {
|
|
11851
|
+
fatal = "The worker process record could not be saved";
|
|
11852
|
+
failed = { reason: "start", detail: fatal };
|
|
11853
|
+
generations.refused(fatal);
|
|
11854
|
+
owned.kill("SIGKILL");
|
|
11855
|
+
controller.abort();
|
|
11856
|
+
resolveReady({ ok: false, ...failed });
|
|
11857
|
+
return;
|
|
11858
|
+
}
|
|
11713
11859
|
// The start, including the runner token for the worker, goes over the private channel only.
|
|
11714
11860
|
owned.send(args.start(generation), error => { if (error !== null && error !== undefined)
|
|
11715
11861
|
owned.kill("SIGTERM"); });
|
|
@@ -11775,6 +11921,14 @@ function startUpRole(args) {
|
|
|
11775
11921
|
catch { /* ending */ }
|
|
11776
11922
|
}
|
|
11777
11923
|
},
|
|
11924
|
+
kill: () => {
|
|
11925
|
+
stopping = true;
|
|
11926
|
+
controller.abort();
|
|
11927
|
+
const owned = child;
|
|
11928
|
+
// Only the still-owned ChildProcess is signalled, never a recorded pid.
|
|
11929
|
+
if (owned !== null && owned.exitCode === null && owned.signalCode === null && !stopProcessTree(owned))
|
|
11930
|
+
owned.kill("SIGKILL");
|
|
11931
|
+
},
|
|
11778
11932
|
pid: () => livePid,
|
|
11779
11933
|
};
|
|
11780
11934
|
}
|
|
@@ -11805,10 +11959,14 @@ async function runUpConsoleRole(context, config, port, build) {
|
|
|
11805
11959
|
const activeRepos = new Set(config.repos);
|
|
11806
11960
|
// Cockpit B1: this start's instance secret, minted once it is listening; the console answers its challenges with it.
|
|
11807
11961
|
let instanceSecret = null;
|
|
11962
|
+
// v137: this console's lifecycle (service_lifecycle "console:<port>"): started, a short heartbeat, draining once
|
|
11963
|
+
// its `up` stops, stopped at the end. While draining, /healthz and every page say Toolroll is restarting.
|
|
11964
|
+
let draining = false;
|
|
11808
11965
|
let console_;
|
|
11809
11966
|
try {
|
|
11810
11967
|
console_ = await startConsole({
|
|
11811
11968
|
instanceSecret: () => instanceSecret,
|
|
11969
|
+
draining: () => draining,
|
|
11812
11970
|
context,
|
|
11813
11971
|
host: config.host,
|
|
11814
11972
|
port: config.port,
|
|
@@ -11839,14 +11997,31 @@ async function runUpConsoleRole(context, config, port, build) {
|
|
|
11839
11997
|
let resolveStopped = () => { };
|
|
11840
11998
|
const stopped = new Promise(resolveStop => (resolveStopped = resolveStop));
|
|
11841
11999
|
port.onStop(() => resolveStopped());
|
|
12000
|
+
const lifecycleComponent = `console:${console_.port}`;
|
|
12001
|
+
const lifecycleHolder = `${process.pid}:${randomUUID().slice(0, 8)}`;
|
|
12002
|
+
try {
|
|
12003
|
+
reportLifecycle(store, lifecycleComponent, "started", lifecycleHolder, clock(), { pid: process.pid });
|
|
12004
|
+
}
|
|
12005
|
+
catch { /* reporting never stops the console */ }
|
|
12006
|
+
const lifecycleBeat = setInterval(() => { try {
|
|
12007
|
+
beatLifecycle(store, lifecycleComponent, lifecycleHolder, clock());
|
|
12008
|
+
}
|
|
12009
|
+
catch { /* the next beat tries again */ } }, RUNNER_HEARTBEAT_MS);
|
|
12010
|
+
lifecycleBeat.unref?.();
|
|
11842
12011
|
port.onMessage(message => {
|
|
11843
12012
|
if (message.type === "repos") {
|
|
11844
12013
|
activeRepos.clear();
|
|
11845
12014
|
for (const repo of message.repos)
|
|
11846
12015
|
activeRepos.add(repo);
|
|
11847
12016
|
}
|
|
11848
|
-
else if (message.type === "stopping")
|
|
12017
|
+
else if (message.type === "stopping") {
|
|
12018
|
+
draining = true;
|
|
12019
|
+
try {
|
|
12020
|
+
reportLifecycle(store, lifecycleComponent, "draining", lifecycleHolder, clock(), { drainUntil: new Date(clock().getTime() + 60_000) });
|
|
12021
|
+
}
|
|
12022
|
+
catch { /* the flag above still answers */ }
|
|
11849
12023
|
closeCoding();
|
|
12024
|
+
}
|
|
11850
12025
|
else if (message.type === "sign-in-link") {
|
|
11851
12026
|
let link = null;
|
|
11852
12027
|
try {
|
|
@@ -11874,13 +12049,19 @@ async function runUpConsoleRole(context, config, port, build) {
|
|
|
11874
12049
|
progress(`attach: ${instanceSocket.problem} — \`toolroll\` in a repository won't find this console`);
|
|
11875
12050
|
port.send({ type: "ready", role: "console", build, pid: process.pid, url: console_.url, port: console_.port });
|
|
11876
12051
|
await stopped;
|
|
12052
|
+
draining = true;
|
|
11877
12053
|
closeCoding();
|
|
12054
|
+
clearInterval(lifecycleBeat);
|
|
11878
12055
|
if (instanceSocket.ok)
|
|
11879
12056
|
instanceSocket.close();
|
|
11880
12057
|
if (mintedSecret !== null)
|
|
11881
12058
|
clearInstance(store, mintedSecret);
|
|
11882
12059
|
let closeProblem = null;
|
|
11883
12060
|
await new Promise(done => console_.server.close(error => { closeProblem = error === undefined || error === null ? null : error.message; done(); }));
|
|
12061
|
+
try {
|
|
12062
|
+
reportLifecycle(store, lifecycleComponent, "stopped", lifecycleHolder, clock());
|
|
12063
|
+
}
|
|
12064
|
+
catch { /* a stale row lapses with its heartbeat */ }
|
|
11884
12065
|
port.send({ type: "closed", problem: closeProblem });
|
|
11885
12066
|
return closeProblem === null ? EXIT.ok : EXIT.failed;
|
|
11886
12067
|
}
|
|
@@ -12165,12 +12346,18 @@ async function runUpWorkerRole(context, config, port, build) {
|
|
|
12165
12346
|
}
|
|
12166
12347
|
catch { /* A display fact; never blocks the worker. */ }
|
|
12167
12348
|
recordRunnerVersion(dirname(context.databaseFile), runnerName, PACKAGE_VERSION, clock());
|
|
12349
|
+
// v137: the continuous database copy, off the hot path (live-backup.ts: a worker thread copies, checks and hashes).
|
|
12350
|
+
// Never inside a project or a build checkout. A test's injected agent runner means a test: no background copies.
|
|
12351
|
+
const stopLiveBackups = context.agentRunner === undefined
|
|
12352
|
+
? startLiveBackups(store, context.databaseFile, () => [...new Set([...activeRepos, ...store.knownRepos(), pool])])
|
|
12353
|
+
: null;
|
|
12168
12354
|
// Ready once every project's loop holds its lease; a loop that failed to start ends this worker instead.
|
|
12169
12355
|
if (fatal === null && !stopping)
|
|
12170
12356
|
port.send({ type: "ready", role: "worker", build, pid: process.pid, url: null, port: null });
|
|
12171
12357
|
await stopped;
|
|
12172
12358
|
await registrySupervisor;
|
|
12173
12359
|
await Promise.all([...loopResults.values()]);
|
|
12360
|
+
await stopLiveBackups?.();
|
|
12174
12361
|
// The follower was aborted with the stop. A send or a chat turn it is still finishing keeps its own durable record;
|
|
12175
12362
|
// the service does not wait past this for it.
|
|
12176
12363
|
await new Promise(done => {
|
package/dist/planner.js
CHANGED
|
@@ -23,7 +23,7 @@ import { run } from "./exec.js";
|
|
|
23
23
|
import { checkoutGit } from "./teardown-git.js";
|
|
24
24
|
import { auditOf, isProviderId } from "./provider.js";
|
|
25
25
|
import { currentClaim, heartbeat } from "./claim.js";
|
|
26
|
-
import { heartbeat as runnerHeartbeat } from "./runner.js";
|
|
26
|
+
import { heartbeat as runnerHeartbeat, RUNNER_HEARTBEAT_MS } from "./runner.js";
|
|
27
27
|
import { parseDecision } from "./decision.js";
|
|
28
28
|
import { parsePlan } from "./plan.js";
|
|
29
29
|
import { PLAN_MODEL_SCHEMA } from "./contracts/plan.js";
|
|
@@ -44,7 +44,7 @@ const GIT = "git";
|
|
|
44
44
|
const AGENT_ENV_DENYLIST = [...TELEGRAM_TOKEN_ENVS];
|
|
45
45
|
const DEFAULT_PLAN_TIMEOUT_MS = 20 * 60_000;
|
|
46
46
|
const DEFAULT_PLAN_TURNS = CLAUDE_LIMITS.maxTurns;
|
|
47
|
-
const DEFAULT_PULSE_MS =
|
|
47
|
+
const DEFAULT_PULSE_MS = RUNNER_HEARTBEAT_MS;
|
|
48
48
|
/**
|
|
49
49
|
* One line of untrusted text made inert for the planner's brief (Codex
|
|
50
50
|
* M5-M8 audit, IV-4): controls and separators collapse, protocol-shaped
|
|
@@ -70,6 +70,33 @@ export type AdapterOptions = {
|
|
|
70
70
|
export declare const BRANCH_GONE_LOOKS = 5;
|
|
71
71
|
export declare const BRANCH_GONE_PAUSE_MS = 2000;
|
|
72
72
|
export declare function releaseAdapters(options: AdapterOptions): ReleaseAdapters;
|
|
73
|
+
/**
|
|
74
|
+
* Why a tap change isn't exactly the formula bump this release proposed (security scan), or null when it is: the tap's
|
|
75
|
+
* own open pull request number `pr` at exactly `head`, from its version branch into the default branch; compared with
|
|
76
|
+
* the default branch's head it is one commit on top of exactly that head, changing exactly one file, the formula; and
|
|
77
|
+
* the formula there is exactly today's with npm's published tarball and that tarball's own sha256.
|
|
78
|
+
*/
|
|
79
|
+
export declare function tapChangeProblem(input: {
|
|
80
|
+
tap: string;
|
|
81
|
+
pr: number;
|
|
82
|
+
head: string;
|
|
83
|
+
version: string;
|
|
84
|
+
formula: string;
|
|
85
|
+
defaultBranch: string;
|
|
86
|
+
baseHead: string;
|
|
87
|
+
baseVersion: string | null;
|
|
88
|
+
view: Json;
|
|
89
|
+
compare: Json;
|
|
90
|
+
expected: string;
|
|
91
|
+
atHead: string;
|
|
92
|
+
sha256: string;
|
|
93
|
+
}): string | null;
|
|
94
|
+
/** Whether a pull request's head is a branch of the tap itself (never a fork's). */
|
|
95
|
+
export declare function tapOwnHead(pr: Json, tap: string): boolean;
|
|
96
|
+
/** The version a formula's tarball url names (`…/toolroll-<x.y.z>.tgz`), or null. */
|
|
97
|
+
export declare function formulaVersion(text: string): string | null;
|
|
98
|
+
/** Whether version `a` comes after `b` (both x.y.z). */
|
|
99
|
+
export declare function versionAfter(a: string, b: string): boolean;
|
|
73
100
|
/**
|
|
74
101
|
* Whether a workflow's triggers (its top-level `on:` as a name, a list or a map) include a change to the repository.
|
|
75
102
|
* A file that doesn't parse, or whose triggers aren't one of those shapes, counts as running, so the release waits
|
|
@@ -103,7 +130,11 @@ export type AutoObservers = {
|
|
|
103
130
|
* lists only the latest of each), every page — with its id, app, check suite and the commit it is filed under, oldest
|
|
104
131
|
* first. A row that doesn't parse stops the read: a partial answer is never taken for the whole.
|
|
105
132
|
*/
|
|
106
|
-
export declare function readCheckRuns(run: Exec, gh: string, repo: string, sha: string
|
|
133
|
+
export declare function readCheckRuns(run: Exec, gh: string, repo: string, sha: string,
|
|
134
|
+
/** A release tag (pipeline-speed.md, item 3): a push of exactly this tag binds its workflow file too. */
|
|
135
|
+
options?: {
|
|
136
|
+
tag?: string;
|
|
137
|
+
}): Promise<CheckRun[]>;
|
|
107
138
|
/** One job's step outcomes as GitHub's Actions API reports them. */
|
|
108
139
|
export type JobRecord = {
|
|
109
140
|
id: number;
|