toolroll 0.9.76 → 0.9.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +273 -0
- package/README.md +11 -4
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/agent-fence.js +46 -10
- package/dist/assignment.js +3 -0
- package/dist/backup-ui.d.ts +9 -1
- package/dist/backup-ui.js +28 -3
- package/dist/backup.js +4 -3
- package/dist/browser/THIRD_PARTY_NOTICES.txt +5 -5
- package/dist/browser/workspace.css +2 -2
- package/dist/browser/workspace.js +31 -31
- package/dist/browser-workspace.d.ts +2 -0
- package/dist/builder.d.ts +18 -3
- package/dist/builder.js +152 -47
- package/dist/chat-decide.js +4 -0
- package/dist/check-failure.d.ts +92 -0
- package/dist/check-failure.js +295 -0
- package/dist/check-progress.d.ts +28 -1
- package/dist/check-progress.js +105 -6
- package/dist/check-report.d.ts +67 -6
- package/dist/check-report.js +254 -9
- package/dist/checkout-cleanup.js +2 -2
- package/dist/claim.d.ts +9 -0
- package/dist/claim.js +9 -1
- package/dist/cli.js +8 -5
- package/dist/coding-handoff.d.ts +1 -0
- package/dist/coding-handoff.js +20 -11
- package/dist/commit-extract.d.ts +6 -0
- package/dist/commit-extract.js +63 -8
- package/dist/console-chrome.d.ts +41 -0
- package/dist/console-chrome.js +166 -0
- package/dist/contracts/console-api.d.ts +1 -1
- package/dist/contracts/console-api.js +1 -1
- package/dist/contracts/release-check.d.ts +12 -0
- package/dist/contracts/release-check.js +17 -0
- package/dist/contracts/release-terms.d.ts +41 -0
- package/dist/contracts/release-terms.js +53 -0
- package/dist/contracts/verification-receipt.d.ts +20 -2
- package/dist/contracts/verification-receipt.js +15 -3
- package/dist/contracts/verification-view.d.ts +8 -0
- package/dist/controller-service.js +4 -1
- package/dist/controller-supervisor.d.ts +8 -0
- package/dist/controller-supervisor.js +33 -1
- package/dist/demo.js +3 -2
- package/dist/deploy-gate.js +5 -0
- package/dist/deploy-rescue.d.ts +41 -0
- package/dist/deploy-rescue.js +94 -0
- package/dist/desktop-update.d.ts +1 -1
- package/dist/dispose.d.ts +3 -0
- package/dist/dispose.js +9 -5
- package/dist/doctor.d.ts +7 -0
- package/dist/doctor.js +10 -2
- package/dist/envelope.js +2 -1
- package/dist/evidence.d.ts +9 -0
- package/dist/evidence.js +19 -4
- package/dist/exec.d.ts +9 -0
- package/dist/exec.js +22 -2
- package/dist/git-floor.d.ts +16 -0
- package/dist/git-floor.js +36 -0
- package/dist/grant.js +5 -1
- package/dist/guides.js +22 -11
- package/dist/latency-timeline.d.ts +46 -0
- package/dist/latency-timeline.js +120 -0
- package/dist/live-backup.d.ts +39 -0
- package/dist/live-backup.js +271 -0
- package/dist/operate.d.ts +13 -0
- package/dist/operate.js +969 -328
- package/dist/planner.js +2 -2
- package/dist/prepared-evidence.d.ts +2 -1
- package/dist/prepared-evidence.js +8 -3
- package/dist/project-knowledge.js +6 -2
- package/dist/project-learning.js +2 -2
- package/dist/proof.js +6 -0
- package/dist/read-executor-worker.js +8 -4
- package/dist/read-executor.d.ts +18 -1
- package/dist/read-executor.js +8 -0
- package/dist/release-adapters.d.ts +80 -2
- package/dist/release-adapters.js +441 -16
- package/dist/release-authority.js +6 -2
- package/dist/release-auto.d.ts +38 -7
- package/dist/release-auto.js +131 -15
- package/dist/release-card.d.ts +15 -3
- package/dist/release-card.js +25 -9
- package/dist/release-carry.d.ts +21 -1
- package/dist/release-carry.js +22 -0
- package/dist/release-ci-hold.d.ts +41 -0
- package/dist/release-ci-hold.js +53 -0
- package/dist/release-cli.d.ts +3 -0
- package/dist/release-cli.js +95 -17
- package/dist/release-coverage.js +5 -2
- package/dist/release-gate.d.ts +5 -0
- package/dist/release-gate.js +1 -1
- package/dist/release-recheck.d.ts +1 -1
- package/dist/release-recheck.js +8 -2
- package/dist/release.d.ts +144 -14
- package/dist/release.js +421 -81
- package/dist/review-context.js +8 -6
- package/dist/runner-process.d.ts +67 -0
- package/dist/runner-process.js +219 -0
- package/dist/runner.d.ts +48 -6
- package/dist/runner.js +73 -9
- package/dist/sandbox-runtime.d.ts +8 -0
- package/dist/sandbox-runtime.js +14 -2
- package/dist/scope.d.ts +19 -1
- package/dist/scope.js +26 -3
- package/dist/scout.js +2 -2
- package/dist/secret-mint.d.ts +1 -0
- package/dist/secret-mint.js +14 -0
- package/dist/serve.js +112 -51
- package/dist/server/chat.js +5 -5
- package/dist/server/http.d.ts +2 -0
- package/dist/server/pages.js +7 -3
- package/dist/server/remote-hooks.d.ts +1 -1
- package/dist/server/remote-hooks.js +5 -3
- package/dist/server/render-results.d.ts +2 -0
- package/dist/server/render-results.js +1 -0
- package/dist/server/render-tasks.d.ts +2 -0
- package/dist/server/render-tasks.js +2 -1
- package/dist/server/request-context.d.ts +8 -0
- package/dist/server/runtime.d.ts +2 -0
- package/dist/server/settings.js +22 -2
- package/dist/server/task-views.js +7 -2
- package/dist/server/tasks.js +4 -1
- package/dist/setup-guide.d.ts +2 -2
- package/dist/storage-sweep.js +1 -1
- package/dist/storage.d.ts +5 -0
- package/dist/storage.js +28 -1
- package/dist/store.d.ts +135 -2
- package/dist/store.js +275 -13
- package/dist/surface.js +3 -1
- package/dist/task-base.d.ts +4 -0
- package/dist/task-base.js +22 -2
- package/dist/teardown-git.d.ts +52 -3
- package/dist/teardown-git.js +145 -33
- package/dist/test-infra.js +1 -1
- package/dist/toolroll-update.js +2 -2
- package/dist/up-processes.d.ts +137 -0
- package/dist/up-processes.js +165 -0
- package/dist/up-role-child.d.ts +1 -0
- package/dist/up-role-child.js +81 -0
- package/dist/verification-evidence.d.ts +1 -0
- package/dist/verification-evidence.js +13 -1
- package/dist/workspace-revision.js +3 -1
- package/package.json +2 -1
package/dist/operate.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { mintSecret } from "./secret-mint.js";
|
|
1
2
|
import { ACP_AGENTS, ACP_PROVIDER_IDS, isAcpProvider, isKeyedProvider } from "./acp-agents.js";
|
|
2
3
|
import { inspectAcpAgent } from "./acp-inspect.js";
|
|
3
4
|
import { runAcpChecks } from "./acp-check.js";
|
|
@@ -43,6 +44,7 @@ import { createConnectionChecker } from "./provider-connection.js";
|
|
|
43
44
|
import { checkIntegrations, integrationsBrokenLine, integrationsNow, renderIntegrations } from "./integrations.js";
|
|
44
45
|
import { buildHandoff, handoffLines, loginAccount, runOnboard } from "./agent-onboard.js";
|
|
45
46
|
import { backupFiles, backupFolderOf, backupOwner, backupNow, restoreDatabase, startBackups } from "./backup.js";
|
|
47
|
+
import { startLiveBackups } from "./live-backup.js";
|
|
46
48
|
import { pushLimitSink } from "./provider-limits.js";
|
|
47
49
|
import { heuristicSizing, installFilingSizer, ownerSizer, refineFiledSizing, settleSizings } from "./task-sizing.js";
|
|
48
50
|
import { tierLines, tierReport } from "./tier-report.js";
|
|
@@ -56,7 +58,7 @@ import { readHooksBase, scheduleFromWords } from "./flow-triggers.js";
|
|
|
56
58
|
import { sendSubagentSummaries } from "./subagent-admin.js";
|
|
57
59
|
import { runRequestedUndos, sendSubagentWeeklies } from "./subagent-week.js";
|
|
58
60
|
import { refreshConnections } from "./mcp-connect.js";
|
|
59
|
-
import { releaseTermsFor } from "./release-carry.js";
|
|
61
|
+
import { releaseCheckPurposeFor, releaseTermsFor } from "./release-carry.js";
|
|
60
62
|
import { refreshTrustedMains, settleReleaseChecks } from "./nightly-check.js";
|
|
61
63
|
import { SECRETS_FIRST, startToolSecrets, toolSecretStorage } from "./project-tools.js";
|
|
62
64
|
import { followDiscord } from "./discord.js";
|
|
@@ -99,7 +101,7 @@ import { taskReviewBrief, renderReviewBrief } from "./task-review-brief.js";
|
|
|
99
101
|
import { homedir, hostname } from "node:os";
|
|
100
102
|
import { basename, dirname, join, resolve } from "node:path";
|
|
101
103
|
import { releaseStalledUpdate, waitingUpdate } from "./toolroll-update.js";
|
|
102
|
-
import { openStore, openStoreNoMigrate, CONCURRENT_WRITER_WAIT_MS, isDatabaseBusy, databasePath, BUILT_IN, parseCapabilityKey, verifiedAuthor, isDigestTime, RESULT_SCREENSHOTS, } from "./store.js";
|
|
104
|
+
import { openStore, openStoreNoMigrate, CONCURRENT_WRITER_WAIT_MS, isDatabaseBusy, databasePath, BUILT_IN, parseCapabilityKey, verifiedAuthor, isDigestTime, predecessorWatches, RESULT_SCREENSHOTS, lifecycleOf, reportLifecycle, beatLifecycle, } from "./store.js";
|
|
103
105
|
import { createHash, randomBytes, randomInt, randomUUID } from "node:crypto";
|
|
104
106
|
import { claimInstance, clearInstance, openInstanceSocket } from "./console-instance.js";
|
|
105
107
|
import { authorizePlanUnderMode } from "./plan-auto.js";
|
|
@@ -122,6 +124,9 @@ import { admissionRecorded, admitProject, commandLineActor } from "./project-adm
|
|
|
122
124
|
import { leadClaim, leadSay, noteLeadWork } from "./lead-voice.js";
|
|
123
125
|
import { createServer as createNetServer } from "node:net";
|
|
124
126
|
import { spawn as spawnChild } from "node:child_process";
|
|
127
|
+
import { fileURLToPath } from "node:url";
|
|
128
|
+
import { superviseController } from "./controller-supervisor.js";
|
|
129
|
+
import { buildIdentity, memoryChannel, RoleGenerations, roleMessage } from "./up-processes.js";
|
|
125
130
|
// Every envelope is checked against its command's schema (logged, never refused), then serialized unchanged.
|
|
126
131
|
import { checkedEnvelopeJson as envelopeJson } from "./contracts/cli.js";
|
|
127
132
|
import { auditCommand } from "./audit-cli.js";
|
|
@@ -190,7 +195,9 @@ import { acquire, acquireIfReady, heartbeat, release, reap, currentClaim, DEFAUL
|
|
|
190
195
|
import { disposeBuildOutcome, holdStaleApproval, regateTask } from "./dispose.js";
|
|
191
196
|
import { proposeGrant, describeGrant, describeWithheld, permits, MUTATION_CLASSES, DEFAULT_MUTATIONS, } from "./grant.js";
|
|
192
197
|
import { builtIn, guarded } from "./backend.js";
|
|
193
|
-
import { register, authenticate, heartbeat as heartbeatRunner, isAlive, recoverDead, recoveredAnything, acquireWatchLeaseAuthed, heartbeatWatchLeaseAuthed, addRunnerReposAuthed, removeRunnerRepoAuthed, registerRunnerIfIdle, retireRunnerIfCurrent, normalizeRunnerName, validRunnerName, canonicalRepos, RUNNER_NAME_MAX, RUNNER_CAPACITY_MAX, parseRunnerCapacity, } from "./runner.js";
|
|
198
|
+
import { register, authenticate, heartbeat as heartbeatRunner, isAlive, recoverDead, recoveredAnything, acquireWatchLeaseAuthed, heartbeatWatchLeaseAuthed, addRunnerReposAuthed, removeRunnerRepoAuthed, registerRunnerIfIdle, retireRunnerIfCurrent, drainRunnerIfCurrent, RUNNER_HEARTBEAT_MS, MAX_WATCH_LEASE_MS, normalizeRunnerName, validRunnerName, canonicalRepos, RUNNER_NAME_MAX, RUNNER_CAPACITY_MAX, parseRunnerCapacity, hashToken as hashRunnerToken, } from "./runner.js";
|
|
199
|
+
import { credentialDigest, recordedHolderProvedGone, processGone, processStart, readRunnerProcessRecord, runnerProcessFile, sealRunnerProcessRecord, writeRunnerProcessRecord } from "./runner-process.js";
|
|
200
|
+
import { stopProcessTree } from "./process-tree.js";
|
|
194
201
|
import { mintCoordinator, revokeCoordinator, listCoordinators } from "./coordinator.js";
|
|
195
202
|
import { serveMcp } from "./mcp.js";
|
|
196
203
|
import { createInterface } from "node:readline";
|
|
@@ -224,7 +231,8 @@ import { scout as scoutTask } from "./scout.js";
|
|
|
224
231
|
import { profileDigestOf } from "./scope.js";
|
|
225
232
|
import { PROVIDER_KEY_ENV, clearProviderKey, keyStatus, readAuthMode, readAuthModeStrict, readProviderKey, saveProviderKey, setAuthMode, verifyProviderKey, verdictWords } from "./keys.js";
|
|
226
233
|
import { run, terminateLiveProviders, run as execRun } from "./exec.js";
|
|
227
|
-
import { hardenedGit } from "./teardown-git.js";
|
|
234
|
+
import { hardenedGit, lazyFetchSafeGit, noLazyFetch, PARTIAL_CLONE_REFUSAL } from "./teardown-git.js";
|
|
235
|
+
import { installedGitFloorProblem } from "./git-floor.js";
|
|
228
236
|
import { noticeGitIncidents } from "./git-incident.js";
|
|
229
237
|
import { containmentNotice, containmentStatus, currentContainment, describeContainment, resolveContainment } from "./containment.js";
|
|
230
238
|
import { readPulls } from "./pulls.js";
|
|
@@ -435,6 +443,8 @@ External trackers — build what a tracker nominates, under local approvals
|
|
|
435
443
|
app; they start without a restart.
|
|
436
444
|
--repo still adds an exact path;
|
|
437
445
|
--no-open skips the browser;
|
|
446
|
+
the app and the builder run as two
|
|
447
|
+
processes (--single-process: one);
|
|
438
448
|
--host 0.0.0.0 --allow-host name:port
|
|
439
449
|
reaches a phone over a tailnet;
|
|
440
450
|
--runner names the worker;
|
|
@@ -656,6 +666,8 @@ export const KEYS_ACTIONS = ["status", "set", "clear", "verify", "auth"];
|
|
|
656
666
|
* tests): every value-taking flag any verb reads, and every boolean.
|
|
657
667
|
* A --flag in neither set is a typo, refused by name.
|
|
658
668
|
*/
|
|
669
|
+
/** Flags whose value is a secret someone types or pastes: taken as the next argument even when it starts with "--". */
|
|
670
|
+
export const SECRET_VALUE_FLAGS = new Set(["token", "password"]);
|
|
659
671
|
export const OPERATE_VALUE_FLAGS = new Set([
|
|
660
672
|
"key", "db", "runner", "ttl", "state", "on", "reason", "until", "id", "backend",
|
|
661
673
|
"allow", "selector", "paths", "credentials", "repo", "token", "capacity",
|
|
@@ -706,7 +718,7 @@ export const OPERATE_BOOLEAN_FLAGS = new Set([
|
|
|
706
718
|
"no-carry",
|
|
707
719
|
"json", "yes", "all", "brief", "local", "history", "latest-watch", "dry-run", "file",
|
|
708
720
|
"clear", "follow", "ready", "all-tasks", "inbound-only", "help", "undo", "anyone", "allow-dispatch", "allow-merge", "merge-delete-branch",
|
|
709
|
-
"no-open", "remove", "no-verify", "no-follow", "end", "report", "off", "tmux",
|
|
721
|
+
"no-open", "single-process", "remove", "no-verify", "no-follow", "end", "report", "off", "tmux",
|
|
710
722
|
"self-heal", "plan-auto", "chat-approve", "full-access", "repair-auto", "review-retry-auto", "no-local",
|
|
711
723
|
"html", "csv", "alerts-only",
|
|
712
724
|
// project network: what its builds may reach (--open | --allowlist).
|
|
@@ -772,11 +784,11 @@ export function parseOperateArgs(argv, ownValues = new Set()) {
|
|
|
772
784
|
const value = equals === -1 ? argv[++index] : argument.slice(equals + 1);
|
|
773
785
|
// A following --flag is not a value — consuming it would swallow a real
|
|
774
786
|
// flag and leave this one holding a name-shaped lie.
|
|
775
|
-
//
|
|
776
|
-
//
|
|
777
|
-
// Explicit --name=value
|
|
778
|
-
const
|
|
779
|
-
if (value === undefined || (equals === -1 && value.startsWith("--") && !
|
|
787
|
+
// A secret (a password, a token) is random base64url and may begin with two hyphens: it is the next argument
|
|
788
|
+
// unless that argument is exactly a flag this parser knows (a real flag is never swallowed, finding 22). Other flags
|
|
789
|
+
// never take a following --flag. Explicit --name=value carries arbitrary literal leading hyphens for every flag.
|
|
790
|
+
const namesFlag = (one) => { const flag = one.slice(2).split("=")[0]; return booleans.has(flag) || wantsValue.has(flag); };
|
|
791
|
+
if (value === undefined || (equals === -1 && value.startsWith("--") && (!SECRET_VALUE_FLAGS.has(name) || namesFlag(value))))
|
|
780
792
|
return { error: `--${name} needs a value` };
|
|
781
793
|
// `flows import --param a=1 --param b=2`: each value kept, one per line.
|
|
782
794
|
flags.set(name, name === "param" && typeof flags.get(name) === "string" ? `${flags.get(name)}\n${value}` : value);
|
|
@@ -943,6 +955,8 @@ async function dispatchOn(command, positional, flags, repoList, base, options, e
|
|
|
943
955
|
...(options.installBin === undefined ? {} : { installBin: options.installBin }),
|
|
944
956
|
...(options.onboardSeams === undefined ? {} : { onboardSeams: options.onboardSeams }),
|
|
945
957
|
...(options.upSeams === undefined ? {} : { upSeams: options.upSeams }),
|
|
958
|
+
...(options.upRole === undefined ? {} : { upRole: options.upRole }),
|
|
959
|
+
...(upNeedsOneProcess(options) ? { upOneProcess: true } : {}),
|
|
946
960
|
...(activeRemote() === null ? {} : { principal: activeRemote() }),
|
|
947
961
|
});
|
|
948
962
|
return await enter(run);
|
|
@@ -1312,11 +1326,11 @@ async function dispatch(command, positional, flags, context) {
|
|
|
1312
1326
|
case "sync":
|
|
1313
1327
|
return syncCommand(flags, context);
|
|
1314
1328
|
case "serve":
|
|
1315
|
-
return serveCommand(flags, context);
|
|
1329
|
+
return gitTooOld(command, context) ?? serveCommand(flags, context);
|
|
1316
1330
|
case "watch":
|
|
1317
|
-
return watchCommand(flags, context);
|
|
1331
|
+
return gitTooOld(command, context) ?? watchCommand(flags, context);
|
|
1318
1332
|
case "up":
|
|
1319
|
-
return upCommand(flags, context);
|
|
1333
|
+
return gitTooOld(command, context) ?? upCommand(flags, context);
|
|
1320
1334
|
case "onboard":
|
|
1321
1335
|
return onboardCommand(flags, context);
|
|
1322
1336
|
case "daemon":
|
|
@@ -1531,6 +1545,10 @@ function claimCommand(positional, flags, context) {
|
|
|
1531
1545
|
return fail(write, json, "claim", "coordinator-filed", "a coordinator filed this — nothing runs until the operator signs its scope", EXIT.refused);
|
|
1532
1546
|
if (result.reason === "sweep-limit")
|
|
1533
1547
|
return fail(write, json, "claim", "sweep-limit", `its sweep already has ${result.limit} running at once — it starts when one finishes`, EXIT.refused, { sweep: result.sweep, limit: result.limit });
|
|
1548
|
+
if (result.reason === "watch-custody")
|
|
1549
|
+
return fail(write, json, "claim", "watch-custody", "This worker no longer holds the project; restart the watch", EXIT.refused);
|
|
1550
|
+
if (result.reason === "draining")
|
|
1551
|
+
return fail(write, json, "claim", "draining", `Toolroll is restarting (draining): ${runner} takes no new work until it is back (by ${result.until})`, EXIT.refused, { until: result.until });
|
|
1534
1552
|
return fail(write, json, "claim", result.reason, `held by ${result.by} until ${result.until}`, EXIT.refused, { holder: result.by, until: result.until });
|
|
1535
1553
|
}
|
|
1536
1554
|
// Taking a task is what makes it running; leaving that to the caller would
|
|
@@ -3105,9 +3123,10 @@ async function tickCommand(flags, context) {
|
|
|
3105
3123
|
buildBase = source.head;
|
|
3106
3124
|
else {
|
|
3107
3125
|
const descendant = exists.code === 0 ? branch : base;
|
|
3108
|
-
|
|
3126
|
+
// On a git without --no-lazy-fetch, through a frozen read view that refuses a partial clone (lazyFetchSafeGit).
|
|
3127
|
+
const ancestry = await lazyFetchSafeGit(git)("git", [...noLazyFetch(), "-c", "protocol.allow=never", "--no-replace-objects", "merge-base", "--is-ancestor", source.head, descendant], { cwd: repo });
|
|
3109
3128
|
if (ancestry.code !== 0)
|
|
3110
|
-
problem = "the requested revision base does not contain its source head";
|
|
3129
|
+
problem = ancestry.stderr.includes(PARTIAL_CLONE_REFUSAL) ? PARTIAL_CLONE_REFUSAL : "the requested revision base does not contain its source head";
|
|
3111
3130
|
}
|
|
3112
3131
|
}
|
|
3113
3132
|
if (problem !== null) {
|
|
@@ -3936,6 +3955,7 @@ async function startConsole(options) {
|
|
|
3936
3955
|
...(options.upConsole === undefined ? {} : { upConsole: options.upConsole }),
|
|
3937
3956
|
...(options.editorLinks === undefined ? {} : { editorLinks: options.editorLinks }),
|
|
3938
3957
|
...(options.instanceSecret === undefined ? {} : { instanceSecret: options.instanceSecret }),
|
|
3958
|
+
...(options.draining === undefined ? {} : { draining: options.draining }),
|
|
3939
3959
|
});
|
|
3940
3960
|
await new Promise((resolve, reject) => {
|
|
3941
3961
|
server.once("error", reject);
|
|
@@ -5992,8 +6012,10 @@ async function daemonCommand(positional, flags, context) {
|
|
|
5992
6012
|
return fail(write, json, "daemon", "usage", "`toolroll daemon [install|status|uninstall|logs]`", EXIT.usage);
|
|
5993
6013
|
}
|
|
5994
6014
|
// ---- the watch loop --------------------------------------------------------
|
|
5995
|
-
|
|
5996
|
-
|
|
6015
|
+
// A watch lease lasts at most a minute and is renewed every 15 s (docs/plans/actors-lessons.md): a crashed worker's
|
|
6016
|
+
// projects are free within a minute, and a clean stop hands them back at once (retireRunnerIfCurrent).
|
|
6017
|
+
const WATCH_LEASE_MS = MAX_WATCH_LEASE_MS;
|
|
6018
|
+
const WATCH_HEARTBEAT_MS = RUNNER_HEARTBEAT_MS;
|
|
5997
6019
|
/** The start repair runs once per process, not once per project loop. */
|
|
5998
6020
|
/**
|
|
5999
6021
|
* The Telegram follower: one actor on the wire that sends notifications and
|
|
@@ -6056,7 +6078,19 @@ async function runWatchLoop(args) {
|
|
|
6056
6078
|
}
|
|
6057
6079
|
// The owner carries lifetime without changing the lease schema. Bounded watches cannot host automatic sweeps.
|
|
6058
6080
|
const incarnation = `${runFor === null ? "toolroll-service:" : ""}${randomUUID()}`;
|
|
6059
|
-
|
|
6081
|
+
let lease = acquireWatchLeaseAuthed(store, { runner, token, repo, owner: incarnation, ttlMs: WATCH_LEASE_MS }, new Date());
|
|
6082
|
+
// Only expiry hands the lease over (store.acquireWatchLease): a predecessor still renewing keeps it, and the takeover
|
|
6083
|
+
// recovers its incarnation in the same transaction, exactly as a fresh `up` after a crash would.
|
|
6084
|
+
const waitUntil = Date.now() + (args.leaseWaitMs ?? 0);
|
|
6085
|
+
let saidWaiting = false;
|
|
6086
|
+
const waitStopping = args.leaseWaitStopping ?? args.isStopping;
|
|
6087
|
+
while (!lease.ok && lease.reason === "watch-busy" && Date.now() < waitUntil && !waitStopping()) {
|
|
6088
|
+
if (!saidWaiting)
|
|
6089
|
+
progress(`watch: ${repo} is still held by the previous worker until ${lease.until ?? "its lease lapses"}; taking it over then`);
|
|
6090
|
+
saidWaiting = true;
|
|
6091
|
+
await new Promise(resolve => setTimeout(resolve, 1_000));
|
|
6092
|
+
lease = acquireWatchLeaseAuthed(store, { runner, token, repo, owner: incarnation, ttlMs: WATCH_LEASE_MS }, new Date());
|
|
6093
|
+
}
|
|
6060
6094
|
if (!lease.ok) {
|
|
6061
6095
|
return {
|
|
6062
6096
|
ok: false,
|
|
@@ -6656,8 +6690,9 @@ async function watchCommand(flags, context) {
|
|
|
6656
6690
|
*
|
|
6657
6691
|
* COMPOSITION ONLY: identities mint through the atomic doors (a first
|
|
6658
6692
|
* approver only while none exists; a runner only while its name is idle),
|
|
6659
|
-
* then the
|
|
6660
|
-
*
|
|
6693
|
+
* then the console and the worker (one watch loop per repository) run as two
|
|
6694
|
+
* supervised processes over one shared store (isolated-checks.md, stage 1;
|
|
6695
|
+
* up-processes.ts says what each holds). Nothing here
|
|
6661
6696
|
* loosens a ceremony: approval, budgets, publication, and every fence work
|
|
6662
6697
|
* exactly as they do for the long-hand verbs.
|
|
6663
6698
|
*/
|
|
@@ -6728,7 +6763,7 @@ async function resolveUpApprover(context, flags, canPrompt) {
|
|
|
6728
6763
|
// the insert (finding 27). A refused insert unlinks only when an
|
|
6729
6764
|
// UNRELATED approver won (finding 32).
|
|
6730
6765
|
const name = asFlag ?? process.env["USER"] ?? process.env["USERNAME"] ?? "operator";
|
|
6731
|
-
const password =
|
|
6766
|
+
const password = mintSecret(12);
|
|
6732
6767
|
try {
|
|
6733
6768
|
writeLoginFileDurably(loginFile, name, password);
|
|
6734
6769
|
}
|
|
@@ -6960,6 +6995,9 @@ async function onboardCommand(flags, context) {
|
|
|
6960
6995
|
}
|
|
6961
6996
|
async function upCommand(flags, context) {
|
|
6962
6997
|
const { store, write, json, clock } = context;
|
|
6998
|
+
// A role of an `up` that is already running (up-role-child.ts): no validation, identity or port of its own.
|
|
6999
|
+
if (context.upRole !== undefined)
|
|
7000
|
+
return upRoleCommand(context, context.upRole);
|
|
6963
7001
|
const demoFence = refuseDemo(context, "up");
|
|
6964
7002
|
if (demoFence !== null)
|
|
6965
7003
|
return demoFence;
|
|
@@ -7082,8 +7120,13 @@ async function upCommand(flags, context) {
|
|
|
7082
7120
|
probe.once("error", () => resolveProbe(false));
|
|
7083
7121
|
probe.listen(port, "127.0.0.1", () => resolveProbe(true));
|
|
7084
7122
|
});
|
|
7085
|
-
if (!reserved)
|
|
7123
|
+
if (!reserved) {
|
|
7124
|
+
// The console that holds it said it is stopping: a restart is under way, so say that instead of "another one runs".
|
|
7125
|
+
const holder = lifecycleOf(store, `console:${port}`);
|
|
7126
|
+
if (holder?.state === "draining")
|
|
7127
|
+
return fail(write, json, "up", "port-busy", `Toolroll is restarting (draining): the console on port ${port} is stopping — try again in a few seconds`, EXIT.refused);
|
|
7086
7128
|
return fail(write, json, "up", "port-busy", `port ${port} is taken — another \`up\` or \`serve\` may already be running; stop it, or pick a different --port`, EXIT.refused);
|
|
7129
|
+
}
|
|
7087
7130
|
// Under a coding agent nothing is asked and no secret is printed, even in a real terminal: what is shown lands in
|
|
7088
7131
|
// its transcript.
|
|
7089
7132
|
const agentRunning = underAgent(context.upSeams?.env ?? process.env);
|
|
@@ -7115,21 +7158,47 @@ async function upCommand(flags, context) {
|
|
|
7115
7158
|
// The runner's authority is BOUND to the canonical roots this up
|
|
7116
7159
|
// serves (MCP spec v6): the claim gate enforces membership, so an
|
|
7117
7160
|
// unbound registration would deny-all its own dispatches.
|
|
7118
|
-
|
|
7161
|
+
// A holder whose every process is proved gone (its record beside the database) no longer blocks the name: an
|
|
7162
|
+
// `up` killed outright can't leave a fresh heartbeat in the way of the next start (the 2026-10-29 deploy).
|
|
7163
|
+
const tryRegister = () => registerRunnerIfIdle(store, { name, host: hostname(), repos, now: clock(),
|
|
7164
|
+
previousGone: holder => recordedHolderProvedGone(runnerProcessFile(context.databaseFile, name), holder, hostname()) });
|
|
7165
|
+
let answer = tryRegister();
|
|
7166
|
+
// Wait for a draining holder to sign off or be proved gone, never just for its deadline to elapse.
|
|
7167
|
+
const drainWaitUntil = Date.now() + 2 * 60_000;
|
|
7168
|
+
let saidDraining = false;
|
|
7169
|
+
while (!answer.ok && answer.reason === "runner-draining" && Date.now() < drainWaitUntil) {
|
|
7170
|
+
if (!saidDraining)
|
|
7171
|
+
progress(`${answer.detail}; waiting for it to stop`);
|
|
7172
|
+
saidDraining = true;
|
|
7173
|
+
await new Promise(resolveWait => setTimeout(resolveWait, 1_000));
|
|
7174
|
+
answer = tryRegister();
|
|
7175
|
+
}
|
|
7119
7176
|
if (answer.ok) {
|
|
7120
7177
|
runnerName = name;
|
|
7121
7178
|
runnerToken = answer.token;
|
|
7179
|
+
if (answer.signedOff !== null)
|
|
7180
|
+
progress(`signed off the previous ${name}: its processes had already exited`);
|
|
7122
7181
|
if (answer.recoveredRuns > 0)
|
|
7123
7182
|
progress(`recovered ${answer.recoveredRuns} interrupted attempt(s) left by the previous ${name}`);
|
|
7183
|
+
// Every process given this credential is recorded (pid and start time) before it gets it: this one now, each
|
|
7184
|
+
// role child as it is spawned. A later start proves them gone by this record.
|
|
7185
|
+
if (!recordRunnerProcess(context.databaseFile, { runner: answer.runner, credentialHash: hashRunnerToken(answer.token) }, "up", process.pid)) {
|
|
7186
|
+
retireRunnerIfCurrent(store, runnerName, runnerToken, clock());
|
|
7187
|
+
return fail(write, json, "up", "runner-alive", "The worker process record could not be saved; nothing started", EXIT.refused);
|
|
7188
|
+
}
|
|
7124
7189
|
doorAnswer = answer;
|
|
7125
7190
|
break;
|
|
7126
7191
|
}
|
|
7127
7192
|
doorAnswer = answer;
|
|
7128
7193
|
if (runnerFlag !== undefined)
|
|
7129
7194
|
break; // an explicit name is never suffixed around
|
|
7195
|
+
if (answer.reason === "runner-draining")
|
|
7196
|
+
break; // a restart under way keeps its name
|
|
7130
7197
|
}
|
|
7131
7198
|
if (runnerName === "") {
|
|
7132
7199
|
const detail = doorAnswer !== null && !doorAnswer.ok ? doorAnswer.detail : "no worker name could be taken";
|
|
7200
|
+
if (doorAnswer !== null && !doorAnswer.ok && doorAnswer.reason === "runner-draining")
|
|
7201
|
+
return fail(write, json, "up", "runner-alive", `${detail}, and it hasn't yet — try again shortly`, EXIT.refused);
|
|
7133
7202
|
return fail(write, json, "up", "runner-alive", `${detail} — that worker looks alive; stop the other \`up\` or watch, or name a different worker with --runner`, EXIT.refused);
|
|
7134
7203
|
}
|
|
7135
7204
|
// 6. Enrollment: repositories and project roots are one atomic machine
|
|
@@ -7158,325 +7227,197 @@ async function upCommand(flags, context) {
|
|
|
7158
7227
|
finally {
|
|
7159
7228
|
await new Promise(done => probe.close(() => done()));
|
|
7160
7229
|
}
|
|
7161
|
-
// 7. The console
|
|
7162
|
-
//
|
|
7163
|
-
//
|
|
7230
|
+
// 7–8. The console and the worker (docs/plans/isolated-checks.md, stage 1): two processes, each started and
|
|
7231
|
+
// restarted by controller-supervisor, so a long check in the worker never holds a console response. With seams
|
|
7232
|
+
// that can't cross a process (tests, the desktop host) or --single-process, the same two roles run in this one.
|
|
7164
7233
|
const pool = join(dirname(context.databaseFile), "worktrees");
|
|
7165
|
-
|
|
7166
|
-
|
|
7167
|
-
|
|
7168
|
-
const
|
|
7169
|
-
// Cockpit B1: this start's instance secret, minted once it is listening; the console answers its challenges with it.
|
|
7170
|
-
let instanceSecret = null;
|
|
7171
|
-
let console_;
|
|
7172
|
-
try {
|
|
7173
|
-
console_ = await startConsole({
|
|
7174
|
-
instanceSecret: () => instanceSecret,
|
|
7175
|
-
context,
|
|
7176
|
-
host: hostFlag,
|
|
7177
|
-
port,
|
|
7178
|
-
...(allowFlag === undefined ? {} : { allowedHosts: allowFlag.split(",").map(one => one.trim()).filter(one => one !== "") }),
|
|
7179
|
-
...(publicUrlFlag === undefined ? {} : { publicUrl: publicUrlFlag }),
|
|
7180
|
-
localRunner: runnerName,
|
|
7181
|
-
poolRoot: pool,
|
|
7182
|
-
repos,
|
|
7183
|
-
currentRepos: () => [...activeRepos],
|
|
7184
|
-
admittedRepos: () => [...activeRepos],
|
|
7185
|
-
upConsole: true,
|
|
7186
|
-
registryPath,
|
|
7187
|
-
projectRoots,
|
|
7188
|
-
...(text(flags, "public-url") === undefined ? {} : { publicUrl: text(flags, "public-url") }),
|
|
7189
|
-
...(text(flags, "editor") === undefined ? {} : { editorLinks: "vscode" }),
|
|
7190
|
-
});
|
|
7191
|
-
}
|
|
7192
|
-
catch (error) {
|
|
7193
|
-
retireRunnerIfCurrent(store, runnerName, runnerToken, clock());
|
|
7194
|
-
return fail(write, json, "up", "port-busy", `${describe(error)} — the port was taken while starting; try again`, EXIT.refused);
|
|
7195
|
-
}
|
|
7196
|
-
// 8. The project supervisor: one watch loop per proved repository, plus a
|
|
7197
|
-
// tiny registry poll that can add more while this process stays up. The
|
|
7198
|
-
// poll is deliberately plain filesystem I/O (portable across macOS,
|
|
7199
|
-
// Linux, and Windows) and never invokes an agent while nothing changed.
|
|
7234
|
+
const split = !flags.has("single-process") && context.upOneProcess !== true;
|
|
7235
|
+
const build = upBuildIdentity();
|
|
7236
|
+
const containmentFlag = text(flags, "containment") ?? null;
|
|
7237
|
+
const stopGraceMs = Number(text(flags, "stop-grace") ?? 30_000);
|
|
7200
7238
|
let stopping = false;
|
|
7201
7239
|
let fatal = null;
|
|
7202
|
-
let graceTimer;
|
|
7203
7240
|
let runTimer;
|
|
7204
|
-
const followControllers = [];
|
|
7205
7241
|
let resolveStopped = () => { };
|
|
7206
7242
|
const stopped = new Promise(resolveStop => (resolveStopped = resolveStop));
|
|
7207
|
-
|
|
7208
|
-
|
|
7209
|
-
|
|
7210
|
-
|
|
7211
|
-
|
|
7212
|
-
|
|
7243
|
+
// What the worker serves now: relayed to the console (its project rail and chat), and a restarted worker's start.
|
|
7244
|
+
let currentRepos = [...repos];
|
|
7245
|
+
let consoleRole = null;
|
|
7246
|
+
let workerRole = null;
|
|
7247
|
+
// A stop that can't finish (a role that ignores its stop, a drain that hangs) still signs off: past the worker's own
|
|
7248
|
+
// drain bound and a margin, both roles' process trees are stopped outright, the runner is retired, and `up` ends. A
|
|
7249
|
+
// service's launchd window (ExitTimeOut 60 s, controller-service's 55 s) is longer than this at the default grace.
|
|
7250
|
+
let resolveForced = () => { };
|
|
7251
|
+
const forced = new Promise(resolveForce => (resolveForced = resolveForce));
|
|
7252
|
+
let forceTimer;
|
|
7253
|
+
const stopDeadlineMs = Math.max(1, stopGraceMs) + 20_000;
|
|
7213
7254
|
const stop = () => {
|
|
7214
7255
|
if (stopping) {
|
|
7215
|
-
hardStop();
|
|
7256
|
+
workerRole?.hardStop();
|
|
7216
7257
|
return;
|
|
7217
7258
|
}
|
|
7218
7259
|
stopping = true;
|
|
7219
7260
|
resolveStopped();
|
|
7220
|
-
for
|
|
7221
|
-
|
|
7222
|
-
|
|
7223
|
-
|
|
7224
|
-
|
|
7225
|
-
|
|
7226
|
-
|
|
7227
|
-
|
|
7228
|
-
|
|
7229
|
-
|
|
7230
|
-
|
|
7231
|
-
|
|
7261
|
+
// Draining, durably: no new work is admitted for this runner from now, and a restart waits for this drain's end
|
|
7262
|
+
// (now + the forced-stop bound) instead of a heartbeat window.
|
|
7263
|
+
try {
|
|
7264
|
+
if (runnerName !== "")
|
|
7265
|
+
drainRunnerIfCurrent(store, runnerName, runnerToken, clock(), new Date(clock().getTime() + stopDeadlineMs));
|
|
7266
|
+
}
|
|
7267
|
+
catch { /* the heartbeat bound still holds */ }
|
|
7268
|
+
forceTimer = setTimeout(() => {
|
|
7269
|
+
fatal ??= `stopping took longer than ${Math.round(stopDeadlineMs / 1000)} s; its console and worker were stopped outright`;
|
|
7270
|
+
workerRole?.kill();
|
|
7271
|
+
consoleRole?.kill();
|
|
7272
|
+
resolveForced();
|
|
7273
|
+
}, stopDeadlineMs);
|
|
7274
|
+
forceTimer.unref?.();
|
|
7275
|
+
// The coding owner record is released first, whatever else is still finishing: launchd kills a service that
|
|
7276
|
+
// outlasts its exit window, and a killed process cannot release it. The console keeps serving until the end.
|
|
7277
|
+
consoleRole?.send({ type: "stopping" });
|
|
7278
|
+
workerRole?.stop();
|
|
7232
7279
|
};
|
|
7233
7280
|
process.on("SIGINT", stop);
|
|
7234
7281
|
process.on("SIGTERM", stop);
|
|
7235
|
-
|
|
7236
|
-
|
|
7237
|
-
|
|
7238
|
-
|
|
7239
|
-
const
|
|
7240
|
-
|
|
7241
|
-
|
|
7242
|
-
|
|
7243
|
-
|
|
7244
|
-
|
|
7245
|
-
|
|
7246
|
-
|
|
7247
|
-
|
|
7248
|
-
|
|
7249
|
-
|
|
7250
|
-
|
|
7251
|
-
|
|
7252
|
-
|
|
7253
|
-
|
|
7282
|
+
// The process that started `up` gone (its service supervisor killed, or the IPC parent that ran it): nobody is left to
|
|
7283
|
+
// forward a stop, so `up` stops itself — it drains, retires its runner and ends instead of running on as an orphan.
|
|
7284
|
+
let parentGone = false;
|
|
7285
|
+
const parentAtStart = process.ppid;
|
|
7286
|
+
const parentWatch = context.upSeams === undefined && parentAtStart > 1 ? setInterval(() => {
|
|
7287
|
+
if (process.ppid === parentAtStart || parentGone)
|
|
7288
|
+
return;
|
|
7289
|
+
parentGone = true;
|
|
7290
|
+
progress("the process that started Toolroll is gone; stopping");
|
|
7291
|
+
stop();
|
|
7292
|
+
}, 1_000) : undefined;
|
|
7293
|
+
parentWatch?.unref?.();
|
|
7294
|
+
const onDisconnect = () => { if (!parentGone) {
|
|
7295
|
+
parentGone = true;
|
|
7296
|
+
progress("the process that started Toolroll closed its channel; stopping");
|
|
7297
|
+
stop();
|
|
7298
|
+
} };
|
|
7299
|
+
if (context.upSeams === undefined && process.channel !== undefined && process.channel !== null)
|
|
7300
|
+
process.once("disconnect", onDisconnect);
|
|
7301
|
+
// A log (a service, an agent, --json or --verbose) says whose each line is; a person at a terminal reads plain lines.
|
|
7302
|
+
const labelled = split && (json || flags.has("verbose") || !(context.upSeams?.terminal ?? process.stdout.isTTY === true));
|
|
7303
|
+
const roleLine = (role) => (line) => progress(labelled ? `[${role}] ${line}` : line);
|
|
7304
|
+
const dropSignals = () => {
|
|
7305
|
+
process.removeListener("SIGINT", stop);
|
|
7306
|
+
process.removeListener("SIGTERM", stop);
|
|
7307
|
+
process.removeListener("disconnect", onDisconnect);
|
|
7308
|
+
if (parentWatch !== undefined)
|
|
7309
|
+
clearInterval(parentWatch);
|
|
7254
7310
|
};
|
|
7255
|
-
const
|
|
7256
|
-
|
|
7257
|
-
|
|
7258
|
-
// terminal), and always when a required policy will refuse spawns.
|
|
7259
|
-
const containmentDetail = flags.has("verbose") || !(context.upSeams?.terminal ?? process.stdout.isTTY === true);
|
|
7260
|
-
let containmentSaid = false;
|
|
7261
|
-
const containmentLines = (effective) => {
|
|
7262
|
-
const notice = containmentNotice(effective);
|
|
7263
|
-
if (notice !== null && !containmentSaid)
|
|
7264
|
-
progress(notice);
|
|
7265
|
-
containmentSaid = true;
|
|
7266
|
-
return containmentDetail || effective.refusal !== null ? [`watch: ${describeContainment(effective)}`] : [];
|
|
7311
|
+
const consoleConfig = {
|
|
7312
|
+
host: hostFlag, port, allowedHosts: allowFlag === undefined ? null : allowFlag.split(",").map(one => one.trim()).filter(one => one !== ""),
|
|
7313
|
+
publicUrl: publicUrlFlag ?? null, localRunner: runnerName, poolRoot: pool, repos: [...repos], projectRoots, registryPath, editor: editorGiven !== undefined,
|
|
7267
7314
|
};
|
|
7268
|
-
const
|
|
7269
|
-
|
|
7270
|
-
|
|
7271
|
-
|
|
7272
|
-
|
|
7273
|
-
|
|
7274
|
-
|
|
7275
|
-
|
|
7276
|
-
|
|
7277
|
-
|
|
7278
|
-
|
|
7279
|
-
|
|
7280
|
-
|
|
7281
|
-
|
|
7282
|
-
|
|
7315
|
+
const pendingLinks = new Map();
|
|
7316
|
+
let linkRequests = 0;
|
|
7317
|
+
const roleSpawned = (role, pid) => {
|
|
7318
|
+
const held = store.getRunner(runnerName);
|
|
7319
|
+
return held !== null && held.credentialHash === hashRunnerToken(runnerToken) && recordRunnerProcess(context.databaseFile, held, role, pid);
|
|
7320
|
+
};
|
|
7321
|
+
consoleRole = startUpRole({
|
|
7322
|
+
role: "console", split, context, build, shutdownMs: 15_000, onSpawned: pid => roleSpawned("console", pid),
|
|
7323
|
+
start: () => ({ type: "start", role: "console", databaseFile: context.databaseFile, containment: containmentFlag, build, config: { ...consoleConfig, repos: [...currentRepos] } }),
|
|
7324
|
+
onMessage: message => {
|
|
7325
|
+
if (message.type === "log")
|
|
7326
|
+
roleLine("console")(message.line);
|
|
7327
|
+
else if (message.type === "sign-in-link") {
|
|
7328
|
+
pendingLinks.get(message.id)?.(message.link);
|
|
7329
|
+
pendingLinks.delete(message.id);
|
|
7283
7330
|
}
|
|
7284
|
-
|
|
7285
|
-
|
|
7286
|
-
|
|
7331
|
+
},
|
|
7332
|
+
// A restarted console is told what the worker serves now.
|
|
7333
|
+
onReady: (_, again) => { if (again) {
|
|
7334
|
+
roleLine("console")("started again");
|
|
7335
|
+
consoleRole?.send({ type: "repos", repos: [...currentRepos] });
|
|
7336
|
+
} },
|
|
7337
|
+
onEnded: (detail, restarting) => {
|
|
7338
|
+
if (restarting)
|
|
7339
|
+
roleLine("console")(`${detail}; starting it again`);
|
|
7340
|
+
else if (!stopping) {
|
|
7341
|
+
fatal = detail;
|
|
7342
|
+
stop();
|
|
7287
7343
|
}
|
|
7288
|
-
}
|
|
7289
|
-
|
|
7290
|
-
|
|
7291
|
-
|
|
7292
|
-
|
|
7293
|
-
|
|
7294
|
-
|
|
7295
|
-
|
|
7296
|
-
|
|
7297
|
-
|
|
7298
|
-
|
|
7299
|
-
|
|
7300
|
-
|
|
7301
|
-
|
|
7302
|
-
|
|
7303
|
-
|
|
7304
|
-
|
|
7305
|
-
|
|
7306
|
-
|
|
7344
|
+
},
|
|
7345
|
+
});
|
|
7346
|
+
const consoleReady = await consoleRole.ready;
|
|
7347
|
+
if (!consoleReady.ok) {
|
|
7348
|
+
dropSignals();
|
|
7349
|
+
retireRunnerIfCurrent(store, runnerName, runnerToken, clock());
|
|
7350
|
+
if (consoleReady.reason === "port-busy")
|
|
7351
|
+
return fail(write, json, "up", "port-busy", `${consoleReady.detail} — the port was taken while starting; try again`, EXIT.refused);
|
|
7352
|
+
return fail(write, json, "up", "failed", `the console could not start (${consoleReady.detail}) — nothing was left running`, EXIT.failed);
|
|
7353
|
+
}
|
|
7354
|
+
const url = consoleReady.url;
|
|
7355
|
+
if (!stopping) {
|
|
7356
|
+
workerRole = startUpRole({
|
|
7357
|
+
role: "worker", split, context, build, onSpawned: pid => roleSpawned("worker", pid),
|
|
7358
|
+
// The worker drains its builds for the stop grace, then stops them; past that it is stopped outright.
|
|
7359
|
+
shutdownMs: Math.max(1, stopGraceMs) + 15_000,
|
|
7360
|
+
start: generation => ({
|
|
7361
|
+
type: "start", role: "worker", databaseFile: context.databaseFile, containment: containmentFlag, build,
|
|
7362
|
+
config: {
|
|
7363
|
+
runner: runnerName, token: runnerToken, repos: generation === 1 ? [...repos] : [...currentRepos], pinnedRepos: [...pinnedRepos], projectRoots, registryPath, pool,
|
|
7364
|
+
forMs: forGiven === undefined ? null : Number(forGiven), publicUrl: publicUrlFlag ?? null, stopGraceMs, verbose: flags.has("verbose"),
|
|
7365
|
+
terminal: context.upSeams?.terminal ?? process.stdout.isTTY === true,
|
|
7366
|
+
// A restart waits out a crashed predecessor's lease (at most a minute), then recovers it: as a fresh `up` after a crash.
|
|
7367
|
+
leaseWaitMs: split ? WATCH_LEASE_MS + 15_000 : 0,
|
|
7368
|
+
},
|
|
7369
|
+
}),
|
|
7370
|
+
onMessage: message => {
|
|
7371
|
+
if (message.type === "log")
|
|
7372
|
+
roleLine("worker")(message.line);
|
|
7373
|
+
else if (message.type === "repos") {
|
|
7374
|
+
// The console's admitted projects widen only to what the builder is bound to in the store (the worker binds
|
|
7375
|
+
// through the authenticated door before it reports): a worker's word alone admits nothing.
|
|
7376
|
+
const bound = new Set(store.getRunner(runnerName)?.runner.repos ?? []);
|
|
7377
|
+
currentRepos = message.repos.filter(repo => bound.has(repo));
|
|
7378
|
+
consoleRole?.send({ type: "repos", repos: [...currentRepos] });
|
|
7379
|
+
}
|
|
7307
7380
|
},
|
|
7308
|
-
|
|
7309
|
-
|
|
7310
|
-
|
|
7311
|
-
|
|
7312
|
-
//
|
|
7313
|
-
|
|
7314
|
-
|
|
7315
|
-
|
|
7381
|
+
onReady: (_, again) => { if (again)
|
|
7382
|
+
roleLine("worker")("started again: its builds and checks continue"); },
|
|
7383
|
+
onEnded: (detail, restarting) => {
|
|
7384
|
+
// A builder whose credential no longer proves can't come back by restarting: the whole `up` stops, and the
|
|
7385
|
+
// service's next start registers it again, as before the split.
|
|
7386
|
+
if (restarting && authenticate(store, runnerName, runnerToken).ok) {
|
|
7387
|
+
roleLine("worker")(`${detail}; starting it again — its interrupted builds are recovered first`);
|
|
7388
|
+
return;
|
|
7389
|
+
}
|
|
7390
|
+
if (!stopping) {
|
|
7391
|
+
fatal = restarting ? `the builder identity changed (${detail})` : detail;
|
|
7316
7392
|
stop();
|
|
7317
7393
|
}
|
|
7318
|
-
|
|
7319
|
-
releasedRepos.delete(endedRepo);
|
|
7320
|
-
progress(`project removed — ${projectName(endedRepo)} is no longer built here`);
|
|
7321
|
-
markReady();
|
|
7322
|
-
return;
|
|
7323
|
-
}
|
|
7324
|
-
// A loop that dies while its siblings live must not leave a partial
|
|
7325
|
-
// cockpit standing silently (finding 5).
|
|
7326
|
-
if (!result.ok && !stopping) {
|
|
7327
|
-
fatal = `${prefix(endedRepo)}${result.detail}`;
|
|
7328
|
-
stop();
|
|
7329
|
-
}
|
|
7330
|
-
markReady(); // acquisition failures must not strand startup readiness
|
|
7331
|
-
});
|
|
7332
|
-
await ready;
|
|
7333
|
-
};
|
|
7334
|
-
await Promise.all(repos.map(repo => launchRepo(repo, false)));
|
|
7335
|
-
// Telegram: one follower for the whole service, never one per project
|
|
7336
|
-
// loop. Replies and notifications go out from it, over the same transport
|
|
7337
|
-
// in the same cycle (Oct 2: a reply sent by a project loop's own follower
|
|
7338
|
-
// failed "fetch failed" while notifications went out). A token saved
|
|
7339
|
-
// while the service runs connects within seconds; a follower that dies
|
|
7340
|
-
// is started again.
|
|
7341
|
-
const telegramController = new AbortController();
|
|
7342
|
-
followControllers.push(telegramController);
|
|
7343
|
-
const telegramSupervisor = (async () => {
|
|
7344
|
-
const pause = (ms) => new Promise(resolvePause => {
|
|
7345
|
-
const timer = setTimeout(done, ms);
|
|
7346
|
-
function done() { clearTimeout(timer); telegramController.signal.removeEventListener("abort", done); resolvePause(); }
|
|
7347
|
-
// An aborted signal never fires again: without this a stop waited the whole pause.
|
|
7348
|
-
if (telegramController.signal.aborted)
|
|
7349
|
-
done();
|
|
7350
|
-
else
|
|
7351
|
-
telegramController.signal.addEventListener("abort", done, { once: true });
|
|
7394
|
+
},
|
|
7352
7395
|
});
|
|
7353
|
-
|
|
7354
|
-
|
|
7355
|
-
|
|
7356
|
-
|
|
7357
|
-
});
|
|
7358
|
-
if (follower !== null)
|
|
7359
|
-
await follower;
|
|
7360
|
-
// Without a token (or on a fenced network) look again shortly; the fence line is said once.
|
|
7361
|
-
if (follower === null && loadBotToken(process.env, context.telegramTokenFile) !== null)
|
|
7362
|
-
break;
|
|
7363
|
-
await pause(follower === null ? 2_000 : 5_000);
|
|
7364
|
-
}
|
|
7365
|
-
})();
|
|
7366
|
-
const dynamicCeiling = resolveCeiling(repos, projectRoots).ceiling;
|
|
7367
|
-
const rejected = new Set();
|
|
7368
|
-
const registrySupervisor = (async () => {
|
|
7369
|
-
const sleep = (ms) => new Promise(resolveSleep => setTimeout(resolveSleep, ms));
|
|
7370
|
-
while (!stopping) {
|
|
7371
|
-
const loaded = await loadProjectRegistry(registryPath);
|
|
7372
|
-
if ("error" in loaded) {
|
|
7373
|
-
fatal = loaded.error;
|
|
7374
|
-
stop();
|
|
7375
|
-
break;
|
|
7376
|
-
}
|
|
7377
|
-
// Removed from the saved list: stop building it and take it out of the console. A pinned start keeps its own.
|
|
7378
|
-
const saved = new Set(loaded.repos.map(one => canonicalProject(one) ?? one));
|
|
7379
|
-
for (const repo of [...activeRepos]) {
|
|
7380
|
-
if (stopping || pinnedRepos.has(repo) || saved.has(repo))
|
|
7381
|
-
continue;
|
|
7382
|
-
activeRepos.delete(repo);
|
|
7383
|
-
releasedRepos.add(repo);
|
|
7384
|
-
if (!loopResults.has(repo))
|
|
7385
|
-
releasedRepos.delete(repo);
|
|
7386
|
-
}
|
|
7387
|
-
for (const candidate of loaded.repos) {
|
|
7388
|
-
if (stopping)
|
|
7389
|
-
break;
|
|
7390
|
-
const canonical = canonicalProject(candidate);
|
|
7391
|
-
// Still letting go of an earlier removal: it rejoins on the next pass after its watch has stopped.
|
|
7392
|
-
if (canonical !== null && (activeRepos.has(canonical) || releasedRepos.has(canonical)))
|
|
7393
|
-
continue;
|
|
7394
|
-
const approvedAdditions = context.additionalProjectRepos?.() ?? [];
|
|
7395
|
-
const admittedCeiling = resolveCeiling([...dynamicCeiling.repos, ...approvedAdditions], dynamicCeiling.roots).ceiling;
|
|
7396
|
-
// A project added after this start (`repos add`, the lead, the console) carries its recorded admission; a
|
|
7397
|
-
// registry line alone is not authority. Either way the folder is proved to be that Git checkout again.
|
|
7398
|
-
const allowed = canonical !== null &&
|
|
7399
|
-
((await authorizedProject(admittedCeiling, canonical)) || admissionRecorded(store, canonical)) &&
|
|
7400
|
-
(await proveRepo(canonical)) === canonical;
|
|
7401
|
-
if (!allowed) {
|
|
7402
|
-
if (!rejected.has(candidate)) {
|
|
7403
|
-
rejected.add(candidate);
|
|
7404
|
-
progress(`project not connected — ${candidate} isn't a Git repository added with \`toolroll repos add\` or the console`);
|
|
7405
|
-
}
|
|
7406
|
-
continue;
|
|
7407
|
-
}
|
|
7408
|
-
rejected.delete(candidate);
|
|
7409
|
-
await launchRepo(canonical, true);
|
|
7410
|
-
}
|
|
7411
|
-
if (!stopping)
|
|
7412
|
-
await sleep(500);
|
|
7396
|
+
const workerReady = await workerRole.ready;
|
|
7397
|
+
if (!workerReady.ok && fatal === null) {
|
|
7398
|
+
fatal = workerReady.detail;
|
|
7399
|
+
stop();
|
|
7413
7400
|
}
|
|
7414
|
-
})().catch(error => {
|
|
7415
|
-
fatal = `the project registry stopped (${describe(error)})`;
|
|
7416
|
-
stop();
|
|
7417
|
-
});
|
|
7418
|
-
// A runner with no projects still stays visibly alive while the UI waits
|
|
7419
|
-
// for the first addition. Watch loops heartbeat during their passes; this
|
|
7420
|
-
// machine-level pulse also covers the intentionally empty state.
|
|
7421
|
-
// Provider readiness (v47): this machine's non-spending observations,
|
|
7422
|
-
// recorded under its own runner name ONCE at start — so every route
|
|
7423
|
-
// projection can say ready / unavailable / unknown per provider, and
|
|
7424
|
-
// the dispatch gate can halt a leg this machine cannot run. There is no
|
|
7425
|
-
// periodic readiness daemon: startup, an explicit `providers --report`,
|
|
7426
|
-
// and the narrow pre-dispatch observation are the roads. Production
|
|
7427
|
-
// only: a test's injected agent runner is a fake, and its readiness
|
|
7428
|
-
// would be a fiction.
|
|
7429
|
-
const observeReadiness = () => {
|
|
7430
|
-
if (context.agentRunner !== undefined)
|
|
7431
|
-
return;
|
|
7432
|
-
void observeProviderReadiness((file, args, options) => run(file, args, { timeoutMs: options?.timeoutMs ?? 5_000, ...(options?.omitEnv === undefined ? {} : { omitEnv: options.omitEnv }) }), process.env, PROVIDER_IDS, provider => store.acpAttestationFor(provider, runnerName))
|
|
7433
|
-
.then(observed => {
|
|
7434
|
-
if (!stopping)
|
|
7435
|
-
reportProviderReadinessAuthed(store, { name: runnerName, token: runnerToken, observations: observed }, clock());
|
|
7436
|
-
})
|
|
7437
|
-
.catch(() => {
|
|
7438
|
-
// A probe that cannot run leaves readiness unknown — never a fiction.
|
|
7439
|
-
});
|
|
7440
|
-
};
|
|
7441
|
-
observeReadiness();
|
|
7442
|
-
try {
|
|
7443
|
-
notifyVersionLive(context.store, dirname(context.databaseFile), PACKAGE_VERSION, clock());
|
|
7444
7401
|
}
|
|
7445
|
-
|
|
7446
|
-
|
|
7402
|
+
// A runner with no projects still stays visibly alive while the UI waits for the first addition; the worker's
|
|
7403
|
+
// loops also renew it with their leases. Across a worker restart this keeps the name held.
|
|
7447
7404
|
const runnerHeartbeat = setInterval(() => {
|
|
7448
7405
|
const beat = heartbeatRunner(store, runnerName, runnerToken, clock());
|
|
7449
7406
|
if (!beat.ok && !stopping) {
|
|
7450
7407
|
fatal = `the builder identity changed (${beat.reason})`;
|
|
7451
7408
|
stop();
|
|
7452
7409
|
}
|
|
7453
|
-
},
|
|
7410
|
+
}, RUNNER_HEARTBEAT_MS);
|
|
7454
7411
|
runnerHeartbeat.unref?.();
|
|
7455
7412
|
// 9. Readiness, then the ONE startup envelope / greeting (finding 11/20).
|
|
7456
7413
|
const approverPlan2 = approver;
|
|
7457
|
-
const url = console_.url;
|
|
7458
7414
|
const terminal = !agentRunning && (context.upSeams?.terminal ?? process.stdout.isTTY === true);
|
|
7459
7415
|
const upHandoff = buildHandoff({
|
|
7460
7416
|
url,
|
|
7461
7417
|
login: { account: approverPlan2.passwordFile === null ? approverPlan2.approver : loginAccount(approverPlan2.passwordFile) ?? approverPlan2.approver, file: approverPlan2.passwordFile },
|
|
7462
7418
|
});
|
|
7463
7419
|
recordUpConsoleUrl(dirname(context.databaseFile), url, clock());
|
|
7464
|
-
|
|
7465
|
-
// (behind the agent fence), and a discovery-only socket beside it. A socket that can't open only means attaching
|
|
7466
|
-
// isn't offered; the console runs either way.
|
|
7467
|
-
// The socket is taken FIRST: only the console that owns it publishes a record. A second console on the same database
|
|
7468
|
-
// (another port) never replaces the first one's record; it runs, but isn't attached to.
|
|
7469
|
-
let mintedSecret = null;
|
|
7470
|
-
const instanceSocket = await openInstanceSocket(context.databaseFile, { pid: process.pid, build: packageVersion(), database: context.databaseFile, port: console_.port }, () => mintedSecret);
|
|
7471
|
-
if (instanceSocket.ok) {
|
|
7472
|
-
mintedSecret = await claimInstance(store, context.databaseFile, { pid: process.pid, port: console_.port, build: packageVersion(), socket: instanceSocket.name }, clock());
|
|
7473
|
-
instanceSecret = mintedSecret;
|
|
7474
|
-
if (mintedSecret === null)
|
|
7475
|
-
progress("attach: another console is running on this database — `toolroll` in a repository opens that one");
|
|
7476
|
-
}
|
|
7477
|
-
else
|
|
7478
|
-
progress(`attach: ${instanceSocket.problem} — \`toolroll\` in a repository won't find this console`);
|
|
7479
|
-
if (fatal === null) {
|
|
7420
|
+
if (fatal === null && !stopping) {
|
|
7480
7421
|
if (json) {
|
|
7481
7422
|
write(envelopeJson({
|
|
7482
7423
|
ok: true,
|
|
@@ -7504,7 +7445,13 @@ async function upCommand(flags, context) {
|
|
|
7504
7445
|
// one-time sign-in link (single use, ten minutes, this computer only); it is never printed. The password stays
|
|
7505
7446
|
// saved beside the database, for later and for other devices, and is never printed either.
|
|
7506
7447
|
const account = approverPlan2.approver;
|
|
7507
|
-
|
|
7448
|
+
// The console mints it (its sign-in secrets stay there); it comes back only to be opened, never printed.
|
|
7449
|
+
const link = account === null || flags.has("no-open") ? null : await new Promise(resolveLink => {
|
|
7450
|
+
const id = ++linkRequests;
|
|
7451
|
+
const timer = setTimeout(() => { pendingLinks.delete(id); resolveLink(null); }, 10_000);
|
|
7452
|
+
pendingLinks.set(id, value => { clearTimeout(timer); resolveLink(value); });
|
|
7453
|
+
consoleRole?.send({ type: "sign-in-link", id, account });
|
|
7454
|
+
});
|
|
7508
7455
|
const home = homedir();
|
|
7509
7456
|
const file = approverPlan2.passwordFile === null ? null : approverPlan2.passwordFile.startsWith(`${home}/`) ? `~${approverPlan2.passwordFile.slice(home.length)}` : approverPlan2.passwordFile;
|
|
7510
7457
|
const signInWords = account === null
|
|
@@ -7523,35 +7470,31 @@ async function upCommand(flags, context) {
|
|
|
7523
7470
|
}
|
|
7524
7471
|
}
|
|
7525
7472
|
}
|
|
7526
|
-
//
|
|
7473
|
+
// A bounded run (--for) counts from readiness, so a slow start never eats the window it asked for.
|
|
7474
|
+
if (forGiven !== undefined && !stopping) {
|
|
7475
|
+
runTimer = setTimeout(stop, Number(forGiven));
|
|
7476
|
+
runTimer.unref?.();
|
|
7477
|
+
}
|
|
7478
|
+
// 10. Supervise to the end: the worker drains first (its builds finish or are saved), the builder retires, and
|
|
7479
|
+
// only then does the console stop serving.
|
|
7527
7480
|
await stopped;
|
|
7528
|
-
|
|
7529
|
-
|
|
7530
|
-
// The follower was aborted with the stop. A send or a chat turn it is still
|
|
7531
|
-
// finishing keeps its own durable record; the service does not wait past
|
|
7532
|
-
// this for it.
|
|
7533
|
-
await new Promise(done => {
|
|
7534
|
-
const timer = setTimeout(done, TELEGRAM_STOP_WAIT_MS);
|
|
7535
|
-
const finish = () => { clearTimeout(timer); done(); };
|
|
7536
|
-
void telegramSupervisor.then(finish, finish);
|
|
7537
|
-
});
|
|
7481
|
+
if (workerRole !== null)
|
|
7482
|
+
await Promise.race([workerRole.done, forced]);
|
|
7538
7483
|
clearInterval(runnerHeartbeat);
|
|
7539
7484
|
if (runTimer !== undefined)
|
|
7540
7485
|
clearTimeout(runTimer);
|
|
7541
|
-
|
|
7542
|
-
clearTimeout(graceTimer);
|
|
7543
|
-
process.removeListener("SIGINT", stop);
|
|
7544
|
-
process.removeListener("SIGTERM", stop);
|
|
7486
|
+
dropSignals();
|
|
7545
7487
|
retireRunnerIfCurrent(store, runnerName, runnerToken, clock());
|
|
7546
|
-
|
|
7547
|
-
|
|
7548
|
-
if (
|
|
7549
|
-
|
|
7550
|
-
|
|
7551
|
-
|
|
7488
|
+
consoleRole.stop();
|
|
7489
|
+
const closed = await Promise.race([consoleRole.done, forced.then(() => ({ problem: null }))]);
|
|
7490
|
+
if (forceTimer !== undefined)
|
|
7491
|
+
clearTimeout(forceTimer);
|
|
7492
|
+
// With nobody left to wait for it, `up` ends once it has signed off, whatever handle lingers.
|
|
7493
|
+
if (parentGone)
|
|
7494
|
+
setTimeout(() => process.exit(fatal === null ? EXIT.ok : EXIT.failed), 1_000).unref?.();
|
|
7552
7495
|
// A live terminal that couldn't be proved stopped (or any other close failure) is said, never "stopped".
|
|
7553
|
-
if (
|
|
7554
|
-
fatal =
|
|
7496
|
+
if (closed.problem !== null && fatal === null)
|
|
7497
|
+
fatal = closed.problem;
|
|
7555
7498
|
if (fatal !== null) {
|
|
7556
7499
|
// The startup envelope (when json) already went out; the exit code is
|
|
7557
7500
|
// the health signal (finding 11/20) — never a second envelope.
|
|
@@ -9746,6 +9689,7 @@ async function scopeTask(positional, flags, context) {
|
|
|
9746
9689
|
// sandbox, say) file the scope without them: every later commit then asks again, as before.
|
|
9747
9690
|
const releaseGiven = text(flags, "release");
|
|
9748
9691
|
let releaseTerms;
|
|
9692
|
+
let releaseCheck;
|
|
9749
9693
|
let releaseNote = null;
|
|
9750
9694
|
if (releaseGiven !== undefined) {
|
|
9751
9695
|
const spec = /^(\d+\.\d+\.\d+):([1-9][0-9]*):([A-Za-z0-9._/-]+)$/.exec(releaseGiven);
|
|
@@ -9757,8 +9701,15 @@ async function scopeTask(positional, flags, context) {
|
|
|
9757
9701
|
const made = releaseTermsFor(store, { repo, version: spec[1], pr: Number(spec[2]), branch: spec[3], candidate: candidateGiven, now, noCarry: flags.get("no-carry") === true });
|
|
9758
9702
|
if (made.ok)
|
|
9759
9703
|
releaseTerms = made.terms;
|
|
9760
|
-
else
|
|
9704
|
+
else {
|
|
9761
9705
|
releaseNote = `Later commits on this release will each ask for approval: ${made.why}.`;
|
|
9706
|
+
// Without terms the candidate is still a release's check (pipeline-speed.md, P0): its purpose is sealed with the
|
|
9707
|
+
// scope, from the plane's own records, so the check runs under the release's rules and the plane may rerun it on
|
|
9708
|
+
// this same candidate after confirmed infrastructure. It never carries a later commit.
|
|
9709
|
+
const purpose = releaseCheckPurposeFor(store, { repo, version: spec[1], pr: Number(spec[2]), branch: spec[3], candidate: candidateGiven });
|
|
9710
|
+
if (purpose.ok)
|
|
9711
|
+
releaseCheck = purpose.purpose;
|
|
9712
|
+
}
|
|
9762
9713
|
}
|
|
9763
9714
|
if (acceptanceParse.criteria.length === 0)
|
|
9764
9715
|
return fail(write, json, "task scope", "acceptance-required", "--acceptance named no valid criteria", EXIT.usage);
|
|
@@ -9826,7 +9777,8 @@ async function scopeTask(positional, flags, context) {
|
|
|
9826
9777
|
: null;
|
|
9827
9778
|
const proposed = propose(store, {
|
|
9828
9779
|
...(candidateGiven === undefined ? {} : { candidate: candidateGiven }),
|
|
9829
|
-
...(releaseTerms === undefined ? {} : { releaseTerms }),
|
|
9780
|
+
...(releaseTerms === undefined ? {} : { releaseTerms, releaseCheck: null }),
|
|
9781
|
+
...(releaseCheck === undefined ? {} : { releaseCheck, releaseTerms: null }),
|
|
9830
9782
|
taskId: id,
|
|
9831
9783
|
goal,
|
|
9832
9784
|
outOfScope: text(flags, "not") ?? null,
|
|
@@ -11732,3 +11684,692 @@ async function projectSandboxCommand(positional, flags, context) {
|
|
|
11732
11684
|
"Scopes written from now on use this; scopes already approved keep what they were approved with.",
|
|
11733
11685
|
]);
|
|
11734
11686
|
}
|
|
11687
|
+
/** Starting the console or the worker on a git older than Toolroll's floor (git-floor.ts): refused, with one message
|
|
11688
|
+
* naming the installed and the required version. Other commands aren't held. */
|
|
11689
|
+
function gitTooOld(command, context) {
|
|
11690
|
+
const problem = installedGitFloorProblem();
|
|
11691
|
+
return problem === null ? null : fail(context.write, context.json, command, "git-too-old", problem, EXIT.refused);
|
|
11692
|
+
}
|
|
11693
|
+
/** This build's identity, as each role reports it: the version and the code of this module. */
|
|
11694
|
+
function upBuildIdentity() {
|
|
11695
|
+
return buildIdentity(PACKAGE_VERSION, fileURLToPath(import.meta.url));
|
|
11696
|
+
}
|
|
11697
|
+
/** Seams that live in this process's memory (a test's fake runner, a store wrapper, the desktop host's proof key): a
|
|
11698
|
+
* child process can't receive them, so `up` keeps both of its roles in this process rather than drop them. */
|
|
11699
|
+
function upNeedsOneProcess(options) {
|
|
11700
|
+
return options.openDatabase !== undefined || options.desktopIdentity !== undefined || options.additionalProjectRepos !== undefined ||
|
|
11701
|
+
options.agentRunner !== undefined || options.gitRunner !== undefined || options.telegramTransport !== undefined || options.telegramEnv !== undefined ||
|
|
11702
|
+
options.publishExec !== undefined || options.dispatchAdapter !== undefined || options.flowTriggerIo !== undefined || options.flowStepIo !== undefined ||
|
|
11703
|
+
options.flowEveryMs !== undefined || options.platform !== undefined || options.fenceAvailable !== undefined || options.flowFetch !== undefined ||
|
|
11704
|
+
options.integrationIo !== undefined || options.shouldStop !== undefined || options.leadSeams !== undefined || options.waitSleep !== undefined ||
|
|
11705
|
+
options.releaseIo !== undefined || options.installBin !== undefined || options.tempRoots !== undefined || options.filingSizer !== undefined ||
|
|
11706
|
+
options.now !== undefined;
|
|
11707
|
+
}
|
|
11708
|
+
/** The entry a role child runs: the built sibling when installed, the TypeScript source through tsx in a checkout. */
|
|
11709
|
+
/**
|
|
11710
|
+
* Add one process to the record of who holds this runner's credential (runner-process.ts), replacing that role's
|
|
11711
|
+
* earlier processes once they are gone. A complete authenticated record is required before delivering credentials.
|
|
11712
|
+
*/
|
|
11713
|
+
function recordRunnerProcess(databaseFile, holder, role, pid) {
|
|
11714
|
+
try {
|
|
11715
|
+
const started = processStart(pid);
|
|
11716
|
+
if (started === null)
|
|
11717
|
+
return false;
|
|
11718
|
+
const file = runnerProcessFile(databaseFile, holder.runner.name);
|
|
11719
|
+
const credential = credentialDigest(holder.credentialHash);
|
|
11720
|
+
const current = readRunnerProcessRecord(file);
|
|
11721
|
+
const same = current !== null && current.runner === holder.runner.name && current.registeredAt === holder.runner.registeredAt && current.credential === credential;
|
|
11722
|
+
// Never bless a tampered or incomplete earlier list while appending a credential holder.
|
|
11723
|
+
if (role !== "up" && (!same || current.signature !== sealRunnerProcessRecord(current, holder.credentialHash).signature))
|
|
11724
|
+
return false;
|
|
11725
|
+
// Keep every possibly live generation; truncating this list would erase custody.
|
|
11726
|
+
const kept = same ? current.processes.filter(one => one.role !== role || !processGone(one)) : [];
|
|
11727
|
+
const record = { v: 1, runner: holder.runner.name, host: holder.runner.host, registeredAt: holder.runner.registeredAt, credential,
|
|
11728
|
+
processes: [...kept.filter(one => one.pid !== pid), { pid, role, started }] };
|
|
11729
|
+
if (record.processes.length > 64)
|
|
11730
|
+
return false;
|
|
11731
|
+
writeRunnerProcessRecord(file, sealRunnerProcessRecord(record, holder.credentialHash));
|
|
11732
|
+
return true;
|
|
11733
|
+
}
|
|
11734
|
+
catch {
|
|
11735
|
+
return false;
|
|
11736
|
+
}
|
|
11737
|
+
}
|
|
11738
|
+
function upRoleChildCommand(role) {
|
|
11739
|
+
const source = import.meta.url.endsWith(".ts");
|
|
11740
|
+
const entry = fileURLToPath(new URL(source ? "./up-role-child.ts" : "./up-role-child.js", import.meta.url));
|
|
11741
|
+
// tsx by its resolved address: the child keeps this process's working directory, which may hold no node_modules.
|
|
11742
|
+
// The role's name is the one argument: a process listing tells the two apart, and carries nothing secret.
|
|
11743
|
+
return { file: process.execPath, argv: [...(source ? ["--import", import.meta.resolve("tsx")] : []), entry, role] };
|
|
11744
|
+
}
|
|
11745
|
+
/**
|
|
11746
|
+
* One role of `up`: a supervised child process (split), or the same role function in this process. Either way its
|
|
11747
|
+
* messages pass the same checks (up-processes.ts), and its build must be this one's.
|
|
11748
|
+
* - onReady fires on every readiness; `again` after a restart.
|
|
11749
|
+
* - onEnded: the role ended while not stopping. `restarting` (split only) when the supervisor will start it again.
|
|
11750
|
+
*/
|
|
11751
|
+
function startUpRole(args) {
|
|
11752
|
+
const { role } = args;
|
|
11753
|
+
let resolveReady = () => { };
|
|
11754
|
+
const ready = new Promise(resolve => (resolveReady = resolve));
|
|
11755
|
+
// Readiness per generation (Codex review round 1, finding 5): a replacement's refusal is never hidden by an earlier
|
|
11756
|
+
// generation having been ready.
|
|
11757
|
+
const generations = new RoleGenerations();
|
|
11758
|
+
let stopping = false;
|
|
11759
|
+
// Reported once: the role ended for good (onEnded with restarting false), whichever path saw it first.
|
|
11760
|
+
let endedForGood = false;
|
|
11761
|
+
const endForGood = (detail) => {
|
|
11762
|
+
if (stopping || endedForGood)
|
|
11763
|
+
return;
|
|
11764
|
+
endedForGood = true;
|
|
11765
|
+
args.onEnded(detail, false);
|
|
11766
|
+
};
|
|
11767
|
+
let failed = null;
|
|
11768
|
+
let fatal = null;
|
|
11769
|
+
let closedProblem = null;
|
|
11770
|
+
let livePid = null;
|
|
11771
|
+
// Shared by both transports: every message is checked against this role before anything acts on it.
|
|
11772
|
+
const receive = (raw, abort) => {
|
|
11773
|
+
const message = roleMessage(raw, role);
|
|
11774
|
+
if (message === null)
|
|
11775
|
+
return;
|
|
11776
|
+
if (message.type === "ready") {
|
|
11777
|
+
if (message.build !== args.build) {
|
|
11778
|
+
// Never a console and a worker from different builds: stop rather than run a mixed pair.
|
|
11779
|
+
fatal = `the ${role} is a different build (${message.build}) from this one (${args.build}) — restart Toolroll`;
|
|
11780
|
+
failed = { reason: "start", detail: fatal };
|
|
11781
|
+
generations.refused(fatal);
|
|
11782
|
+
abort();
|
|
11783
|
+
return;
|
|
11784
|
+
}
|
|
11785
|
+
livePid = message.pid;
|
|
11786
|
+
const again = !generations.ready();
|
|
11787
|
+
if (!again)
|
|
11788
|
+
resolveReady({ ok: true, url: message.url ?? "", port: message.port, pid: message.pid });
|
|
11789
|
+
args.onReady(message, again);
|
|
11790
|
+
return;
|
|
11791
|
+
}
|
|
11792
|
+
if (message.type === "failed") {
|
|
11793
|
+
failed = { reason: message.reason, detail: message.detail };
|
|
11794
|
+
generations.refused(message.detail);
|
|
11795
|
+
return;
|
|
11796
|
+
}
|
|
11797
|
+
if (message.type === "fatal") {
|
|
11798
|
+
fatal = message.detail;
|
|
11799
|
+
return;
|
|
11800
|
+
}
|
|
11801
|
+
if (message.type === "closed") {
|
|
11802
|
+
closedProblem = message.problem;
|
|
11803
|
+
return;
|
|
11804
|
+
}
|
|
11805
|
+
args.onMessage(message);
|
|
11806
|
+
};
|
|
11807
|
+
if (!args.split) {
|
|
11808
|
+
const channel = memoryChannel(role);
|
|
11809
|
+
channel.parent.onMessage(message => receive(message, () => channel.parent.stop(false)));
|
|
11810
|
+
const running = upRoleCommand(args.context, { start: args.start(1), port: channel.port }).catch(error => {
|
|
11811
|
+
fatal = describe(error);
|
|
11812
|
+
return EXIT.failed;
|
|
11813
|
+
});
|
|
11814
|
+
const done = running.then(code => {
|
|
11815
|
+
livePid = null;
|
|
11816
|
+
if (!generations.wasReady())
|
|
11817
|
+
resolveReady({ ok: false, ...(failed ?? { reason: "start", detail: fatal ?? `the ${role} stopped while starting (exit ${code})` }) });
|
|
11818
|
+
else
|
|
11819
|
+
endForGood(fatal ?? `the ${role} stopped (exit ${code})`);
|
|
11820
|
+
return { problem: closedProblem };
|
|
11821
|
+
});
|
|
11822
|
+
return {
|
|
11823
|
+
ready, done,
|
|
11824
|
+
send: message => channel.parent.send(message),
|
|
11825
|
+
stop: () => { stopping = true; channel.parent.stop(false); },
|
|
11826
|
+
hardStop: () => { channel.parent.stop(true); },
|
|
11827
|
+
kill: () => { stopping = true; channel.parent.stop(true); },
|
|
11828
|
+
pid: () => livePid,
|
|
11829
|
+
};
|
|
11830
|
+
}
|
|
11831
|
+
const controller = new AbortController();
|
|
11832
|
+
let child = null;
|
|
11833
|
+
const command = upRoleChildCommand(role);
|
|
11834
|
+
const supervised = superviseController({
|
|
11835
|
+
file: command.file, argv: command.argv, signal: controller.signal, ipc: true, shutdownMs: args.shutdownMs,
|
|
11836
|
+
// Each role's own output carries its name; a JSON log stays one JSON object a line.
|
|
11837
|
+
label: envValue(process.env, "LOG_FORMAT") === "json" ? "" : `[${role}]`,
|
|
11838
|
+
onSpawn: (owned, generation) => {
|
|
11839
|
+
child = owned;
|
|
11840
|
+
fatal = null;
|
|
11841
|
+
failed = null;
|
|
11842
|
+
generations.spawned();
|
|
11843
|
+
owned.on("message", raw => receive(raw, () => controller.abort()));
|
|
11844
|
+
owned.on("error", () => { });
|
|
11845
|
+
let recorded = false;
|
|
11846
|
+
try {
|
|
11847
|
+
recorded = owned.pid !== undefined && (args.onSpawned?.(owned.pid) ?? true);
|
|
11848
|
+
}
|
|
11849
|
+
catch { /* refuse credential delivery */ }
|
|
11850
|
+
if (!recorded) {
|
|
11851
|
+
fatal = "The worker process record could not be saved";
|
|
11852
|
+
failed = { reason: "start", detail: fatal };
|
|
11853
|
+
generations.refused(fatal);
|
|
11854
|
+
owned.kill("SIGKILL");
|
|
11855
|
+
controller.abort();
|
|
11856
|
+
resolveReady({ ok: false, ...failed });
|
|
11857
|
+
return;
|
|
11858
|
+
}
|
|
11859
|
+
// The start, including the runner token for the worker, goes over the private channel only.
|
|
11860
|
+
owned.send(args.start(generation), error => { if (error !== null && error !== undefined)
|
|
11861
|
+
owned.kill("SIGTERM"); });
|
|
11862
|
+
},
|
|
11863
|
+
onState: state => {
|
|
11864
|
+
if (state.phase !== "backoff")
|
|
11865
|
+
return;
|
|
11866
|
+
child = null;
|
|
11867
|
+
livePid = null;
|
|
11868
|
+
const exit = state.exit?.signal ?? state.exit?.code ?? "unknown";
|
|
11869
|
+
if (stopping && generations.wasReady())
|
|
11870
|
+
return;
|
|
11871
|
+
const verdict = generations.ended();
|
|
11872
|
+
if (verdict === "fail-start") {
|
|
11873
|
+
// A role that never became ready is not retried: the start fails as one, as before the split.
|
|
11874
|
+
controller.abort();
|
|
11875
|
+
resolveReady({ ok: false, ...(failed ?? { reason: "start", detail: fatal ?? `the ${role} stopped while starting (exit ${exit})` }) });
|
|
11876
|
+
return;
|
|
11877
|
+
}
|
|
11878
|
+
if (verdict === "end") {
|
|
11879
|
+
// A replacement that refused its start (another build or database), or kept ending before it was ready: a
|
|
11880
|
+
// restart can't fix that, so the whole `up` stops instead of looping with the other role still running.
|
|
11881
|
+
controller.abort();
|
|
11882
|
+
endForGood(generations.refusedWith() ?? fatal ?? `the ${role} kept stopping before it was ready (exit ${exit})`);
|
|
11883
|
+
return;
|
|
11884
|
+
}
|
|
11885
|
+
// A build mismatch already aborted: that is the end, not a restart.
|
|
11886
|
+
if (controller.signal.aborted) {
|
|
11887
|
+
endForGood(fatal ?? `the ${role} stopped (exit ${exit})`);
|
|
11888
|
+
return;
|
|
11889
|
+
}
|
|
11890
|
+
args.onEnded(`${fatal === null ? "" : `${fatal} — `}the ${role} stopped unexpectedly (exit ${exit})`, true);
|
|
11891
|
+
},
|
|
11892
|
+
}).catch(error => { if (!generations.wasReady())
|
|
11893
|
+
resolveReady({ ok: false, reason: "start", detail: describe(error) }); });
|
|
11894
|
+
const done = supervised.then(() => {
|
|
11895
|
+
if (!generations.wasReady())
|
|
11896
|
+
resolveReady({ ok: false, ...(failed ?? { reason: "start", detail: fatal ?? `the ${role} did not start` }) });
|
|
11897
|
+
// The supervisor ended without `up` stopping it (an abort on a refusal emits no backoff): `up` hears it either way.
|
|
11898
|
+
else
|
|
11899
|
+
endForGood(generations.refusedWith() ?? fatal ?? `the ${role} stopped`);
|
|
11900
|
+
return { problem: closedProblem };
|
|
11901
|
+
});
|
|
11902
|
+
return {
|
|
11903
|
+
ready, done,
|
|
11904
|
+
send: message => {
|
|
11905
|
+
const owned = child;
|
|
11906
|
+
if (owned !== null && owned.connected) {
|
|
11907
|
+
try {
|
|
11908
|
+
owned.send(message, () => { });
|
|
11909
|
+
}
|
|
11910
|
+
catch { /* it is ending; the supervisor sees it */ }
|
|
11911
|
+
}
|
|
11912
|
+
},
|
|
11913
|
+
// The supervisor's stop is SIGTERM, which a role takes as "drain"; past shutdownMs its whole tree is stopped.
|
|
11914
|
+
stop: () => { stopping = true; controller.abort(); },
|
|
11915
|
+
hardStop: () => {
|
|
11916
|
+
const owned = child;
|
|
11917
|
+
if (owned !== null && owned.connected) {
|
|
11918
|
+
try {
|
|
11919
|
+
owned.send({ type: "hard-stop" }, () => { });
|
|
11920
|
+
}
|
|
11921
|
+
catch { /* ending */ }
|
|
11922
|
+
}
|
|
11923
|
+
},
|
|
11924
|
+
kill: () => {
|
|
11925
|
+
stopping = true;
|
|
11926
|
+
controller.abort();
|
|
11927
|
+
const owned = child;
|
|
11928
|
+
// Only the still-owned ChildProcess is signalled, never a recorded pid.
|
|
11929
|
+
if (owned !== null && owned.exitCode === null && owned.signalCode === null && !stopProcessTree(owned))
|
|
11930
|
+
owned.kill("SIGKILL");
|
|
11931
|
+
},
|
|
11932
|
+
pid: () => livePid,
|
|
11933
|
+
};
|
|
11934
|
+
}
|
|
11935
|
+
/** A role child's whole run (up-role-child.ts): the ordinary command prelude (containment, secrets, limits), then the
|
|
11936
|
+
* role itself. Only a process started by `up` gets here; no command line selects a role. */
|
|
11937
|
+
export async function runUpRole(start, port) {
|
|
11938
|
+
return runOperate("up", ["--db", start.databaseFile, ...(start.containment === null ? [] : ["--containment", start.containment])], line => port.send({ type: "log", line }), { upRole: { start, port } });
|
|
11939
|
+
}
|
|
11940
|
+
async function upRoleCommand(context, host) {
|
|
11941
|
+
const { start, port } = host;
|
|
11942
|
+
// A role runs exactly this build, on exactly this database: anything else ends it before it touches either.
|
|
11943
|
+
if (start.build !== upBuildIdentity()) {
|
|
11944
|
+
port.send({ type: "failed", reason: "start", detail: `this ${start.role} is a different build (${upBuildIdentity()}) from the \`up\` that started it (${start.build})` });
|
|
11945
|
+
return EXIT.refused;
|
|
11946
|
+
}
|
|
11947
|
+
if (resolve(start.databaseFile) !== resolve(context.databaseFile)) {
|
|
11948
|
+
port.send({ type: "failed", reason: "start", detail: `the ${start.role} opened a different database` });
|
|
11949
|
+
return EXIT.refused;
|
|
11950
|
+
}
|
|
11951
|
+
return start.role === "console" ? runUpConsoleRole(context, start.config, port, start.build) : runUpWorkerRole(context, start.config, port, start.build);
|
|
11952
|
+
}
|
|
11953
|
+
/** The console role: HTTP only — serving, reading, and the person's ceremonies. It never holds the runner token. */
|
|
11954
|
+
async function runUpConsoleRole(context, config, port, build) {
|
|
11955
|
+
const { store, clock } = context;
|
|
11956
|
+
const progress = (line) => port.send({ type: "log", line });
|
|
11957
|
+
// Mutated only by the worker's word (relayed by `up`): the console's project rail and unified chat follow the live
|
|
11958
|
+
// builder instead of freezing at process start.
|
|
11959
|
+
const activeRepos = new Set(config.repos);
|
|
11960
|
+
// Cockpit B1: this start's instance secret, minted once it is listening; the console answers its challenges with it.
|
|
11961
|
+
let instanceSecret = null;
|
|
11962
|
+
// v137: this console's lifecycle (service_lifecycle "console:<port>"): started, a short heartbeat, draining once
|
|
11963
|
+
// its `up` stops, stopped at the end. While draining, /healthz and every page say Toolroll is restarting.
|
|
11964
|
+
let draining = false;
|
|
11965
|
+
let console_;
|
|
11966
|
+
try {
|
|
11967
|
+
console_ = await startConsole({
|
|
11968
|
+
instanceSecret: () => instanceSecret,
|
|
11969
|
+
draining: () => draining,
|
|
11970
|
+
context,
|
|
11971
|
+
host: config.host,
|
|
11972
|
+
port: config.port,
|
|
11973
|
+
...(config.allowedHosts === null ? {} : { allowedHosts: config.allowedHosts }),
|
|
11974
|
+
...(config.publicUrl === null ? {} : { publicUrl: config.publicUrl }),
|
|
11975
|
+
localRunner: config.localRunner,
|
|
11976
|
+
poolRoot: config.poolRoot,
|
|
11977
|
+
repos: config.repos,
|
|
11978
|
+
currentRepos: () => [...activeRepos],
|
|
11979
|
+
admittedRepos: () => [...activeRepos],
|
|
11980
|
+
upConsole: true,
|
|
11981
|
+
registryPath: config.registryPath,
|
|
11982
|
+
projectRoots: config.projectRoots,
|
|
11983
|
+
...(config.editor ? { editorLinks: "vscode" } : {}),
|
|
11984
|
+
});
|
|
11985
|
+
}
|
|
11986
|
+
catch (error) {
|
|
11987
|
+
port.send({ type: "failed", reason: "port-busy", detail: describe(error) });
|
|
11988
|
+
return EXIT.refused;
|
|
11989
|
+
}
|
|
11990
|
+
let codingClosed = false;
|
|
11991
|
+
const closeCoding = () => {
|
|
11992
|
+
if (codingClosed)
|
|
11993
|
+
return;
|
|
11994
|
+
codingClosed = true;
|
|
11995
|
+
console_.server.closeCoding().catch(() => { });
|
|
11996
|
+
};
|
|
11997
|
+
let resolveStopped = () => { };
|
|
11998
|
+
const stopped = new Promise(resolveStop => (resolveStopped = resolveStop));
|
|
11999
|
+
port.onStop(() => resolveStopped());
|
|
12000
|
+
const lifecycleComponent = `console:${console_.port}`;
|
|
12001
|
+
const lifecycleHolder = `${process.pid}:${randomUUID().slice(0, 8)}`;
|
|
12002
|
+
try {
|
|
12003
|
+
reportLifecycle(store, lifecycleComponent, "started", lifecycleHolder, clock(), { pid: process.pid });
|
|
12004
|
+
}
|
|
12005
|
+
catch { /* reporting never stops the console */ }
|
|
12006
|
+
const lifecycleBeat = setInterval(() => { try {
|
|
12007
|
+
beatLifecycle(store, lifecycleComponent, lifecycleHolder, clock());
|
|
12008
|
+
}
|
|
12009
|
+
catch { /* the next beat tries again */ } }, RUNNER_HEARTBEAT_MS);
|
|
12010
|
+
lifecycleBeat.unref?.();
|
|
12011
|
+
port.onMessage(message => {
|
|
12012
|
+
if (message.type === "repos") {
|
|
12013
|
+
activeRepos.clear();
|
|
12014
|
+
for (const repo of message.repos)
|
|
12015
|
+
activeRepos.add(repo);
|
|
12016
|
+
}
|
|
12017
|
+
else if (message.type === "stopping") {
|
|
12018
|
+
draining = true;
|
|
12019
|
+
try {
|
|
12020
|
+
reportLifecycle(store, lifecycleComponent, "draining", lifecycleHolder, clock(), { drainUntil: new Date(clock().getTime() + 60_000) });
|
|
12021
|
+
}
|
|
12022
|
+
catch { /* the flag above still answers */ }
|
|
12023
|
+
closeCoding();
|
|
12024
|
+
}
|
|
12025
|
+
else if (message.type === "sign-in-link") {
|
|
12026
|
+
let link = null;
|
|
12027
|
+
try {
|
|
12028
|
+
link = console_.server.mintSignInLink(message.account);
|
|
12029
|
+
}
|
|
12030
|
+
catch {
|
|
12031
|
+
link = null;
|
|
12032
|
+
}
|
|
12033
|
+
port.send({ type: "sign-in-link", id: message.id, link });
|
|
12034
|
+
}
|
|
12035
|
+
});
|
|
12036
|
+
// Cockpit B1: who this console is, for bare `toolroll` to prove before it attaches: a fresh secret in the database
|
|
12037
|
+
// (behind the agent fence), and a discovery-only socket beside it. A socket that can't open only means attaching
|
|
12038
|
+
// isn't offered; the console runs either way. The socket is taken FIRST: only the console that owns it publishes a
|
|
12039
|
+
// record. A second console on the same database (another port) never replaces the first one's record.
|
|
12040
|
+
let mintedSecret = null;
|
|
12041
|
+
const instanceSocket = await openInstanceSocket(context.databaseFile, { pid: process.pid, build: packageVersion(), database: context.databaseFile, port: console_.port }, () => mintedSecret);
|
|
12042
|
+
if (instanceSocket.ok) {
|
|
12043
|
+
mintedSecret = await claimInstance(store, context.databaseFile, { pid: process.pid, port: console_.port, build: packageVersion(), socket: instanceSocket.name }, clock());
|
|
12044
|
+
instanceSecret = mintedSecret;
|
|
12045
|
+
if (mintedSecret === null)
|
|
12046
|
+
progress("attach: another console is running on this database — `toolroll` in a repository opens that one");
|
|
12047
|
+
}
|
|
12048
|
+
else
|
|
12049
|
+
progress(`attach: ${instanceSocket.problem} — \`toolroll\` in a repository won't find this console`);
|
|
12050
|
+
port.send({ type: "ready", role: "console", build, pid: process.pid, url: console_.url, port: console_.port });
|
|
12051
|
+
await stopped;
|
|
12052
|
+
draining = true;
|
|
12053
|
+
closeCoding();
|
|
12054
|
+
clearInterval(lifecycleBeat);
|
|
12055
|
+
if (instanceSocket.ok)
|
|
12056
|
+
instanceSocket.close();
|
|
12057
|
+
if (mintedSecret !== null)
|
|
12058
|
+
clearInstance(store, mintedSecret);
|
|
12059
|
+
let closeProblem = null;
|
|
12060
|
+
await new Promise(done => console_.server.close(error => { closeProblem = error === undefined || error === null ? null : error.message; done(); }));
|
|
12061
|
+
try {
|
|
12062
|
+
reportLifecycle(store, lifecycleComponent, "stopped", lifecycleHolder, clock());
|
|
12063
|
+
}
|
|
12064
|
+
catch { /* a stale row lapses with its heartbeat */ }
|
|
12065
|
+
port.send({ type: "closed", problem: closeProblem });
|
|
12066
|
+
return closeProblem === null ? EXIT.ok : EXIT.failed;
|
|
12067
|
+
}
|
|
12068
|
+
/** The worker role: one watch loop per proved repository (builds, checks, batch checks, follow-ups, reconcilers, the
|
|
12069
|
+
* storage sweep and checkout sizing), the project registry, and the service's one Telegram follower. It opens no
|
|
12070
|
+
* port and mints no console secret. */
|
|
12071
|
+
async function runUpWorkerRole(context, config, port, build) {
|
|
12072
|
+
const { store, clock } = context;
|
|
12073
|
+
const progress = (line) => port.send({ type: "log", line });
|
|
12074
|
+
const runnerName = config.runner;
|
|
12075
|
+
const runnerToken = config.token;
|
|
12076
|
+
const pool = config.pool;
|
|
12077
|
+
const registryPath = config.registryPath;
|
|
12078
|
+
const pinnedRepos = new Set(config.pinnedRepos);
|
|
12079
|
+
const gitRun = context.gitRunner ?? ((file, args, opts) => run(file, [...args], { ...(opts?.cwd === undefined ? {} : { cwd: opts.cwd }), timeoutMs: 10_000 }));
|
|
12080
|
+
const proveRepo = async (input) => {
|
|
12081
|
+
const top = await gitRun("git", ["rev-parse", "--show-toplevel"], { cwd: resolve(input) });
|
|
12082
|
+
if (top.code !== 0)
|
|
12083
|
+
return null;
|
|
12084
|
+
try {
|
|
12085
|
+
return realpathSync(top.stdout.trim());
|
|
12086
|
+
}
|
|
12087
|
+
catch {
|
|
12088
|
+
return null;
|
|
12089
|
+
}
|
|
12090
|
+
};
|
|
12091
|
+
// The proved set this worker builds; `up` relays each change to the console.
|
|
12092
|
+
const activeRepos = new Set(config.repos);
|
|
12093
|
+
const reportRepos = () => port.send({ type: "repos", repos: [...activeRepos] });
|
|
12094
|
+
// The historical signal shell: the first stop ends admission and starts the grace clock; the second — or the
|
|
12095
|
+
// clock — stops every live provider group (M6.12).
|
|
12096
|
+
let stopping = false;
|
|
12097
|
+
let fatal = null;
|
|
12098
|
+
let graceTimer;
|
|
12099
|
+
const followControllers = [];
|
|
12100
|
+
let resolveStopped = () => { };
|
|
12101
|
+
const stopped = new Promise(resolveStop => (resolveStopped = resolveStop));
|
|
12102
|
+
const hardStop = () => {
|
|
12103
|
+
const terminated = terminateLiveProviders();
|
|
12104
|
+
if (terminated > 0)
|
|
12105
|
+
progress(`Stopped ${terminated} running build(s). Their work is saved, and each resumes when the builder starts again.`);
|
|
12106
|
+
};
|
|
12107
|
+
const stop = () => {
|
|
12108
|
+
if (stopping) {
|
|
12109
|
+
hardStop();
|
|
12110
|
+
return;
|
|
12111
|
+
}
|
|
12112
|
+
stopping = true;
|
|
12113
|
+
resolveStopped();
|
|
12114
|
+
for (const controller of followControllers)
|
|
12115
|
+
controller.abort();
|
|
12116
|
+
const building = store.liveClaimCount(runnerName, clock());
|
|
12117
|
+
if (building > 0) {
|
|
12118
|
+
progress(`Stopping: ${building} build(s) running. Waiting up to ${Math.round(config.stopGraceMs / 1000)}s for them to finish; any still running then is saved and resumes when the builder starts again.`);
|
|
12119
|
+
}
|
|
12120
|
+
graceTimer = setTimeout(hardStop, config.stopGraceMs);
|
|
12121
|
+
graceTimer.unref?.();
|
|
12122
|
+
};
|
|
12123
|
+
port.onStop(hard => {
|
|
12124
|
+
if (!stopping)
|
|
12125
|
+
stop();
|
|
12126
|
+
if (hard)
|
|
12127
|
+
hardStop();
|
|
12128
|
+
});
|
|
12129
|
+
const loopFlagsFor = (repo) => {
|
|
12130
|
+
const copy = new Map();
|
|
12131
|
+
// A bounded `up` must not advertise its child watches as an unbounded sweep service either.
|
|
12132
|
+
if (config.forMs !== null)
|
|
12133
|
+
copy.set("for", String(config.forMs));
|
|
12134
|
+
copy.set("runner", runnerName);
|
|
12135
|
+
copy.set("token", runnerToken);
|
|
12136
|
+
copy.set("repo", repo);
|
|
12137
|
+
copy.set("pool", pool);
|
|
12138
|
+
// The co-hosted console's stated origin rides into each loop's follower: a phone link must name THIS console, or not exist.
|
|
12139
|
+
if (config.publicUrl !== null)
|
|
12140
|
+
copy.set("public-url", config.publicUrl);
|
|
12141
|
+
return copy;
|
|
12142
|
+
};
|
|
12143
|
+
const prefix = (repo) => (activeRepos.size > 1 ? `[${projectName(repo)}] ` : "");
|
|
12144
|
+
// Containment at start: one plain sentence, once, for a person at a terminal; the exact status line with --verbose,
|
|
12145
|
+
// in a service log (no terminal), and always when a required policy will refuse spawns.
|
|
12146
|
+
const containmentDetail = config.verbose || !config.terminal;
|
|
12147
|
+
let containmentSaid = false;
|
|
12148
|
+
const containmentLines = (effective) => {
|
|
12149
|
+
const notice = containmentNotice(effective);
|
|
12150
|
+
if (notice !== null && !containmentSaid)
|
|
12151
|
+
progress(notice);
|
|
12152
|
+
containmentSaid = true;
|
|
12153
|
+
return containmentDetail || effective.refusal !== null ? [`watch: ${describeContainment(effective)}`] : [];
|
|
12154
|
+
};
|
|
12155
|
+
const loopResults = new Map();
|
|
12156
|
+
// One record of project passes for the whole service: fair scheduling across its projects (project-concurrency.ts).
|
|
12157
|
+
const projectPasses = new ProjectPasses();
|
|
12158
|
+
// Projects removed while running: each one's watch stops admitting work, finishes what it holds, then lets go.
|
|
12159
|
+
const releasedRepos = new Set();
|
|
12160
|
+
const launchRepo = async (repo, bind, drain = false) => {
|
|
12161
|
+
if (loopResults.has(repo) || stopping)
|
|
12162
|
+
return;
|
|
12163
|
+
// A predecessor's watch on a project this start doesn't name (it was being removed): take it over, recovering what
|
|
12164
|
+
// the predecessor held, and let it go (round 1, finding 2).
|
|
12165
|
+
if (drain)
|
|
12166
|
+
releasedRepos.add(repo);
|
|
12167
|
+
if (bind) {
|
|
12168
|
+
const bound = addRunnerReposAuthed(store, { name: runnerName, token: runnerToken, repos: [repo] }, clock());
|
|
12169
|
+
if (!bound.ok) {
|
|
12170
|
+
fatal = `the builder identity changed while adding ${repo} (${bound.reason})`;
|
|
12171
|
+
stop();
|
|
12172
|
+
return;
|
|
12173
|
+
}
|
|
12174
|
+
activeRepos.add(repo);
|
|
12175
|
+
reportRepos();
|
|
12176
|
+
if (!store.listProjects().some(one => one.path === repo)) {
|
|
12177
|
+
store.upsertProject(repo, projectName(repo), clock());
|
|
12178
|
+
}
|
|
12179
|
+
}
|
|
12180
|
+
let markReady = () => { };
|
|
12181
|
+
const ready = new Promise(resolveReady => (markReady = resolveReady));
|
|
12182
|
+
const loop = runWatchLoop({
|
|
12183
|
+
flags: loopFlagsFor(repo),
|
|
12184
|
+
context,
|
|
12185
|
+
runner: runnerName,
|
|
12186
|
+
token: runnerToken,
|
|
12187
|
+
repo,
|
|
12188
|
+
progress: line => progress(`${prefix(repo)}${line}`),
|
|
12189
|
+
isStopping: () => stopping || releasedRepos.has(repo),
|
|
12190
|
+
onFollowController: controller => followControllers.push(controller),
|
|
12191
|
+
containmentLines,
|
|
12192
|
+
passes: projectPasses,
|
|
12193
|
+
serviceBridge: true,
|
|
12194
|
+
leaseWaitMs: config.leaseWaitMs,
|
|
12195
|
+
...(drain ? { leaseWaitStopping: () => stopping } : {}),
|
|
12196
|
+
onReady: () => {
|
|
12197
|
+
markReady();
|
|
12198
|
+
if (bind)
|
|
12199
|
+
progress(`${prefix(repo)}builder connected — queued work can start`);
|
|
12200
|
+
},
|
|
12201
|
+
}).then(result => ({ repo, result }), error => ({ repo, result: { ok: false, reason: "loop-failed", detail: describe(error), ticks: 0, built: 0, broke: 0 } }));
|
|
12202
|
+
loopResults.set(repo, loop);
|
|
12203
|
+
void loop.then(({ repo: endedRepo, result }) => {
|
|
12204
|
+
if (releasedRepos.has(endedRepo) && !stopping && !result.ok) {
|
|
12205
|
+
// A removed project's watch that never held its lease (or lost it) has recovered nothing: keep the project
|
|
12206
|
+
// bound so the next worker recovers it, and stop rather than strand its work (round 1, finding 2).
|
|
12207
|
+
fatal = `${prefix(endedRepo)}${result.detail}`;
|
|
12208
|
+
stop();
|
|
12209
|
+
markReady();
|
|
12210
|
+
return;
|
|
12211
|
+
}
|
|
12212
|
+
if (releasedRepos.has(endedRepo) && !stopping) {
|
|
12213
|
+
// A removed project's watch has stopped: only now does the builder let go of it.
|
|
12214
|
+
const unbound = removeRunnerRepoAuthed(store, { name: runnerName, token: runnerToken, repo: endedRepo }, clock());
|
|
12215
|
+
if (!unbound.ok) {
|
|
12216
|
+
fatal = `the builder identity changed while removing ${endedRepo} (${unbound.reason})`;
|
|
12217
|
+
stop();
|
|
12218
|
+
}
|
|
12219
|
+
loopResults.delete(endedRepo);
|
|
12220
|
+
releasedRepos.delete(endedRepo);
|
|
12221
|
+
progress(`project removed — ${projectName(endedRepo)} is no longer built here`);
|
|
12222
|
+
markReady();
|
|
12223
|
+
return;
|
|
12224
|
+
}
|
|
12225
|
+
// A loop that dies while its siblings live must not leave a partial cockpit standing silently (finding 5).
|
|
12226
|
+
if (!result.ok && !stopping) {
|
|
12227
|
+
fatal = `${prefix(endedRepo)}${result.detail}`;
|
|
12228
|
+
stop();
|
|
12229
|
+
}
|
|
12230
|
+
markReady(); // acquisition failures must not strand startup readiness
|
|
12231
|
+
});
|
|
12232
|
+
await ready;
|
|
12233
|
+
};
|
|
12234
|
+
// Before any new work: every watch a predecessor left needing recovery on a project this start doesn't name — one
|
|
12235
|
+
// being removed when it died — is taken over (its incarnation recovered in that step) and let go. Only projects the
|
|
12236
|
+
// store still binds to this runner; the start's own projects recover in their own loops below (round 1, finding 2).
|
|
12237
|
+
const named = new Set(config.repos);
|
|
12238
|
+
const boundNow = new Set(store.getRunner(runnerName)?.runner.repos ?? []);
|
|
12239
|
+
const draining = [...new Set(predecessorWatches(store, runnerName, clock()).map(one => one.repo))].filter(repo => !named.has(repo) && boundNow.has(repo));
|
|
12240
|
+
for (const repo of draining)
|
|
12241
|
+
progress(`${projectName(repo)} was being removed when the previous worker stopped: recovering its work first`);
|
|
12242
|
+
await Promise.all(draining.map(repo => launchRepo(repo, false, true)));
|
|
12243
|
+
await Promise.all(config.repos.map(repo => launchRepo(repo, false)));
|
|
12244
|
+
// Telegram: one follower for the whole service, never one per project loop. Replies and notifications go out from
|
|
12245
|
+
// it, over the same transport in the same cycle. A token saved while the service runs connects within seconds; a
|
|
12246
|
+
// follower that dies is started again.
|
|
12247
|
+
const telegramController = new AbortController();
|
|
12248
|
+
followControllers.push(telegramController);
|
|
12249
|
+
if (stopping)
|
|
12250
|
+
telegramController.abort();
|
|
12251
|
+
const telegramSupervisor = (async () => {
|
|
12252
|
+
const pause = (ms) => new Promise(resolvePause => {
|
|
12253
|
+
const timer = setTimeout(done, ms);
|
|
12254
|
+
function done() { clearTimeout(timer); telegramController.signal.removeEventListener("abort", done); resolvePause(); }
|
|
12255
|
+
// An aborted signal never fires again: without this a stop waited the whole pause.
|
|
12256
|
+
if (telegramController.signal.aborted)
|
|
12257
|
+
done();
|
|
12258
|
+
else
|
|
12259
|
+
telegramController.signal.addEventListener("abort", done, { once: true });
|
|
12260
|
+
});
|
|
12261
|
+
while (!stopping && !telegramController.signal.aborted) {
|
|
12262
|
+
const follower = startTelegramFollower({
|
|
12263
|
+
context, publicUrl: config.publicUrl ?? undefined, signal: telegramController.signal, progress, label: "telegram:",
|
|
12264
|
+
died: { prefix: "", then: "starting it again" }, owner: `service-${randomUUID()}`,
|
|
12265
|
+
});
|
|
12266
|
+
if (follower !== null)
|
|
12267
|
+
await follower;
|
|
12268
|
+
// Without a token (or on a fenced network) look again shortly; the fence line is said once.
|
|
12269
|
+
if (follower === null && loadBotToken(process.env, context.telegramTokenFile) !== null)
|
|
12270
|
+
break;
|
|
12271
|
+
await pause(follower === null ? 2_000 : 5_000);
|
|
12272
|
+
}
|
|
12273
|
+
})();
|
|
12274
|
+
const dynamicCeiling = resolveCeiling(config.repos, config.projectRoots).ceiling;
|
|
12275
|
+
const rejected = new Set();
|
|
12276
|
+
const registrySupervisor = (async () => {
|
|
12277
|
+
const sleep = (ms) => new Promise(resolveSleep => setTimeout(resolveSleep, ms));
|
|
12278
|
+
while (!stopping) {
|
|
12279
|
+
const loaded = await loadProjectRegistry(registryPath);
|
|
12280
|
+
if ("error" in loaded) {
|
|
12281
|
+
fatal = loaded.error;
|
|
12282
|
+
stop();
|
|
12283
|
+
break;
|
|
12284
|
+
}
|
|
12285
|
+
// Removed from the saved list: stop building it and take it out of the console. A pinned start keeps its own.
|
|
12286
|
+
const saved = new Set(loaded.repos.map(one => canonicalProject(one) ?? one));
|
|
12287
|
+
let removed = false;
|
|
12288
|
+
for (const repo of [...activeRepos]) {
|
|
12289
|
+
if (stopping || pinnedRepos.has(repo) || saved.has(repo))
|
|
12290
|
+
continue;
|
|
12291
|
+
activeRepos.delete(repo);
|
|
12292
|
+
removed = true;
|
|
12293
|
+
releasedRepos.add(repo);
|
|
12294
|
+
if (!loopResults.has(repo))
|
|
12295
|
+
releasedRepos.delete(repo);
|
|
12296
|
+
}
|
|
12297
|
+
if (removed)
|
|
12298
|
+
reportRepos();
|
|
12299
|
+
for (const candidate of loaded.repos) {
|
|
12300
|
+
if (stopping)
|
|
12301
|
+
break;
|
|
12302
|
+
const canonical = canonicalProject(candidate);
|
|
12303
|
+
// Still letting go of an earlier removal: it rejoins on the next pass after its watch has stopped.
|
|
12304
|
+
if (canonical !== null && (activeRepos.has(canonical) || releasedRepos.has(canonical)))
|
|
12305
|
+
continue;
|
|
12306
|
+
const approvedAdditions = context.additionalProjectRepos?.() ?? [];
|
|
12307
|
+
const admittedCeiling = resolveCeiling([...dynamicCeiling.repos, ...approvedAdditions], dynamicCeiling.roots).ceiling;
|
|
12308
|
+
// A project added after this start (`repos add`, the lead, the console) carries its recorded admission; a
|
|
12309
|
+
// registry line alone is not authority. Either way the folder is proved to be that Git checkout again.
|
|
12310
|
+
const allowed = canonical !== null &&
|
|
12311
|
+
((await authorizedProject(admittedCeiling, canonical)) || admissionRecorded(store, canonical)) &&
|
|
12312
|
+
(await proveRepo(canonical)) === canonical;
|
|
12313
|
+
if (!allowed) {
|
|
12314
|
+
if (!rejected.has(candidate)) {
|
|
12315
|
+
rejected.add(candidate);
|
|
12316
|
+
progress(`project not connected — ${candidate} isn't a Git repository added with \`toolroll repos add\` or the console`);
|
|
12317
|
+
}
|
|
12318
|
+
continue;
|
|
12319
|
+
}
|
|
12320
|
+
rejected.delete(candidate);
|
|
12321
|
+
await launchRepo(canonical, true);
|
|
12322
|
+
}
|
|
12323
|
+
if (!stopping)
|
|
12324
|
+
await sleep(500);
|
|
12325
|
+
}
|
|
12326
|
+
})().catch(error => {
|
|
12327
|
+
fatal = `the project registry stopped (${describe(error)})`;
|
|
12328
|
+
stop();
|
|
12329
|
+
});
|
|
12330
|
+
// Provider readiness (v47): this machine's non-spending observations, recorded under its own runner name ONCE at
|
|
12331
|
+
// start — so every route projection can say ready / unavailable / unknown per provider, and the dispatch gate can
|
|
12332
|
+
// halt a leg this machine cannot run. Production only: a test's injected agent runner is a fake, and its readiness
|
|
12333
|
+
// would be a fiction.
|
|
12334
|
+
if (context.agentRunner === undefined) {
|
|
12335
|
+
void observeProviderReadiness((file, args, options) => run(file, args, { timeoutMs: options?.timeoutMs ?? 5_000, ...(options?.omitEnv === undefined ? {} : { omitEnv: options.omitEnv }) }), process.env, PROVIDER_IDS, provider => store.acpAttestationFor(provider, runnerName))
|
|
12336
|
+
.then(observed => {
|
|
12337
|
+
if (!stopping)
|
|
12338
|
+
reportProviderReadinessAuthed(store, { name: runnerName, token: runnerToken, observations: observed }, clock());
|
|
12339
|
+
})
|
|
12340
|
+
.catch(() => {
|
|
12341
|
+
// A probe that cannot run leaves readiness unknown — never a fiction.
|
|
12342
|
+
});
|
|
12343
|
+
}
|
|
12344
|
+
try {
|
|
12345
|
+
notifyVersionLive(context.store, dirname(context.databaseFile), PACKAGE_VERSION, clock());
|
|
12346
|
+
}
|
|
12347
|
+
catch { /* A display fact; never blocks the worker. */ }
|
|
12348
|
+
recordRunnerVersion(dirname(context.databaseFile), runnerName, PACKAGE_VERSION, clock());
|
|
12349
|
+
// v137: the continuous database copy, off the hot path (live-backup.ts: a worker thread copies, checks and hashes).
|
|
12350
|
+
// Never inside a project or a build checkout. A test's injected agent runner means a test: no background copies.
|
|
12351
|
+
const stopLiveBackups = context.agentRunner === undefined
|
|
12352
|
+
? startLiveBackups(store, context.databaseFile, () => [...new Set([...activeRepos, ...store.knownRepos(), pool])])
|
|
12353
|
+
: null;
|
|
12354
|
+
// Ready once every project's loop holds its lease; a loop that failed to start ends this worker instead.
|
|
12355
|
+
if (fatal === null && !stopping)
|
|
12356
|
+
port.send({ type: "ready", role: "worker", build, pid: process.pid, url: null, port: null });
|
|
12357
|
+
await stopped;
|
|
12358
|
+
await registrySupervisor;
|
|
12359
|
+
await Promise.all([...loopResults.values()]);
|
|
12360
|
+
await stopLiveBackups?.();
|
|
12361
|
+
// The follower was aborted with the stop. A send or a chat turn it is still finishing keeps its own durable record;
|
|
12362
|
+
// the service does not wait past this for it.
|
|
12363
|
+
await new Promise(done => {
|
|
12364
|
+
const timer = setTimeout(done, TELEGRAM_STOP_WAIT_MS);
|
|
12365
|
+
const finish = () => { clearTimeout(timer); done(); };
|
|
12366
|
+
void telegramSupervisor.then(finish, finish);
|
|
12367
|
+
});
|
|
12368
|
+
if (graceTimer !== undefined)
|
|
12369
|
+
clearTimeout(graceTimer);
|
|
12370
|
+
if (fatal !== null) {
|
|
12371
|
+
port.send({ type: "fatal", detail: fatal });
|
|
12372
|
+
return EXIT.failed;
|
|
12373
|
+
}
|
|
12374
|
+
return EXIT.ok;
|
|
12375
|
+
}
|