tickmarkr 2.6.6 → 2.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-code.js +8 -2
- package/dist/adapters/codex-commit-check.d.ts +3 -2
- package/dist/adapters/codex-commit-check.js +37 -10
- package/dist/adapters/codex.js +10 -4
- package/dist/cli/commands/beat.d.ts +6 -1
- package/dist/cli/commands/beat.js +123 -76
- package/dist/cli/commands/fleet.js +64 -21
- package/dist/cli/commands/init.js +18 -7
- package/dist/cli/help.d.ts +1 -0
- package/dist/cli/help.js +10 -5
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +1 -1
- package/dist/gates/baseline.d.ts +9 -0
- package/dist/gates/baseline.js +14 -3
- package/dist/gates/cache.js +1 -1
- package/dist/gates/review.js +15 -0
- package/dist/gates/run-gates.d.ts +18 -6
- package/dist/gates/run-gates.js +324 -204
- package/dist/gates/test-manifest.d.ts +8 -0
- package/dist/gates/test-manifest.js +151 -111
- package/dist/run/beat-lifecycle.d.ts +154 -0
- package/dist/run/beat-lifecycle.js +825 -0
- package/dist/run/daemon.js +18 -7
- package/dist/run/git-trust.d.ts +27 -0
- package/dist/run/git-trust.js +178 -0
- package/dist/run/git.js +34 -14
- package/dist/run/journal.js +15 -3
- package/dist/run/lease.d.ts +23 -0
- package/dist/run/lease.js +232 -38
- package/dist/run/merge.js +2 -2
- package/dist/run/stall.js +20 -1
- package/dist/run/supervision.d.ts +8 -0
- package/dist/run/supervision.js +24 -4
- package/dist/run/verification-job.d.ts +51 -0
- package/dist/run/verification-job.js +137 -0
- package/dist/tui/ink/fleet-app.d.ts +9 -0
- package/dist/tui/ink/fleet-app.js +209 -127
- package/package.json +1 -1
- package/skills/tickmarkr-auto/SKILL.md +19 -10
- package/skills/tickmarkr-loop/SKILL.md +19 -10
- package/skills/tickmarkr-overseer/SKILL.md +25 -16
|
@@ -0,0 +1,825 @@
|
|
|
1
|
+
import { spawn as nodeSpawn, spawnSync } from "node:child_process";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { closeSync, linkSync, mkdirSync, openSync, readFileSync, readlinkSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { SUPERVISION_BEAT_MS, isSeatTier, publishStandDown, readSupervisionArm, supervisionArmPath, supervisionBeatPath, supervisionStatus, } from "./supervision.js";
|
|
6
|
+
// H + D-1001: the detached beat is OWNED by the product, not by a shell recipe. One claim file per
|
|
7
|
+
// (repository, tier) serializes start, stop and the legacy arm-mutating writers; one owner record names
|
|
8
|
+
// the exact generation (seat, armId, pid, process birth/argv/cwd) that start launched. Every signal and
|
|
9
|
+
// removal rechecks that generation first and reads the result back afterwards, so an older invocation can
|
|
10
|
+
// never retire a replacement, and a pid that now belongs to another process is a mismatch, never a target.
|
|
11
|
+
// Status takes no claim and writes nothing. Uncertain state is RETAINED and refused (nonzero), never
|
|
12
|
+
// guessed: a stale beat is not proof of death and a foreign or stale claim is never stolen.
|
|
13
|
+
/** start's whole budget — launch, read-back and any overrun — before it must refuse and roll back. */
|
|
14
|
+
export const BEAT_STARTUP_CEILING_MS = 60_000;
|
|
15
|
+
/** The generation a started child carries; it lets the child act under its launcher's claim only. */
|
|
16
|
+
export const BEAT_GENERATION_ENV = "TICKMARKR_BEAT_GENERATION";
|
|
17
|
+
export function defaultBeatDeps() {
|
|
18
|
+
return {
|
|
19
|
+
cli: [process.execPath, ...process.execArgv, process.argv[1] ?? ""],
|
|
20
|
+
spawn: nodeSpawn,
|
|
21
|
+
kill: (pid, signal) => { process.kill(pid, signal); },
|
|
22
|
+
identity: readProcessIdentity,
|
|
23
|
+
openLog: (path) => openSync(path, "a"),
|
|
24
|
+
now: Date.now,
|
|
25
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
26
|
+
env: process.env,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
/** A named, nonzero refusal: dispatch renders the message and exits 1. */
|
|
30
|
+
export class BeatRefusal extends Error {
|
|
31
|
+
}
|
|
32
|
+
function refuse(message) { throw new BeatRefusal(message); }
|
|
33
|
+
/**
|
|
34
|
+
* The live process behind a pid: birth (lstart), full argv, cwd and process group. A missing process is
|
|
35
|
+
* DEAD only when the kernel says so (ESRCH or a zombie); anything unreadable is UNKNOWN, never DEAD.
|
|
36
|
+
*/
|
|
37
|
+
export function readProcessIdentity(pid) {
|
|
38
|
+
try {
|
|
39
|
+
process.kill(pid, 0);
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
return error.code === "ESRCH" ? "DEAD" : "UNKNOWN";
|
|
43
|
+
}
|
|
44
|
+
const ps = spawnSync("ps", ["-ww", "-o", "pgid=", "-o", "stat=", "-o", "lstart=", "-o", "command=", "-p", String(pid)], {
|
|
45
|
+
encoding: "utf8", env: { ...process.env, LC_ALL: "C" },
|
|
46
|
+
});
|
|
47
|
+
const row = /^\s*(\d+)\s+(\S+)\s+(\w{3} \w{3} [ \d]\d \d\d:\d\d:\d\d \d{4})\s+(.*\S)\s*$/.exec(ps.stdout ?? "");
|
|
48
|
+
if (ps.status !== 0 || !row) {
|
|
49
|
+
try {
|
|
50
|
+
process.kill(pid, 0);
|
|
51
|
+
return "UNKNOWN";
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
return error.code === "ESRCH" ? "DEAD" : "UNKNOWN";
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (row[2].startsWith("Z"))
|
|
58
|
+
return "DEAD";
|
|
59
|
+
let cwd;
|
|
60
|
+
try {
|
|
61
|
+
if (process.platform === "linux")
|
|
62
|
+
cwd = readlinkSync(`/proc/${pid}/cwd`);
|
|
63
|
+
else {
|
|
64
|
+
const lsof = spawnSync("lsof", ["-a", "-p", String(pid), "-d", "cwd", "-Fn"], { encoding: "utf8" });
|
|
65
|
+
cwd = lsof.stdout?.split("\n").find((line) => line.startsWith("n"))?.slice(1);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
cwd = undefined;
|
|
70
|
+
}
|
|
71
|
+
return cwd ? { pgid: Number(row[1]), birth: row[3], command: row[4], cwd } : "UNKNOWN";
|
|
72
|
+
}
|
|
73
|
+
const supervisionDirOf = (repoRoot, tier) => dirname(supervisionBeatPath(repoRoot, tier));
|
|
74
|
+
export const beatClaimPath = (repoRoot, tier) => join(supervisionDirOf(repoRoot, tier), `${tier}.claim`);
|
|
75
|
+
export const beatOwnerPath = (repoRoot, tier) => join(supervisionDirOf(repoRoot, tier), `${tier}.owner`);
|
|
76
|
+
export const beatLogPath = (repoRoot, tier) => join(supervisionDirOf(repoRoot, tier), `${tier}.log`);
|
|
77
|
+
const OWNER_CHANGED = "the owner record changed during inspection";
|
|
78
|
+
function readJson(path) {
|
|
79
|
+
let bytes;
|
|
80
|
+
try {
|
|
81
|
+
bytes = readFileSync(path, "utf8");
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
const code = error.code;
|
|
85
|
+
return code === "ENOENT" || code === "ENOTDIR" ? "NONE" : "UNREADABLE";
|
|
86
|
+
}
|
|
87
|
+
try {
|
|
88
|
+
const value = JSON.parse(bytes);
|
|
89
|
+
return value && typeof value === "object" ? value : "UNREADABLE";
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return "UNREADABLE";
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const text = (v) => typeof v === "string" && v.trim() !== "";
|
|
96
|
+
const optional = (v, ok) => v === undefined || ok(v);
|
|
97
|
+
const positive = (v) => Number.isInteger(v) && v > 0;
|
|
98
|
+
const percent = (v) => typeof v === "number" && v >= 0 && v <= 100;
|
|
99
|
+
/** The beat record's own schema (what writeSupervisionBeat publishes), checked independently of any marker. */
|
|
100
|
+
const validBeat = (rec, tier) => rec.tier === tier && text(rec.beatAt) && !Number.isNaN(Date.parse(rec.beatAt)) &&
|
|
101
|
+
(isSeatTier(tier) ? text(rec.seat) : optional(rec.seat, text)) && optional(rec.armId, text) &&
|
|
102
|
+
optional(rec.armEpoch, (v) => text(v) && !Number.isNaN(Date.parse(v))) && optional(rec.markerFence, (v) => typeof v === "string") &&
|
|
103
|
+
optional(rec.pct, percent) && optional(rec.thresholdPct, percent) &&
|
|
104
|
+
(rec.pid === undefined ? positive(rec.exitedWriterPid) : positive(rec.pid) && rec.exitedWriterPid === undefined);
|
|
105
|
+
export function readBeatOwner(repoRoot, tier) {
|
|
106
|
+
const rec = readJson(beatOwnerPath(repoRoot, tier));
|
|
107
|
+
if (typeof rec !== "object")
|
|
108
|
+
return rec;
|
|
109
|
+
const launched = rec.pid === undefined && rec.birth === undefined && rec.command === undefined && rec.pgid === undefined;
|
|
110
|
+
const running = positive(rec.pid) && text(rec.birth) && text(rec.command) && positive(rec.pgid);
|
|
111
|
+
if (rec.tier !== tier || !text(rec.seat) || !text(rec.generation) || !text(rec.armId) || !text(rec.cwd) ||
|
|
112
|
+
!text(rec.startedAt) || Number.isNaN(Date.parse(rec.startedAt)) || (rec.priorArmId !== undefined && !text(rec.priorArmId)) || (!launched && !running) ||
|
|
113
|
+
[rec.armEpoch, rec.priorArmEpoch].some((epoch) => epoch !== undefined && (!text(epoch) || Number.isNaN(Date.parse(epoch)))))
|
|
114
|
+
return "UNREADABLE";
|
|
115
|
+
return rec;
|
|
116
|
+
}
|
|
117
|
+
function readClaim(repoRoot, tier) {
|
|
118
|
+
const rec = readJson(beatClaimPath(repoRoot, tier));
|
|
119
|
+
if (typeof rec !== "object")
|
|
120
|
+
return rec;
|
|
121
|
+
return text(rec.token) && Number.isInteger(rec.pid) ? { token: rec.token, pid: rec.pid } : "UNREADABLE";
|
|
122
|
+
}
|
|
123
|
+
function publish(path, value) {
|
|
124
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
125
|
+
const tmp = `${path}.${randomUUID()}.tmp`;
|
|
126
|
+
try {
|
|
127
|
+
writeFileSync(tmp, JSON.stringify(value) + "\n");
|
|
128
|
+
renameSync(tmp, path);
|
|
129
|
+
}
|
|
130
|
+
finally {
|
|
131
|
+
rmSync(tmp, { force: true });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/** Atomic first-writer-wins claim; a held claim is BUSY and is never taken over. */
|
|
135
|
+
export function acquireBeatClaim(repoRoot, tier, token) {
|
|
136
|
+
const path = beatClaimPath(repoRoot, tier);
|
|
137
|
+
const tmp = `${path}.${token}.tmp`;
|
|
138
|
+
try {
|
|
139
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
140
|
+
writeFileSync(tmp, JSON.stringify({ tier, token, pid: process.pid, claimedAt: new Date().toISOString() }) + "\n");
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
try {
|
|
144
|
+
rmSync(tmp, { force: true });
|
|
145
|
+
}
|
|
146
|
+
catch { /* the path that failed holds nothing to remove */ }
|
|
147
|
+
refuse(`${tier} claim failed at ${path}: ${error.message}`);
|
|
148
|
+
}
|
|
149
|
+
try {
|
|
150
|
+
linkSync(tmp, path);
|
|
151
|
+
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
if (error.code !== "EEXIST")
|
|
154
|
+
refuse(`${tier} claim failed at ${path}: ${error.message}`);
|
|
155
|
+
const holder = readClaim(repoRoot, tier);
|
|
156
|
+
refuse(`${tier} BUSY — ${typeof holder === "object" ? `pid ${holder.pid} holds` : "an unreadable claim occupies"} ${path}; ` +
|
|
157
|
+
"retry after it finishes (a claim is never taken over automatically)");
|
|
158
|
+
}
|
|
159
|
+
finally {
|
|
160
|
+
rmSync(tmp, { force: true });
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
export function releaseBeatClaim(repoRoot, tier, token) {
|
|
164
|
+
const claim = readClaim(repoRoot, tier);
|
|
165
|
+
if (typeof claim !== "object" || claim.token !== token)
|
|
166
|
+
refuse(`${tier} claim changed before release — left in place`);
|
|
167
|
+
try {
|
|
168
|
+
rmSync(beatClaimPath(repoRoot, tier));
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
refuse(`${tier} claim release failed: ${error.message}`);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Run one legacy write (one-shot tick, new arm, EVERY loop tick, stand-down) under the tier claim. A launched
|
|
176
|
+
* child acts under its launcher's claim while that claim is still held for its generation; afterwards it takes
|
|
177
|
+
* the tier claim itself like every other writer. `act` receives the recheck it must call immediately before it
|
|
178
|
+
* writes: the claim still names this holder and the tier's ownership still admits this writer. A claim held by
|
|
179
|
+
* anyone else refuses with nothing written — `waits` only lets a running loop's tick outlast a brief holder.
|
|
180
|
+
*/
|
|
181
|
+
export async function withLegacyClaim(repoRoot, tier, writer, act, waits = { tries: 1, sleep: async () => { } }) {
|
|
182
|
+
const { generation, settled = false } = writer;
|
|
183
|
+
const held = (token) => {
|
|
184
|
+
// The exact owner this claim's previous recheck validated: the generation every later recheck is held to.
|
|
185
|
+
let checked;
|
|
186
|
+
return (arm = writer.arm, created) => {
|
|
187
|
+
const claim = readClaim(repoRoot, tier);
|
|
188
|
+
if (typeof claim !== "object" || claim.token !== token)
|
|
189
|
+
refuse(`${tier} claim is no longer held by this writer — nothing written`);
|
|
190
|
+
checked = assertLegacyOwnership(repoRoot, tier, writer, arm, created, checked);
|
|
191
|
+
};
|
|
192
|
+
};
|
|
193
|
+
if (generation !== undefined) {
|
|
194
|
+
const claim = readClaim(repoRoot, tier);
|
|
195
|
+
if (typeof claim === "object" && claim.token === generation) {
|
|
196
|
+
const recheck = held(generation);
|
|
197
|
+
recheck();
|
|
198
|
+
return act(recheck);
|
|
199
|
+
}
|
|
200
|
+
if (!settled)
|
|
201
|
+
refuse(`${tier} superseded — the launch claim for generation ${generation} is gone`);
|
|
202
|
+
}
|
|
203
|
+
const token = randomUUID();
|
|
204
|
+
for (let attempt = 1;; attempt++) {
|
|
205
|
+
try {
|
|
206
|
+
acquireBeatClaim(repoRoot, tier, token);
|
|
207
|
+
break;
|
|
208
|
+
}
|
|
209
|
+
catch (error) {
|
|
210
|
+
if (attempt >= waits.tries || !(error instanceof BeatRefusal) || !error.message.includes(" BUSY — "))
|
|
211
|
+
throw error;
|
|
212
|
+
await waits.sleep(250);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
let result;
|
|
216
|
+
try {
|
|
217
|
+
const recheck = held(token);
|
|
218
|
+
recheck();
|
|
219
|
+
result = await act(recheck);
|
|
220
|
+
}
|
|
221
|
+
finally {
|
|
222
|
+
releaseBeatClaim(repoRoot, tier, token);
|
|
223
|
+
}
|
|
224
|
+
return result;
|
|
225
|
+
}
|
|
226
|
+
/** The ownership a legacy writer needs before each mutation: its own COMPLETE launched generation, or no owner at all. */
|
|
227
|
+
export function assertLegacyOwnership(repoRoot, tier, writer, arm, created = false, held) {
|
|
228
|
+
if (writer.generation !== undefined)
|
|
229
|
+
return assertOwnedGeneration(repoRoot, tier, { ...writer, generation: writer.generation }, arm, created, held);
|
|
230
|
+
const owner = readBeatOwner(repoRoot, tier);
|
|
231
|
+
if (owner !== "NONE") {
|
|
232
|
+
refuse(`${tier} is owned by a detached beat (${typeof owner === "object" ? `seat ${owner.seat}, generation ${owner.generation}` : "unreadable owner record"})` +
|
|
233
|
+
` — run \`tickmarkr beat stop ${tier} --seat <seat>\` first`);
|
|
234
|
+
}
|
|
235
|
+
return undefined;
|
|
236
|
+
}
|
|
237
|
+
const sameOwner = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
|
238
|
+
/**
|
|
239
|
+
* The ONE owner change a writer accepts from someone else: its launcher completing an UNBOUND owner — publishing
|
|
240
|
+
* this writer's own pid and/or binding the epoch of the arm this writer itself holds. Verified field by field
|
|
241
|
+
* against the record the writer last checked; everything else the record says must be unchanged.
|
|
242
|
+
*/
|
|
243
|
+
function launcherBound(before, after, pid, arm) {
|
|
244
|
+
if (before.armEpoch !== undefined)
|
|
245
|
+
return false;
|
|
246
|
+
const fixed = (o) => JSON.stringify([o.tier, o.seat, o.generation, o.armId, o.cwd, o.startedAt, o.priorArmId, o.priorArmEpoch]);
|
|
247
|
+
const proc = (o) => JSON.stringify([o.pid, o.birth, o.command, o.pgid]);
|
|
248
|
+
const published = before.pid === undefined && after.pid === pid;
|
|
249
|
+
const bound = after.armEpoch !== undefined;
|
|
250
|
+
return fixed(before) === fixed(after) && (published || proc(before) === proc(after)) && (published || bound) &&
|
|
251
|
+
(!bound || (after.pid === pid && arm !== undefined && after.armId === arm.armId && after.armEpoch === arm.armEpoch));
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* ONE RULE: every legacy write is preceded by the shared complete owned-generation check of the CURRENT owner
|
|
255
|
+
* record and durable arm against the EXACT generation this writer holds — token, seat, pid (once settled), armId
|
|
256
|
+
* and armEpoch. `held` is the owner this claim's previous check validated and `arm` the arm the writer holds (a
|
|
257
|
+
* loop's own arm; before the launcher binds the epoch that arm is the evidence). The first check of a claim by a
|
|
258
|
+
* writer holding neither takes the generation on record; from then on the held generation changes in exactly
|
|
259
|
+
* two verified ways, and any other change — also one between this check's own reads — refuses with no write:
|
|
260
|
+
* (i) the writer's OWN re-arm (`created`: it published `arm` under this claim after the complete check): the
|
|
261
|
+
* arm on disk must be exactly that arm and the owner exactly the owner checked before the arm write; only
|
|
262
|
+
* then is the owner rebound to the arm (same token, seat, process), and the complete check runs AGAIN as
|
|
263
|
+
* the new generation before the beat. After any mismatch the owner is never rewritten.
|
|
264
|
+
* (ii) the launcher completing an UNBOUND owner for this writer (launcherBound) — never adopted from disk.
|
|
265
|
+
* Once `settled` (past its first beat) the record must name its pid: a launch that refused without proving it
|
|
266
|
+
* could signal its child leaves no pid behind, and that child retires itself.
|
|
267
|
+
*/
|
|
268
|
+
export function assertOwnedGeneration(repoRoot, tier, writer, arm = writer.arm, created = false, held, pid = process.pid) {
|
|
269
|
+
const { generation, seat, settled = false, deps } = writer;
|
|
270
|
+
const otherArm = (owner) => arm !== undefined && (arm.armId !== owner.armId || (owner.armEpoch !== undefined && arm.armEpoch !== owner.armEpoch));
|
|
271
|
+
const binds = (owner) => `${tier} superseded — the owner record binds arm ${owner.armId} (epoch ${owner.armEpoch ?? "unbound"}), not this writer's arm ${arm?.armId} (epoch ${arm?.armEpoch}); nothing written`;
|
|
272
|
+
const changed = (owner) => refuse(typeof owner === "object" && otherArm(owner) ? binds(owner)
|
|
273
|
+
: `${tier} superseded — the owner record changed under generation ${generation} while it was checked; nothing written`);
|
|
274
|
+
let expected = held;
|
|
275
|
+
for (let moved = false;; moved = true) {
|
|
276
|
+
const owner = readBeatOwner(repoRoot, tier);
|
|
277
|
+
if (owner === "UNREADABLE")
|
|
278
|
+
refuse(`${tier} owner record is unreadable — this loop cannot prove it is still owned`);
|
|
279
|
+
// The generation speaks for its recorded seat only: another seat carrying the same generation is foreign.
|
|
280
|
+
if (owner !== "NONE" && owner.generation === generation && owner.seat !== seat)
|
|
281
|
+
refuse(`${tier} is owned by seat ${owner.seat}, not ${seat} — nothing written`);
|
|
282
|
+
if (owner === "NONE" || owner.generation !== generation || (owner.pid !== undefined && owner.pid !== pid)) {
|
|
283
|
+
refuse(`${tier} superseded — the owner record no longer names generation ${generation}`);
|
|
284
|
+
}
|
|
285
|
+
if (settled && owner.pid === undefined)
|
|
286
|
+
refuse(`${tier} superseded — generation ${generation} never recorded pid ${pid}; its launch was abandoned`);
|
|
287
|
+
// `moved`: the shared check below saw the owner change between its reads — only the launcher's step passes.
|
|
288
|
+
if (expected !== undefined && (moved || !sameOwner(expected, owner)) && !launcherBound(expected, owner, pid, arm))
|
|
289
|
+
changed(owner);
|
|
290
|
+
expected = owner;
|
|
291
|
+
if (arm && created && owner.armEpoch !== undefined && otherArm(owner)) {
|
|
292
|
+
if (held === undefined)
|
|
293
|
+
refuse(`${tier} superseded — generation ${generation} re-armed without its complete check; owner left in place`);
|
|
294
|
+
const records = checkedRecords(repoRoot, tier, deps);
|
|
295
|
+
if (records.kind === "UNKNOWN")
|
|
296
|
+
refuse(`${tier} UNREADABLE — ${records.reason}; owner left in place`);
|
|
297
|
+
if (records.arm?.armId !== arm.armId || records.arm.armEpoch !== arm.armEpoch) {
|
|
298
|
+
refuse(`${tier} superseded — durable arm ${records.arm ? `${records.arm.armId} (epoch ${records.arm.armEpoch})` : "(absent)"} replaced the arm ${arm.armId} (epoch ${arm.armEpoch}) generation ${generation} just created; owner left in place`);
|
|
299
|
+
}
|
|
300
|
+
const current = readBeatOwner(repoRoot, tier);
|
|
301
|
+
if (!sameOwner(owner, current))
|
|
302
|
+
changed(current);
|
|
303
|
+
const next = { ...owner, armId: arm.armId, armEpoch: arm.armEpoch };
|
|
304
|
+
publish(beatOwnerPath(repoRoot, tier), next);
|
|
305
|
+
requireOwned(checkOwnedGeneration(repoRoot, tier, expectedGeneration(next), deps, undefined, true), tier);
|
|
306
|
+
return next;
|
|
307
|
+
}
|
|
308
|
+
const check = checkOwnedGeneration(repoRoot, tier, { owner, armEpoch: owner.armEpoch ?? arm?.armEpoch }, deps, undefined, true);
|
|
309
|
+
if (check.kind === "UNKNOWN" && check.reason === OWNER_CHANGED)
|
|
310
|
+
continue;
|
|
311
|
+
requireOwned(check, tier);
|
|
312
|
+
// Records that agree with each other are still another generation when they are not the arm this writer holds.
|
|
313
|
+
if (otherArm(owner))
|
|
314
|
+
refuse(binds(owner));
|
|
315
|
+
return owner;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
const sameIdentity = (a, id) => typeof id === "object" && id.birth === a.birth && id.command === a.command && id.cwd === a.cwd;
|
|
319
|
+
const sameProcess = (owner, id) => sameIdentity(owner, id);
|
|
320
|
+
/**
|
|
321
|
+
* Linux frees an exiting process's memory (its argv) before its cwd and before it is a zombie, so in that window ps prints
|
|
322
|
+
* `[comm]` beside the same birth, group and cwd. Only a wait for death may read that as "not gone yet": it never authorises a
|
|
323
|
+
* signal or a removal (both still need a full identity or checked death); any other difference stays a mismatch.
|
|
324
|
+
*/
|
|
325
|
+
const exitingAfterSignal = (a, id) => typeof id === "object" && id.birth === a.birth && id.pgid === a.pgid && id.cwd === a.cwd && /^\[.+\]$/.test(id.command);
|
|
326
|
+
/** The durable arm, or UNREADABLE for a torn one — never mistaken for no arm. */
|
|
327
|
+
function armOf(repoRoot, tier) {
|
|
328
|
+
try {
|
|
329
|
+
return readSupervisionArm(repoRoot, tier);
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
return "UNREADABLE";
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
function checkedRecords(repoRoot, tier, deps) {
|
|
336
|
+
const writer = readJson(supervisionBeatPath(repoRoot, tier));
|
|
337
|
+
if (typeof writer === "object" && !validBeat(writer, tier))
|
|
338
|
+
return { kind: "UNKNOWN", reason: `beat record ${supervisionBeatPath(repoRoot, tier)} is malformed` };
|
|
339
|
+
const state = supervisionStatus(repoRoot, tier, deps.now());
|
|
340
|
+
if (writer === "UNREADABLE" || state.state === "UNREADABLE")
|
|
341
|
+
return { kind: "UNKNOWN", reason: "the tier's supervision records are unreadable" };
|
|
342
|
+
const arm = armOf(repoRoot, tier);
|
|
343
|
+
if (arm === "UNREADABLE")
|
|
344
|
+
return { kind: "UNKNOWN", reason: `durable arm ${supervisionArmPath(repoRoot, tier)} is unreadable` };
|
|
345
|
+
return { kind: "RECORDS", writer, arm, state };
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* An arm that is neither this generation's COMPLETE arm (armId and, once bound, armEpoch) nor — only while the
|
|
349
|
+
* launch has not yet bound its epoch — the fenced arm it launched over, has replaced it.
|
|
350
|
+
*/
|
|
351
|
+
const replacedArm = (arm, owner) => {
|
|
352
|
+
// Once the epoch is bound, an ABSENT arm proves nothing: ownership is the durable arm, so it is a mismatch.
|
|
353
|
+
if (arm === undefined)
|
|
354
|
+
return owner.armEpoch === undefined ? undefined : "(absent)";
|
|
355
|
+
if (arm.armId === owner.armId) {
|
|
356
|
+
return owner.armEpoch === undefined || arm.armEpoch === owner.armEpoch ? undefined : `${arm.armId} (epoch ${arm.armEpoch})`;
|
|
357
|
+
}
|
|
358
|
+
const prior = owner.armEpoch === undefined && arm.armId === owner.priorArmId && arm.armEpoch === owner.priorArmEpoch;
|
|
359
|
+
return prior ? undefined : arm.armId;
|
|
360
|
+
};
|
|
361
|
+
const expectedGeneration = (owner) => ({ owner, armEpoch: owner.armEpoch });
|
|
362
|
+
/**
|
|
363
|
+
* ONE owned-generation barrier for inspection, launch, retirement and rollback. `expected` is the snapshot
|
|
364
|
+
* the caller owns, never a replacement read from disk. Every identity inspection goes through this barrier:
|
|
365
|
+
* identity FIRST, then fresh, independently validated beat/marker/arm records and the exact expected owner.
|
|
366
|
+
* A caller without a process to inspect still uses the same barrier immediately before publishing/removing.
|
|
367
|
+
*/
|
|
368
|
+
function checkOwnedGeneration(repoRoot, tier, snapshot, deps, pid, launching = false) {
|
|
369
|
+
const expected = snapshot.owner;
|
|
370
|
+
let identity = "UNKNOWN";
|
|
371
|
+
if (pid !== undefined) {
|
|
372
|
+
try {
|
|
373
|
+
identity = deps.identity(pid);
|
|
374
|
+
}
|
|
375
|
+
catch { /* an inspection failure proves neither life nor death */ }
|
|
376
|
+
}
|
|
377
|
+
const records = checkedRecords(repoRoot, tier, deps);
|
|
378
|
+
if (records.kind === "UNKNOWN")
|
|
379
|
+
return records;
|
|
380
|
+
const current = readBeatOwner(repoRoot, tier);
|
|
381
|
+
if (current === "UNREADABLE")
|
|
382
|
+
return { kind: "UNKNOWN", reason: `malformed owner record ${beatOwnerPath(repoRoot, tier)}` };
|
|
383
|
+
if (JSON.stringify(current) !== JSON.stringify(expected))
|
|
384
|
+
return {
|
|
385
|
+
kind: "UNKNOWN", reason: OWNER_CHANGED,
|
|
386
|
+
mutationReason: `the owner record no longer names this generation; generation ${expected.generation} was replaced before its stop — left in place`,
|
|
387
|
+
};
|
|
388
|
+
const armOwner = { ...expected, armEpoch: snapshot.armEpoch ?? expected.armEpoch };
|
|
389
|
+
const replaced = replacedArm(records.arm, armOwner);
|
|
390
|
+
if (replaced)
|
|
391
|
+
return {
|
|
392
|
+
kind: "FOREIGN", reason: `durable arm ${replaced} is not generation ${expected.generation}'s arm`,
|
|
393
|
+
mutationReason: `durable arm ${replaced} replaced generation ${expected.generation}'s arm — left in place`,
|
|
394
|
+
};
|
|
395
|
+
if (!launching && armOwner.armEpoch === undefined && records.arm?.armId === expected.armId)
|
|
396
|
+
return {
|
|
397
|
+
kind: "UNKNOWN", reason: `generation ${expected.generation} never proved its arm epoch; the current arm cannot authorize cleanup`,
|
|
398
|
+
};
|
|
399
|
+
return { kind: "OWNED", records, identity };
|
|
400
|
+
}
|
|
401
|
+
function requireOwned(check, tier) {
|
|
402
|
+
if (check.kind !== "OWNED")
|
|
403
|
+
refuse(`${tier} ${check.mutationReason ?? `${check.kind === "UNKNOWN" ? "UNREADABLE" : "MISMATCH"} — ${check.reason}; left in place`}`);
|
|
404
|
+
return check;
|
|
405
|
+
}
|
|
406
|
+
/** Freshness retains the existing mtime boundaries; payload attribution also requires a post-launch beat. */
|
|
407
|
+
function advancingBeat(records, owner) {
|
|
408
|
+
const w = typeof records.writer === "object" ? records.writer : {};
|
|
409
|
+
return records.state.state === "ARMED" && owner.armEpoch !== undefined &&
|
|
410
|
+
records.arm?.armId === owner.armId && records.arm.armEpoch === owner.armEpoch &&
|
|
411
|
+
w.tier === owner.tier && w.seat === owner.seat && w.armId === owner.armId && w.armEpoch === owner.armEpoch &&
|
|
412
|
+
w.pid === owner.pid && w.exitedWriterPid === undefined &&
|
|
413
|
+
Date.parse(w.beatAt) >= Date.parse(owner.startedAt);
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* The checked state of one tier, read-only: no claim, no write, no signal. Every authoritative record —
|
|
417
|
+
* owner, marker, beat and durable arm — is validated before any outcome is selected, and ownership binds to
|
|
418
|
+
* this repository's canonical root, the recorded seat and the recorded arm before the process is consulted.
|
|
419
|
+
*/
|
|
420
|
+
export function inspectBeat(repoRoot, tier, seat, deps) {
|
|
421
|
+
const owner = readBeatOwner(repoRoot, tier);
|
|
422
|
+
if (owner === "UNREADABLE")
|
|
423
|
+
return { kind: "UNKNOWN", reason: `malformed owner record ${beatOwnerPath(repoRoot, tier)}` };
|
|
424
|
+
const initial = checkedRecords(repoRoot, tier, deps);
|
|
425
|
+
if (initial.kind === "UNKNOWN")
|
|
426
|
+
return initial;
|
|
427
|
+
const tierState = initial.state;
|
|
428
|
+
if (owner === "NONE") {
|
|
429
|
+
if (tierState.state === "ABSENT")
|
|
430
|
+
return { kind: "EMPTY" };
|
|
431
|
+
if (tierState.state === "DISARMED")
|
|
432
|
+
return { kind: "DISARMED", ...(tierState.seat ? { seat: tierState.seat } : {}) };
|
|
433
|
+
return { kind: "FOREIGN", reason: `${tierState.state} by an unowned writer${tierState.seat ? ` (seat ${tierState.seat})` : ""}` };
|
|
434
|
+
}
|
|
435
|
+
const root = realpathSync(repoRoot);
|
|
436
|
+
if (owner.cwd !== root)
|
|
437
|
+
return { kind: "FOREIGN", reason: `owner record names repository ${owner.cwd}, not ${root}` };
|
|
438
|
+
if (seat !== undefined && owner.seat !== seat)
|
|
439
|
+
return { kind: "FOREIGN", reason: `owned by seat ${owner.seat}, not ${seat}` };
|
|
440
|
+
const checked = checkOwnedGeneration(repoRoot, tier, expectedGeneration(owner), deps, owner.pid);
|
|
441
|
+
if (checked.kind !== "OWNED")
|
|
442
|
+
return checked;
|
|
443
|
+
const id = checked.identity;
|
|
444
|
+
if (owner.pid === undefined)
|
|
445
|
+
return { kind: "UNKNOWN", reason: `generation ${owner.generation} recorded no process (launch in progress or abandoned)` };
|
|
446
|
+
if (owner.armEpoch === undefined)
|
|
447
|
+
return { kind: "UNKNOWN", reason: `generation ${owner.generation} never bound its arm epoch (launch in progress or abandoned)` };
|
|
448
|
+
const owned = owner;
|
|
449
|
+
if (id === "DEAD")
|
|
450
|
+
return { kind: "OWN-DEAD", owner: owned };
|
|
451
|
+
if (id === "UNKNOWN")
|
|
452
|
+
return { kind: "UNKNOWN", reason: `identity of recorded pid ${owned.pid} is unreadable` };
|
|
453
|
+
if (!sameProcess(owned, id))
|
|
454
|
+
return { kind: "PID-REUSED", owner: owned };
|
|
455
|
+
// Advancing means THIS child wrote the fresh beat — tier, seat, arm and its own live pid. A one-shot record
|
|
456
|
+
// (exitedWriterPid) or any other writer reusing the arm id is not the owned child beating.
|
|
457
|
+
return { kind: "OWN-LIVE", owner: owned, advancing: advancingBeat(checked.records, owned) };
|
|
458
|
+
}
|
|
459
|
+
/** Status: the recorded liveness, rendered without writing; healthy states exit 0, the rest nonzero. */
|
|
460
|
+
export function beatStatus(repoRoot, tier, seat, deps) {
|
|
461
|
+
const checked = inspectBeat(repoRoot, tier, seat, deps);
|
|
462
|
+
const claim = readClaim(repoRoot, tier);
|
|
463
|
+
const busy = claim === "NONE" ? "" : ` · BUSY (${typeof claim === "object" ? `claim held by pid ${claim.pid}` : "unreadable claim"})`;
|
|
464
|
+
const row = (() => {
|
|
465
|
+
switch (checked.kind) {
|
|
466
|
+
case "EMPTY": return ["ABSENT — no beat recorded", 0];
|
|
467
|
+
case "DISARMED": return [`DISARMED${checked.seat ? ` (${checked.seat})` : ""}`, 0];
|
|
468
|
+
case "OWN-LIVE": return checked.advancing
|
|
469
|
+
? [`ARMED (${checked.owner.seat}) — detached pid ${checked.owner.pid}, generation ${checked.owner.generation}`, 0]
|
|
470
|
+
: [`STALE (${checked.owner.seat}) — detached pid ${checked.owner.pid} is alive but its beat is not advancing`, 1];
|
|
471
|
+
case "OWN-DEAD": return [`STALE (${checked.owner.seat}) — recorded pid ${checked.owner.pid} is gone; run \`tickmarkr beat stop ${tier} --seat ${checked.owner.seat}\``, 1];
|
|
472
|
+
case "FOREIGN": return [`MISMATCH — ${checked.reason}`, 1];
|
|
473
|
+
case "PID-REUSED": return [`MISMATCH — recorded pid ${checked.owner.pid} now belongs to another process`, 1];
|
|
474
|
+
case "UNKNOWN": return [`UNREADABLE — ${checked.reason}`, 1];
|
|
475
|
+
}
|
|
476
|
+
})();
|
|
477
|
+
return { out: `${tier} ${row[0]}${busy}`, code: row[1] };
|
|
478
|
+
}
|
|
479
|
+
const exited = (child) => child.exitCode !== null || child.signalCode !== null;
|
|
480
|
+
/** start: one claim, one child; exit zero only after this exact child's own advancing beat reads back. */
|
|
481
|
+
export async function startBeat(repoRoot, tier, seat, deps) {
|
|
482
|
+
const generation = randomUUID();
|
|
483
|
+
acquireBeatClaim(repoRoot, tier, generation);
|
|
484
|
+
let result;
|
|
485
|
+
try {
|
|
486
|
+
const checked = inspectBeat(repoRoot, tier, seat, deps);
|
|
487
|
+
switch (checked.kind) {
|
|
488
|
+
case "EMPTY":
|
|
489
|
+
case "DISARMED":
|
|
490
|
+
result = await launch(repoRoot, tier, seat, generation, deps);
|
|
491
|
+
break;
|
|
492
|
+
case "OWN-LIVE":
|
|
493
|
+
if (!checked.advancing)
|
|
494
|
+
refuse(`${tier} STALE — owned pid ${checked.owner.pid} is alive but silent; run \`tickmarkr beat stop ${tier} --seat ${seat}\` first`);
|
|
495
|
+
result = `${tier} already running as ${seat} — detached pid ${checked.owner.pid}, generation ${checked.owner.generation}; checked no-op`;
|
|
496
|
+
break;
|
|
497
|
+
case "OWN-DEAD":
|
|
498
|
+
refuse(`${tier} STALE — owned pid ${checked.owner.pid} is gone; run \`tickmarkr beat stop ${tier} --seat ${seat}\` first`);
|
|
499
|
+
break;
|
|
500
|
+
case "FOREIGN":
|
|
501
|
+
refuse(`${tier} MISMATCH — ${checked.reason}; nothing started`);
|
|
502
|
+
break;
|
|
503
|
+
case "PID-REUSED":
|
|
504
|
+
refuse(`${tier} MISMATCH — recorded pid ${checked.owner.pid} now belongs to another process; nothing started`);
|
|
505
|
+
break;
|
|
506
|
+
case "UNKNOWN":
|
|
507
|
+
refuse(`${tier} UNREADABLE — ${checked.reason}; nothing started`);
|
|
508
|
+
break;
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
finally {
|
|
512
|
+
releaseBeatClaim(repoRoot, tier, generation);
|
|
513
|
+
}
|
|
514
|
+
return result;
|
|
515
|
+
}
|
|
516
|
+
async function launch(repoRoot, tier, seat, generation, deps) {
|
|
517
|
+
const startedAt = deps.now();
|
|
518
|
+
const deadline = startedAt + BEAT_STARTUP_CEILING_MS;
|
|
519
|
+
const prior = armOf(repoRoot, tier);
|
|
520
|
+
const base = {
|
|
521
|
+
tier, seat, generation, armId: generation, cwd: realpathSync(repoRoot), startedAt: new Date(startedAt).toISOString(),
|
|
522
|
+
...(typeof prior === "object" ? { priorArmId: prior.armId, priorArmEpoch: prior.armEpoch } : {}),
|
|
523
|
+
};
|
|
524
|
+
// This launch's expected snapshot is carried through EVERY failure, including rollback. Never replace
|
|
525
|
+
// it with a reread owner: even the same token under a different arm epoch is another generation.
|
|
526
|
+
let expected = expectedGeneration(base);
|
|
527
|
+
const snapshot = () => expected;
|
|
528
|
+
try {
|
|
529
|
+
publish(beatOwnerPath(repoRoot, tier), base);
|
|
530
|
+
requireOwned(checkOwnedGeneration(repoRoot, tier, snapshot(), deps, undefined, true), tier);
|
|
531
|
+
}
|
|
532
|
+
catch (error) {
|
|
533
|
+
return rollback(repoRoot, tier, snapshot(), undefined, undefined, deps, `owner publication failed: ${error.message}`);
|
|
534
|
+
}
|
|
535
|
+
const log = beatLogPath(repoRoot, tier);
|
|
536
|
+
let child;
|
|
537
|
+
try {
|
|
538
|
+
const fd = deps.openLog(log);
|
|
539
|
+
try {
|
|
540
|
+
child = deps.spawn(deps.cli[0], [...deps.cli.slice(1), "beat", tier, "--seat", seat, "--loop", "--arm-id", generation], {
|
|
541
|
+
cwd: repoRoot, detached: true, stdio: ["ignore", fd, fd], env: { ...deps.env, [BEAT_GENERATION_ENV]: generation },
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
finally {
|
|
545
|
+
closeSync(fd);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
catch (error) {
|
|
549
|
+
return rollback(repoRoot, tier, snapshot(), undefined, undefined, deps, `log-open/spawn failed: ${error.message}`);
|
|
550
|
+
}
|
|
551
|
+
child.on("error", () => { });
|
|
552
|
+
child.unref();
|
|
553
|
+
const pid = child.pid;
|
|
554
|
+
if (!pid)
|
|
555
|
+
return rollback(repoRoot, tier, snapshot(), child, undefined, deps, "spawn produced no process");
|
|
556
|
+
// Until its identity proves it is this generation's child (this cwd, this generation in its argv), the pid
|
|
557
|
+
// is never a signal target: an unproven rollback retains the record and the child retires itself.
|
|
558
|
+
const proof = checkOwnedGeneration(repoRoot, tier, snapshot(), deps, pid, true);
|
|
559
|
+
if (proof.kind !== "OWNED")
|
|
560
|
+
return rollback(repoRoot, tier, snapshot(), child, undefined, deps, proof.mutationReason ?? proof.reason);
|
|
561
|
+
const id = proof.identity;
|
|
562
|
+
if (exited(child) || id === "DEAD")
|
|
563
|
+
return rollback(repoRoot, tier, snapshot(), child, undefined, deps, `child pid ${pid} died before its beat read back (log ${log})`);
|
|
564
|
+
if (id === "UNKNOWN")
|
|
565
|
+
return rollback(repoRoot, tier, snapshot(), child, undefined, deps, `child pid ${pid} identity is unreadable`);
|
|
566
|
+
if (id.cwd !== base.cwd || !id.command.includes(generation)) {
|
|
567
|
+
return rollback(repoRoot, tier, snapshot(), child, undefined, deps, `child pid ${pid} is not the launched generation (cwd ${id.cwd})`);
|
|
568
|
+
}
|
|
569
|
+
const fail = (why) => rollback(repoRoot, tier, snapshot(), child, id, deps, why);
|
|
570
|
+
const captureArm = (records) => {
|
|
571
|
+
const w = typeof records.writer === "object" ? records.writer : {};
|
|
572
|
+
if (expected.armEpoch === undefined && w.tier === tier && w.seat === seat && w.armId === generation &&
|
|
573
|
+
w.pid === pid && w.exitedWriterPid === undefined && text(w.armEpoch)) {
|
|
574
|
+
expected = { ...expected, armEpoch: w.armEpoch };
|
|
575
|
+
}
|
|
576
|
+
return requireOwned(checkOwnedGeneration(repoRoot, tier, snapshot(), deps, undefined, true), tier).records;
|
|
577
|
+
};
|
|
578
|
+
try {
|
|
579
|
+
captureArm(proof.records);
|
|
580
|
+
}
|
|
581
|
+
catch (error) {
|
|
582
|
+
return fail(error.message);
|
|
583
|
+
}
|
|
584
|
+
// Detached means its OWN session and process group, never "ppid 1": an orphan of a dying session has
|
|
585
|
+
// ppid 1 too and still dies with that session's group.
|
|
586
|
+
if (id.pgid !== pid)
|
|
587
|
+
return fail(`child pid ${pid} is not detached (process group ${id.pgid})`);
|
|
588
|
+
const record = (next) => {
|
|
589
|
+
requireOwned(checkOwnedGeneration(repoRoot, tier, snapshot(), deps, undefined, true), tier);
|
|
590
|
+
// If publication or its read-back fails, the attempted snapshot remains the rollback authority.
|
|
591
|
+
expected = { ...expected, owner: next };
|
|
592
|
+
publish(beatOwnerPath(repoRoot, tier), next);
|
|
593
|
+
return requireOwned(checkOwnedGeneration(repoRoot, tier, snapshot(), deps, undefined, true), tier).records;
|
|
594
|
+
};
|
|
595
|
+
try {
|
|
596
|
+
record({ ...base, pid, birth: id.birth, command: id.command, pgid: id.pgid });
|
|
597
|
+
}
|
|
598
|
+
catch (error) {
|
|
599
|
+
return fail(`owner publication failed: ${error.message}`);
|
|
600
|
+
}
|
|
601
|
+
for (;;) {
|
|
602
|
+
const live = checkOwnedGeneration(repoRoot, tier, snapshot(), deps, pid, true);
|
|
603
|
+
if (live.kind !== "OWNED")
|
|
604
|
+
return fail(live.mutationReason ?? `UNREADABLE — ${live.reason}`);
|
|
605
|
+
if (exited(child) || live.identity === "DEAD")
|
|
606
|
+
return fail(`child pid ${pid} died before its beat read back (log ${log})`);
|
|
607
|
+
// Exit can fall between ps and cwd reads. UNKNOWN proves no identity change: yield so the
|
|
608
|
+
// exit is observable, then repeat the full generation/records check within the startup budget.
|
|
609
|
+
if (live.identity === "UNKNOWN") {
|
|
610
|
+
if (deps.now() >= deadline)
|
|
611
|
+
return fail(`UNREADABLE — child pid ${pid} identity is unreadable during read-back`);
|
|
612
|
+
await deps.sleep(Math.min(100, Math.max(0, deadline - deps.now())));
|
|
613
|
+
continue;
|
|
614
|
+
}
|
|
615
|
+
if (!sameProcess(expected.owner, live.identity))
|
|
616
|
+
return fail(`child pid ${pid} changed identity during read-back`);
|
|
617
|
+
let records = live.records;
|
|
618
|
+
if (expected.owner.armEpoch === undefined) {
|
|
619
|
+
// Bind the epoch from the exact child's independently validated beat, never from whatever arm is
|
|
620
|
+
// current after an identity read. A same-id replacement cannot become this launch's authority.
|
|
621
|
+
try {
|
|
622
|
+
records = captureArm(records);
|
|
623
|
+
if (expected.armEpoch !== undefined)
|
|
624
|
+
records = record({ ...expected.owner, armEpoch: expected.armEpoch });
|
|
625
|
+
}
|
|
626
|
+
catch (error) {
|
|
627
|
+
return fail(`owner publication failed: ${error.message}`);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
// the read-back is the checked state itself: this exact child, advancing, over validated records
|
|
631
|
+
if (advancingBeat(records, expected.owner))
|
|
632
|
+
break;
|
|
633
|
+
if (deps.now() >= deadline)
|
|
634
|
+
return fail(`no beat from pid ${pid} within ${BEAT_STARTUP_CEILING_MS} ms (log ${log})`);
|
|
635
|
+
await deps.sleep(Math.min(100, Math.max(0, deadline - deps.now())));
|
|
636
|
+
}
|
|
637
|
+
// The final barrier, in order: identity, then the owner record and the COMPLETE durable arm re-read after it,
|
|
638
|
+
// then the records' own schema — and the ceiling LAST, so time spent in these blocking reads counts too.
|
|
639
|
+
const final = checkOwnedGeneration(repoRoot, tier, snapshot(), deps, pid);
|
|
640
|
+
if (final.kind !== "OWNED")
|
|
641
|
+
return fail(final.mutationReason ?? `UNREADABLE — ${final.reason}`);
|
|
642
|
+
if (exited(child) || final.identity === "DEAD")
|
|
643
|
+
return fail(`child pid ${pid} died before its beat read back (log ${log})`);
|
|
644
|
+
if (final.identity === "UNKNOWN")
|
|
645
|
+
return fail(`UNREADABLE — child pid ${pid} identity is unreadable during read-back`);
|
|
646
|
+
if (!sameProcess(expected.owner, final.identity))
|
|
647
|
+
return fail(`child pid ${pid} changed identity during read-back`);
|
|
648
|
+
const advancing = advancingBeat(final.records, expected.owner);
|
|
649
|
+
if (deps.now() - startedAt > BEAT_STARTUP_CEILING_MS)
|
|
650
|
+
return fail(`startup overran ${BEAT_STARTUP_CEILING_MS} ms`);
|
|
651
|
+
if (!advancing)
|
|
652
|
+
return fail(`STALE — child pid ${pid}'s beat is not advancing at final read-back`);
|
|
653
|
+
return `${tier} ARMED as ${seat} — detached pid ${pid}, generation ${generation}; log ${log}`;
|
|
654
|
+
}
|
|
655
|
+
/** Wait until `gone()` holds or the deadline passes; true when it held. */
|
|
656
|
+
async function waitFor(gone, ms, deps) {
|
|
657
|
+
const deadline = deps.now() + ms;
|
|
658
|
+
while (!gone()) {
|
|
659
|
+
if (deps.now() >= deadline)
|
|
660
|
+
return false;
|
|
661
|
+
await deps.sleep(50);
|
|
662
|
+
}
|
|
663
|
+
return true;
|
|
664
|
+
}
|
|
665
|
+
/**
|
|
666
|
+
* Undo only what this generation did; anything it cannot prove stays in place and the refusal says so.
|
|
667
|
+
* EVERY signal is preceded by a recheck of both the owner generation and the child's live identity against
|
|
668
|
+
* the identity proven at launch: a replaced generation or an unproven or changed identity is never signalled.
|
|
669
|
+
*/
|
|
670
|
+
async function rollback(repoRoot, tier, expected, child, proven, deps, why) {
|
|
671
|
+
const failed = (what) => refuse(`${tier} start refused: ${why}; ${what}`);
|
|
672
|
+
const ownerRead = () => readBeatOwner(repoRoot, tier);
|
|
673
|
+
const check = (phase, pid) => {
|
|
674
|
+
const checked = checkOwnedGeneration(repoRoot, tier, expected, deps, pid);
|
|
675
|
+
if (checked.kind !== "OWNED")
|
|
676
|
+
failed(`${phase} stopped — ${checked.mutationReason ?? `UNREADABLE — ${checked.reason}`}; ${pid ? `pid ${pid} not signalled` : "left in place"}`);
|
|
677
|
+
return checked;
|
|
678
|
+
};
|
|
679
|
+
const pid = child?.pid;
|
|
680
|
+
if (child && pid) {
|
|
681
|
+
const gone = () => {
|
|
682
|
+
const checked = check("rollback", pid);
|
|
683
|
+
if (checked.identity === "DEAD")
|
|
684
|
+
return true;
|
|
685
|
+
// A process exiting after our signal can disappear between its ps and cwd reads. UNKNOWN
|
|
686
|
+
// proves neither death nor a changed identity: wait for checked death, and still require a
|
|
687
|
+
// matching live identity at the separate barrier before any further signal.
|
|
688
|
+
if (proven && checked.identity !== "UNKNOWN" && !sameIdentity(proven, checked.identity) && !exitingAfterSignal(proven, checked.identity))
|
|
689
|
+
failed(`pid ${pid} not signalled — its identity is not proven to be this generation's child; owner record retained`);
|
|
690
|
+
return false;
|
|
691
|
+
};
|
|
692
|
+
for (const signal of ["SIGTERM", "SIGKILL"]) {
|
|
693
|
+
if (gone())
|
|
694
|
+
break;
|
|
695
|
+
// identity FIRST, ownership re-read after it: a generation replaced during the identity read is not signalled
|
|
696
|
+
const checked = check("rollback", pid);
|
|
697
|
+
if (!proven || !sameIdentity(proven, checked.identity)) {
|
|
698
|
+
failed(`pid ${pid} not signalled — its identity is not proven to be this generation's child; owner record retained`);
|
|
699
|
+
}
|
|
700
|
+
try {
|
|
701
|
+
deps.kill(pid, signal);
|
|
702
|
+
}
|
|
703
|
+
catch { /* the wait below decides */ }
|
|
704
|
+
await waitFor(gone, 3_000, deps);
|
|
705
|
+
}
|
|
706
|
+
if (!gone())
|
|
707
|
+
failed(`pid ${pid} did not exit — owner record retained`);
|
|
708
|
+
}
|
|
709
|
+
if (ownerRead() === "NONE")
|
|
710
|
+
failed("no owner record was published; nothing to roll back");
|
|
711
|
+
try {
|
|
712
|
+
const checked = check("cleanup");
|
|
713
|
+
if (checked.records.arm?.armId === expected.owner.armId) {
|
|
714
|
+
publishStandDown(repoRoot, tier, expected.owner.seat);
|
|
715
|
+
check("cleanup");
|
|
716
|
+
if (checkedRecords(repoRoot, tier, deps).kind === "UNKNOWN" || supervisionStatus(repoRoot, tier, deps.now()).state !== "DISARMED")
|
|
717
|
+
failed("rollback stand-down read-back failed; owner retained");
|
|
718
|
+
}
|
|
719
|
+
removeOwner(repoRoot, tier, expected, deps);
|
|
720
|
+
}
|
|
721
|
+
catch (error) {
|
|
722
|
+
if (error instanceof BeatRefusal)
|
|
723
|
+
throw error;
|
|
724
|
+
failed(`rollback cleanup failed: ${error.message}`);
|
|
725
|
+
}
|
|
726
|
+
if (ownerRead() !== "NONE")
|
|
727
|
+
failed("owner record could not be removed");
|
|
728
|
+
return failed("rolled back this generation only");
|
|
729
|
+
}
|
|
730
|
+
/** stop: retire only the recorded generation, signalling its exact pid, then read back DISARMED. */
|
|
731
|
+
export async function stopBeat(repoRoot, tier, seat, deps) {
|
|
732
|
+
const token = randomUUID();
|
|
733
|
+
acquireBeatClaim(repoRoot, tier, token);
|
|
734
|
+
let result;
|
|
735
|
+
try {
|
|
736
|
+
const checked = inspectBeat(repoRoot, tier, seat, deps);
|
|
737
|
+
switch (checked.kind) {
|
|
738
|
+
case "EMPTY":
|
|
739
|
+
publishStandDown(repoRoot, tier, seat);
|
|
740
|
+
result = readBackDisarmed(repoRoot, tier, deps, "published DISARMED; nothing signalled");
|
|
741
|
+
break;
|
|
742
|
+
case "DISARMED":
|
|
743
|
+
result = `${tier} DISARMED — already stood down; checked no-op`;
|
|
744
|
+
break;
|
|
745
|
+
case "OWN-LIVE":
|
|
746
|
+
result = await retire(repoRoot, tier, checked.owner, deps);
|
|
747
|
+
break;
|
|
748
|
+
case "OWN-DEAD": {
|
|
749
|
+
recheckOwned(repoRoot, tier, checked.owner, deps, false);
|
|
750
|
+
const state = supervisionStatus(repoRoot, tier, deps.now()).state;
|
|
751
|
+
if (state === "UNREADABLE")
|
|
752
|
+
refuse(`${tier} UNREADABLE — the tier's supervision records became unreadable; nothing removed`);
|
|
753
|
+
if (state !== "DISARMED")
|
|
754
|
+
publishStandDown(repoRoot, tier, checked.owner.seat);
|
|
755
|
+
recheckOwned(repoRoot, tier, checked.owner, deps, false);
|
|
756
|
+
removeOwner(repoRoot, tier, expectedGeneration(checked.owner), deps);
|
|
757
|
+
result = readBackDisarmed(repoRoot, tier, deps, `pid ${checked.owner.pid} was already gone; cleaned up, nothing signalled`);
|
|
758
|
+
break;
|
|
759
|
+
}
|
|
760
|
+
case "FOREIGN":
|
|
761
|
+
refuse(`${tier} MISMATCH — ${checked.reason}; nothing signalled or removed`);
|
|
762
|
+
break;
|
|
763
|
+
case "PID-REUSED":
|
|
764
|
+
refuse(`${tier} MISMATCH — recorded pid ${checked.owner.pid} now belongs to another process; nothing signalled or removed`);
|
|
765
|
+
break;
|
|
766
|
+
case "UNKNOWN":
|
|
767
|
+
refuse(`${tier} UNREADABLE — ${checked.reason}; nothing signalled or removed`);
|
|
768
|
+
break;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
finally {
|
|
772
|
+
releaseBeatClaim(repoRoot, tier, token);
|
|
773
|
+
}
|
|
774
|
+
return result;
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* The generation barrier before every publication, signal and removal. Identity first (while the process is
|
|
778
|
+
* meant to be live), then the owner record, then the durable arm: a replacement published while the identity
|
|
779
|
+
* was being read still refuses, and so does an arm that is no longer this generation's (or is gone).
|
|
780
|
+
*/
|
|
781
|
+
function recheckOwned(repoRoot, tier, owner, deps, live = true) {
|
|
782
|
+
const checked = requireOwned(checkOwnedGeneration(repoRoot, tier, expectedGeneration(owner), deps, live ? owner.pid : undefined), tier);
|
|
783
|
+
if (live && !sameProcess(owner, checked.identity))
|
|
784
|
+
refuse(`${tier} pid ${owner.pid} no longer matches its recorded identity — nothing further signalled`);
|
|
785
|
+
}
|
|
786
|
+
async function retire(repoRoot, tier, owner, deps) {
|
|
787
|
+
recheckOwned(repoRoot, tier, owner, deps);
|
|
788
|
+
publishStandDown(repoRoot, tier, owner.seat);
|
|
789
|
+
recheckOwned(repoRoot, tier, owner, deps);
|
|
790
|
+
if (!Number.isInteger(owner.pid) || owner.pid <= 0)
|
|
791
|
+
refuse(`${tier} recorded pid ${owner.pid} is not a positive pid`);
|
|
792
|
+
const gone = () => {
|
|
793
|
+
const { identity } = requireOwned(checkOwnedGeneration(repoRoot, tier, expectedGeneration(owner), deps, owner.pid), tier);
|
|
794
|
+
if (identity !== "DEAD" && identity !== "UNKNOWN" && !sameProcess(owner, identity) && !exitingAfterSignal(owner, identity))
|
|
795
|
+
refuse(`${tier} MISMATCH — pid ${owner.pid} changed identity while stopping; owner record retained`);
|
|
796
|
+
return identity === "DEAD";
|
|
797
|
+
};
|
|
798
|
+
try {
|
|
799
|
+
deps.kill(owner.pid, "SIGTERM");
|
|
800
|
+
}
|
|
801
|
+
catch (error) {
|
|
802
|
+
if (!gone())
|
|
803
|
+
refuse(`${tier} signal to pid ${owner.pid} failed: ${error.message}; owner record retained`);
|
|
804
|
+
}
|
|
805
|
+
if (!await waitFor(gone, SUPERVISION_BEAT_MS, deps))
|
|
806
|
+
refuse(`${tier} pid ${owner.pid} did not exit after SIGTERM; owner record retained`);
|
|
807
|
+
recheckOwned(repoRoot, tier, owner, deps, false);
|
|
808
|
+
removeOwner(repoRoot, tier, expectedGeneration(owner), deps);
|
|
809
|
+
return readBackDisarmed(repoRoot, tier, deps, `retired detached pid ${owner.pid} (generation ${owner.generation})`);
|
|
810
|
+
}
|
|
811
|
+
function removeOwner(repoRoot, tier, expected, deps) {
|
|
812
|
+
requireOwned(checkOwnedGeneration(repoRoot, tier, expected, deps), tier);
|
|
813
|
+
rmSync(beatOwnerPath(repoRoot, tier));
|
|
814
|
+
if (readBeatOwner(repoRoot, tier) !== "NONE")
|
|
815
|
+
refuse(`${tier} owner record could not be removed — cleanup unacknowledged`);
|
|
816
|
+
}
|
|
817
|
+
function readBackDisarmed(repoRoot, tier, deps, what) {
|
|
818
|
+
const checked = checkedRecords(repoRoot, tier, deps);
|
|
819
|
+
if (checked.kind === "UNKNOWN")
|
|
820
|
+
refuse(`${tier} stop read back UNREADABLE — ${checked.reason}`);
|
|
821
|
+
const state = checked.state.state;
|
|
822
|
+
if (state !== "DISARMED" || readBeatOwner(repoRoot, tier) !== "NONE")
|
|
823
|
+
refuse(`${tier} stop read back ${state}, not DISARMED`);
|
|
824
|
+
return `${tier} DISARMED — ${what}`;
|
|
825
|
+
}
|