pi-crew 0.11.53 → 0.11.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +152 -0
- package/dist/index.mjs +523 -523
- package/package.json +1 -1
- package/src/config/env-allowlist-registry.ts +4 -0
- package/src/config/env-vars.ts +4 -0
- package/src/extension/team-tool/parallel-dispatch.ts +5 -0
- package/src/extension/team-tool/respond.ts +65 -2
- package/src/extension/team-tool/run.ts +8 -0
- package/src/runtime/broker/protocol/msg-inbox.ts +27 -3
- package/src/runtime/child-pi/child-pi-spawn.ts +25 -1
- package/src/runtime/child-pi/child-pi-timers.ts +25 -0
- package/src/runtime/child-pi/child-pi.ts +9 -0
- package/src/runtime/foreground-control.ts +164 -116
- package/src/runtime/goal-workflow/goal-loop-runner.ts +6 -0
- package/src/runtime/pi-spawn.ts +112 -2
- package/src/runtime/task-runner/child-executor.ts +7 -0
- package/src/runtime/task-runner.ts +17 -0
- package/src/runtime/verification/verification-gates.ts +103 -7
- package/src/runtime/verification/verify-gate.ts +209 -21
- package/src/state/atomic-write.ts +59 -0
- package/src/state/contracts.ts +7 -0
- package/src/state/coordination/mailbox.ts +52 -6
- package/src/state/event-log/event-log.ts +108 -27
- package/src/state/event-log/event-reconstructor.ts +5 -0
- package/src/state/stores/state-store.ts +8 -0
- package/src/state/types.ts +16 -0
- package/src/worktree/worktree-manager.ts +12 -0
package/package.json
CHANGED
|
@@ -195,6 +195,10 @@ const CREW_ENV_FORWARDING_DECL: Record<string, CrewEnvForwardDecision> = {
|
|
|
195
195
|
kind: "child-write",
|
|
196
196
|
reason: "SEC-2026-11 w1a: PUBLIC base64 SPKI Ed25519 verify key handed to children via built.env (child-pi-spawn) for coordination-row verify-only; public material — possession cannot forge rows (private key is heap-only in the root session)",
|
|
197
197
|
},
|
|
198
|
+
PI_CREW_MAILBOX_VERIFY_KEYS: {
|
|
199
|
+
kind: "child-write",
|
|
200
|
+
reason: "FINDING B (2026-10-31): PUBLIC comma-separated base64 SPKI Ed25519 verify-key SET (dispatcher + spawner-relayed) handed to children via built.env (child-pi-spawn) so leader-signed cross-process rows (respond / steering fs-fallback in async runs) verify; public material — possession cannot forge rows (singular PI_CREW_MAILBOX_VERIFY_KEY semantics unchanged)",
|
|
201
|
+
},
|
|
198
202
|
PI_CREW_AGENT_EVENTS_PATH: { kind: "child-write", reason: "prepareSurfaceSpawn recorder log" },
|
|
199
203
|
PI_CREW_BROKER_SOCKET: { kind: "child-write", reason: "child-pi-spawn broker handshake (never ambient — per-run socket)" },
|
|
200
204
|
PI_CREW_BROKER_TOKEN: {
|
package/src/config/env-vars.ts
CHANGED
|
@@ -640,6 +640,10 @@ export const CREW_ENV_VARS: Record<string, CrewEnvVarSpec> = {
|
|
|
640
640
|
name: "PI_CREW_MAILBOX_VERIFY_KEY",
|
|
641
641
|
doc: "SEC-2026-11 w1a coordination sender-binding: base64 SPKI Ed25519 public verify key handed to child workers at spawn (child-pi-spawn.ts) so they can VERIFY parent-signed mailbox/steering rows; the PRIVATE signing key never leaves the root session heap (mailbox.ts signingKeypair). Children verify-only — possession of the public key cannot forge rows",
|
|
642
642
|
},
|
|
643
|
+
PI_CREW_MAILBOX_VERIFY_KEYS: {
|
|
644
|
+
name: "PI_CREW_MAILBOX_VERIFY_KEYS",
|
|
645
|
+
doc: "FINDING B (2026-10-31) plural sibling of PI_CREW_MAILBOX_VERIFY_KEY: comma-separated base64 SPKI Ed25519 PUBLIC verify keys (dispatcher + spawner-relayed) handed to child workers at spawn so cross-process rows (leader-signed respond / steering fs-fallback in async runs) verify instead of dropping as noise (mailbox.ts resolveCoordinationVerifyKeys merges + dedupes plural env, singular env and the in-process keypair). PUBLIC material only — possession cannot forge rows",
|
|
646
|
+
},
|
|
643
647
|
PI_CREW_MAX_AUTO_FALLBACKS: {
|
|
644
648
|
name: "PI_CREW_MAX_AUTO_FALLBACKS",
|
|
645
649
|
parser: "int",
|
|
@@ -12,6 +12,7 @@ import { spawnBackgroundTeamRun } from "../../runtime/async-runner.ts";
|
|
|
12
12
|
import { resolveCrewRuntime } from "../../runtime/model/runtime-resolver.ts";
|
|
13
13
|
import { captureRunModelContext } from "../../runtime/model/session-model.ts";
|
|
14
14
|
import type { TeamToolParamsValue } from "../../schema/team-tool-schema.ts";
|
|
15
|
+
import { ensureCoordinationPublicVerifyKey } from "../../state/coordination/mailbox.ts";
|
|
15
16
|
import { appendEventAsync } from "../../state/event-log/event-log.ts";
|
|
16
17
|
import { createRunManifest } from "../../state/stores/state-store.ts";
|
|
17
18
|
import { discoverTeams } from "../../teams/discover-teams.ts";
|
|
@@ -178,6 +179,10 @@ async function spawnSingleTask(
|
|
|
178
179
|
// Finding #1: persist the parent session's model routing so the detached
|
|
179
180
|
// runner restores it instead of default-chaining (see handleParallel).
|
|
180
181
|
modelContext,
|
|
182
|
+
// FINDING B (2026-10-31): mirror handleRun — stamp THIS (dispatcher)
|
|
183
|
+
// session's PUBLIC verify key so the detached runner's workers verify
|
|
184
|
+
// the respond/steering rows this session signs. PUBLIC material only.
|
|
185
|
+
dispatcherVerifyKey: ensureCoordinationPublicVerifyKey(),
|
|
181
186
|
});
|
|
182
187
|
|
|
183
188
|
await appendEventAsync(created.manifest.eventsPath, {
|
|
@@ -113,10 +113,73 @@ export function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): Pi
|
|
|
113
113
|
const taskId = params.taskId;
|
|
114
114
|
const message = params.message ?? "";
|
|
115
115
|
|
|
116
|
-
|
|
116
|
+
let tasks = fresh.tasks;
|
|
117
|
+
let targetTasks = taskId
|
|
117
118
|
? fresh.tasks.filter((t) => t.id === taskId && t.status === "waiting")
|
|
118
119
|
: fresh.tasks.filter((t) => t.status === "waiting");
|
|
119
120
|
|
|
121
|
+
if (targetTasks.length === 0) {
|
|
122
|
+
// ASK-PARK CLOBBER SELF-HEAL (2026-10-31, live incidents team_20261031084714
|
|
123
|
+
// + team_20261031045633): the HOST process's coalesced tasks.json flush can
|
|
124
|
+
// race the BROKER process's run-locked sync waiting-park (the coalesce
|
|
125
|
+
// buffer is per-process, so the broker's write cannot cancel it) and blind-
|
|
126
|
+
// flush a pre-park `running` snapshot over `waiting` — losing BOTH the
|
|
127
|
+
// status and the questionId marker. Every subsequent respond then hit the
|
|
128
|
+
// reject below ("is running, not waiting") and every ask timed out.
|
|
129
|
+
// The broker's manifest.waitState pointer survives the clobber (manifest.json
|
|
130
|
+
// is not coalesced), so it is the authoritative repair source: when it is
|
|
131
|
+
// still live while tasks.json lost the park, restore `waiting` + marker
|
|
132
|
+
// from it inside the SAME run-lock section, then let the normal answer
|
|
133
|
+
// machinery below run verbatim. Fix 1 (atomic-write flush-time CAS) closes
|
|
134
|
+
// the root cause; this is the defense-in-depth backstop.
|
|
135
|
+
const waitState = fresh.manifest.waitState;
|
|
136
|
+
if (waitState && typeof waitState.deadline === "number" && waitState.deadline > Date.now()) {
|
|
137
|
+
// Narrowed const - the typeof guard on the property does not survive
|
|
138
|
+
// into the map callback below.
|
|
139
|
+
const waitDeadline = waitState.deadline;
|
|
140
|
+
const parked = fresh.tasks.find((t) => t.id === waitState.taskId);
|
|
141
|
+
if (
|
|
142
|
+
parked &&
|
|
143
|
+
parked.status === "running" &&
|
|
144
|
+
(parked.waiting === undefined || parked.waiting.questionId !== waitState.questionId)
|
|
145
|
+
) {
|
|
146
|
+
tasks = fresh.tasks.map((t) =>
|
|
147
|
+
t.id === waitState.taskId
|
|
148
|
+
? {
|
|
149
|
+
...t,
|
|
150
|
+
status: "waiting" as const,
|
|
151
|
+
waiting: {
|
|
152
|
+
questionId: waitState.questionId,
|
|
153
|
+
askedAt: waitState.askedAt,
|
|
154
|
+
deadline: waitDeadline,
|
|
155
|
+
},
|
|
156
|
+
}
|
|
157
|
+
: t,
|
|
158
|
+
);
|
|
159
|
+
// Sync save inside the SAME run-lock critical section — also cancels
|
|
160
|
+
// any same-process pending coalesced tasks.json write (stale
|
|
161
|
+
// snapshot) before it can clobber the repair.
|
|
162
|
+
saveRunTasks(fresh.manifest, tasks);
|
|
163
|
+
void appendEventAsync(fresh.manifest.eventsPath, {
|
|
164
|
+
type: "task.waiting_repaired",
|
|
165
|
+
runId: fresh.manifest.runId,
|
|
166
|
+
taskId: waitState.taskId,
|
|
167
|
+
message: `Repaired clobbered ask park for question ${waitState.questionId}: tasks.json showed the parked task running with no waiting marker (host coalesced-flush raced the broker park); waiting restored from manifest.waitState so the answer path could run.`,
|
|
168
|
+
data: { taskId: waitState.taskId, questionId: waitState.questionId, reason: "coalesce-clobber" },
|
|
169
|
+
}).catch((error) =>
|
|
170
|
+
logInternalError(
|
|
171
|
+
"respond.waiting-repaired-event",
|
|
172
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
173
|
+
`runId=${fresh.manifest.runId}`,
|
|
174
|
+
),
|
|
175
|
+
);
|
|
176
|
+
targetTasks = taskId
|
|
177
|
+
? tasks.filter((t) => t.id === taskId && t.status === "waiting")
|
|
178
|
+
: tasks.filter((t) => t.status === "waiting");
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
120
183
|
if (targetTasks.length === 0) {
|
|
121
184
|
const existing = taskId ? fresh.tasks.find((t) => t.id === taskId) : undefined;
|
|
122
185
|
const hint =
|
|
@@ -155,7 +218,7 @@ export function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): Pi
|
|
|
155
218
|
pidAlive: boolean;
|
|
156
219
|
pidDetail: string;
|
|
157
220
|
}> = [];
|
|
158
|
-
let updatedTasks =
|
|
221
|
+
let updatedTasks = tasks;
|
|
159
222
|
|
|
160
223
|
for (const task of targetTasks) {
|
|
161
224
|
const questionId = task.waiting?.questionId;
|
|
@@ -5,6 +5,7 @@ import type { executeTeamRun as ExecuteTeamRunFn } from "../../runtime/team-runn
|
|
|
5
5
|
import type { TeamToolParamsValue } from "../../schema/team-tool-schema.ts";
|
|
6
6
|
import { atomicWriteJson } from "../../state/atomic-write.ts";
|
|
7
7
|
import { withRunLockSync } from "../../state/coordination/locks.ts";
|
|
8
|
+
import { ensureCoordinationPublicVerifyKey } from "../../state/coordination/mailbox.ts";
|
|
8
9
|
import { registerActiveRun, unregisterActiveRun } from "../../state/stores/active-run-registry.ts";
|
|
9
10
|
import { writeArtifact } from "../../state/stores/artifact-store.ts";
|
|
10
11
|
import { createRunManifest, loadRunManifestById, updateRunStatus } from "../../state/stores/state-store.ts";
|
|
@@ -359,6 +360,13 @@ export async function handleRun(params: TeamToolParamsValue, ctx: TeamContext):
|
|
|
359
360
|
ownerSessionId: ctx.sessionId,
|
|
360
361
|
runKind: effectiveRunKind,
|
|
361
362
|
args: params.args,
|
|
363
|
+
// FINDING B (2026-10-31): stamp THIS (dispatcher) session's PUBLIC verify
|
|
364
|
+
// key onto the manifest so async runs — whose workers are spawned by the
|
|
365
|
+
// detached runner, not by us — can still verify the rows WE sign
|
|
366
|
+
// (respond, steering fs-fallback). Threaded by child-executor into every
|
|
367
|
+
// worker's PI_CREW_MAILBOX_VERIFY_KEYS env. PUBLIC material — never the
|
|
368
|
+
// private key.
|
|
369
|
+
dispatcherVerifyKey: ensureCoordinationPublicVerifyKey(),
|
|
362
370
|
});
|
|
363
371
|
const goalArtifact = writeArtifact(paths.artifactsRoot, {
|
|
364
372
|
kind: "prompt",
|
|
@@ -17,9 +17,14 @@ export interface MsgInboxHelpers {
|
|
|
17
17
|
sendResult(conn: ServerConnection, id: string, result: unknown): void;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
/** Phase 1.1: read the durable mailbox for the addressed task
|
|
21
|
-
*
|
|
22
|
-
* is
|
|
20
|
+
/** Phase 1.1: read the durable mailbox for the addressed task with
|
|
21
|
+
* cursor/limit pagination. The run-level "inbox" lane (connection with no
|
|
22
|
+
* taskId) is readable ONLY with compound auth (S-2, 2026-10-31 fix wave E):
|
|
23
|
+
* the orchestrator key is compound by construction, while a legacy
|
|
24
|
+
* bare-runId ("runId-fallback") worker connection is rejected — the same
|
|
25
|
+
* task-scoped-token rule wait.* (protocol/wait-auth.ts) and delegate.request
|
|
26
|
+
* enforce. Returns messages whose status is NOT yet `acknowledged`; callers
|
|
27
|
+
* should ack after processing. */
|
|
23
28
|
export async function handleMsgInbox(
|
|
24
29
|
conn: ServerConnection,
|
|
25
30
|
id: string,
|
|
@@ -54,6 +59,25 @@ export async function handleMsgInbox(
|
|
|
54
59
|
}
|
|
55
60
|
const limit = Math.min(Math.max(parsed.limit ?? 100, 1), 1000);
|
|
56
61
|
const taskId = conn.taskId ?? undefined;
|
|
62
|
+
// S-2 (fix wave E): reading the run-level (orchestrator) inbox lane
|
|
63
|
+
// requires COMPOUND auth. A legacy bare-runId ("runId-fallback") worker
|
|
64
|
+
// connection has no proven task identity, yet previously fell through to
|
|
65
|
+
// the run-level read here. The orchestrator key is compound BY
|
|
66
|
+
// CONSTRUCTION (crew-broker-tokens.ts tokenRoleWithMatchKind), so the
|
|
67
|
+
// legitimate run-level reader is unaffected; authMatchKind is undefined
|
|
68
|
+
// only for pre-hello connections, which the !conn.runId check above
|
|
69
|
+
// already excludes — fail-closed here regardless. Task-lane reads
|
|
70
|
+
// (conn.taskId set) are never runId-fallback: w1a proved explicit taskId
|
|
71
|
+
// claims only via the compound key at hello.
|
|
72
|
+
if (taskId === undefined && conn.authMatchKind !== "compound") {
|
|
73
|
+
helpers.sendError(
|
|
74
|
+
conn,
|
|
75
|
+
id,
|
|
76
|
+
"forbidden",
|
|
77
|
+
"msg.inbox: run-level lane requires a task-scoped token; re-dispatch with PI_CREW_BROKER_TASK_ID",
|
|
78
|
+
);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
57
81
|
const all = readMailbox(manifest, "inbox", taskId);
|
|
58
82
|
const filtered = all.filter((m) => m.status !== "acknowledged");
|
|
59
83
|
const offset = parsed.cursor ? Number.parseInt(parsed.cursor, 10) || 0 : 0;
|
|
@@ -22,7 +22,7 @@ import * as path from "node:path";
|
|
|
22
22
|
import { childPiBaseAllowlist } from "../../config/env-allowlist-registry.ts";
|
|
23
23
|
import { getCrewEnv } from "../../config/env-vars.ts";
|
|
24
24
|
import { isScratchpadEnabledForRole } from "../../config/role-tools.ts";
|
|
25
|
-
import { COORDINATION_VERIFY_KEY_ENV, getMailboxVerifyKeyPublic } from "../../state/coordination/mailbox.ts";
|
|
25
|
+
import { COORDINATION_VERIFY_KEY_ENV, COORDINATION_VERIFY_KEYS_ENV, getMailboxVerifyKeyPublic } from "../../state/coordination/mailbox.ts";
|
|
26
26
|
import { buildScopedAllowList, providerEnvKeys, sanitizeEnvSecrets } from "../../utils/env-filter.ts";
|
|
27
27
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
28
28
|
import { resolveRealContainedPath } from "../../utils/safe-paths.ts";
|
|
@@ -507,6 +507,30 @@ export function prepareSpawnContext(
|
|
|
507
507
|
const relayMatchesRun =
|
|
508
508
|
relayedVerifyKey !== undefined && input.runId !== undefined && getCrewEnv("PI_CREW_BROKER_RUN_ID") === input.runId;
|
|
509
509
|
built.env[COORDINATION_VERIFY_KEY_ENV] = relayMatchesRun ? relayedVerifyKey : getMailboxVerifyKeyPublic();
|
|
510
|
+
// FINDING B (2026-10-31): ALSO hand the child the PLURAL verify-key set —
|
|
511
|
+
// the coordination rows a child must verify are written by MORE processes
|
|
512
|
+
// than its spawner: async runs have the DISPATCHER session (leader) sign
|
|
513
|
+
// respond rows + steering fs-fallback while the detached runner spawns the
|
|
514
|
+
// workers, so the singular key above never covers those rows and every
|
|
515
|
+
// leader-signed row dropped as noise (ask poll timeout). Sources, merged
|
|
516
|
+
// + deduped: (1) our own inherited plural env when we are a worker
|
|
517
|
+
// spawning a same-run grandchild (same relay-match guard as the singular
|
|
518
|
+
// relay — never leak another run's keys), (2) the dispatcher key threaded
|
|
519
|
+
// from the run manifest (input.dispatcherVerifyKey). PUBLIC keys only.
|
|
520
|
+
const pluralVerifyKeys: string[] = [];
|
|
521
|
+
if (relayMatchesRun) {
|
|
522
|
+
const inheritedPlural = getCrewEnv(COORDINATION_VERIFY_KEYS_ENV);
|
|
523
|
+
if (inheritedPlural !== undefined) {
|
|
524
|
+
for (const key of inheritedPlural.split(",")) {
|
|
525
|
+
const trimmed = key.trim();
|
|
526
|
+
if (trimmed.length > 0 && !pluralVerifyKeys.includes(trimmed)) pluralVerifyKeys.push(trimmed);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
if (input.dispatcherVerifyKey && !pluralVerifyKeys.includes(input.dispatcherVerifyKey)) {
|
|
531
|
+
pluralVerifyKeys.push(input.dispatcherVerifyKey);
|
|
532
|
+
}
|
|
533
|
+
if (pluralVerifyKeys.length > 0) built.env[COORDINATION_VERIFY_KEYS_ENV] = pluralVerifyKeys.join(",");
|
|
510
534
|
}
|
|
511
535
|
// Phase 0 inter-pi broker: inject socket path + token (control-namespace keys,
|
|
512
536
|
// safe under assertOnlyControlEnvKeys). Only when the parent broker issued
|
|
@@ -24,6 +24,13 @@
|
|
|
24
24
|
* constructs — previously cancelHardKill was a local const inside abort() and
|
|
25
25
|
* was NOT cleared by clearChildPiTimeouts(), leaking a 200 ms unref'd timer per
|
|
26
26
|
* cancel on short-lived children.
|
|
27
|
+
*
|
|
28
|
+
* F6 (deep-review 2026-10-31): the final-drain ceiling callback now cancels the
|
|
29
|
+
* quiet-window poller before firing (it previously left it armed, so a
|
|
30
|
+
* quiet-after-ceiling child triggered a SECOND forced drain and overwrote the
|
|
31
|
+
* hardKillTimer variable, orphaning the ceiling's timer → duplicate hard_kill
|
|
32
|
+
* events/SIGKILLs), and hard-kill emission is hardKilled-gated (idempotent) so
|
|
33
|
+
* a second timer can never re-emit.
|
|
27
34
|
*/
|
|
28
35
|
import type { ChildProcess } from "node:child_process";
|
|
29
36
|
import { DEFAULT_CHILD_PI } from "../../config/defaults.ts";
|
|
@@ -231,6 +238,9 @@ export function createChildPiTimers(deps: ChildPiTimersDeps): ChildPiTimers {
|
|
|
231
238
|
// still reaped if it ever fires later.
|
|
232
239
|
hardKillTimer = setTimeout(() => {
|
|
233
240
|
if (deps.state.getSettled() || deps.state.getChildExited()) return;
|
|
241
|
+
// F6: idempotent hard-kill — a second (orphaned/overwritten)
|
|
242
|
+
// hard-kill timer must never re-emit hard_kill/SIGKILL.
|
|
243
|
+
if (deps.state.getHardKilled()) return;
|
|
234
244
|
try {
|
|
235
245
|
deps.state.setHardKilled(true);
|
|
236
246
|
deps.input.onLifecycleEvent?.({
|
|
@@ -255,6 +265,15 @@ export function createChildPiTimers(deps: ChildPiTimersDeps): ChildPiTimers {
|
|
|
255
265
|
}
|
|
256
266
|
finalDrainTimer = setTimeout(() => {
|
|
257
267
|
if (deps.state.getSettled() || deps.state.getChildExited()) return;
|
|
268
|
+
// F6: the ceiling owns the drain from here on — cancel the
|
|
269
|
+
// quiet-window poller (mirroring its self-cancel above) so it
|
|
270
|
+
// cannot fire a SECOND forced drain when the child goes quiet
|
|
271
|
+
// after this ceiling already fired.
|
|
272
|
+
if (pollHandle) {
|
|
273
|
+
clearInterval(pollHandle);
|
|
274
|
+
pollHandle.unref();
|
|
275
|
+
}
|
|
276
|
+
pollHandle = undefined;
|
|
258
277
|
deps.state.setForcedFinalDrain(true);
|
|
259
278
|
deps.state.setFinalDrainFiredMonotonicMs(performance.now()); // Phase-0 diagnostic: race timing.
|
|
260
279
|
deps.input.onLifecycleEvent?.({
|
|
@@ -269,6 +288,9 @@ export function createChildPiTimers(deps: ChildPiTimersDeps): ChildPiTimers {
|
|
|
269
288
|
}
|
|
270
289
|
hardKillTimer = setTimeout(() => {
|
|
271
290
|
if (deps.state.getSettled() || deps.state.getChildExited()) return;
|
|
291
|
+
// F6: idempotent hard-kill — a second (orphaned/overwritten)
|
|
292
|
+
// hard-kill timer must never re-emit hard_kill/SIGKILL.
|
|
293
|
+
if (deps.state.getHardKilled()) return;
|
|
272
294
|
try {
|
|
273
295
|
deps.state.setHardKilled(true);
|
|
274
296
|
deps.input.onLifecycleEvent?.({
|
|
@@ -295,6 +317,9 @@ export function createChildPiTimers(deps: ChildPiTimersDeps): ChildPiTimers {
|
|
|
295
317
|
// are for graceful drain, not user-initiated cancel.
|
|
296
318
|
cancelHardKillTimer = setTimeout(() => {
|
|
297
319
|
if (deps.state.getSettled() || deps.state.getChildExited()) return;
|
|
320
|
+
// F6: idempotent hard-kill — skip when a drain-path hard kill
|
|
321
|
+
// already fired (SIGKILL was already delivered).
|
|
322
|
+
if (deps.state.getHardKilled()) return;
|
|
298
323
|
try {
|
|
299
324
|
deps.state.setHardKilled(true);
|
|
300
325
|
deps.child.kill(process.platform === "win32" ? undefined : "SIGKILL");
|
|
@@ -233,6 +233,15 @@ export interface ChildPiRunInput {
|
|
|
233
233
|
* config). Env PI_CREW_SESSION_RECOVERY overrides in either direction;
|
|
234
234
|
* default ON. Only consulted on crash-ish settles (exitCode null / killed). */
|
|
235
235
|
sessionRecovery?: boolean;
|
|
236
|
+
/** FINDING B (2026-10-31) — w1a cross-process verify: PUBLIC base64 SPKI
|
|
237
|
+
* DER Ed25519 verify key of the run's DISPATCHING session (the leader that
|
|
238
|
+
* signed the manifest; threaded from TeamRunManifest.dispatcherVerifyKey
|
|
239
|
+
* by child-executor). child-pi-spawn merges it into the child's
|
|
240
|
+
* PI_CREW_MAILBOX_VERIFY_KEYS env (plural set) so the dispatcher's
|
|
241
|
+
* respond/steering rows verify even when the SPAWNER is the detached
|
|
242
|
+
* runner (a different process with a different keypair). PUBLIC material —
|
|
243
|
+
* never the private key; absent on non-team spawns. */
|
|
244
|
+
dispatcherVerifyKey?: string;
|
|
236
245
|
/** Path to steering JSONL file for real-time steer injection. */
|
|
237
246
|
steeringFile?: string;
|
|
238
247
|
/** Run ID for cleanup tracking */
|
|
@@ -31,6 +31,129 @@ export interface ForegroundControlRequest {
|
|
|
31
31
|
acknowledged: boolean;
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* On-disk control-file shape, parsed leniently: hand-written control files
|
|
36
|
+
* (tests, older writers) may omit `id`, so readers treat it as optional.
|
|
37
|
+
*/
|
|
38
|
+
type ForegroundControlFile = {
|
|
39
|
+
requests?: Array<{ id?: string; type: string; acknowledged?: boolean }>;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const FOREGROUND_CONTROL_LOCK_TIMEOUT_MS = 5000;
|
|
43
|
+
const FOREGROUND_CONTROL_LOCK_STALE_MS = 10000;
|
|
44
|
+
|
|
45
|
+
function isLockContention(err: unknown): boolean {
|
|
46
|
+
return typeof err === "object" && err !== null && (err as { code?: unknown }).code === "EEXIST";
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// F1 (2026-10-31 verified review): the previous producer lock used
|
|
50
|
+
// `fs.mkdirSync(lockDir, { recursive: true })`, which NEVER throws EEXIST —
|
|
51
|
+
// it provided ZERO mutual exclusion and its contention branches were
|
|
52
|
+
// unreachable dead code. This is the proven event-log dir-lock protocol
|
|
53
|
+
// (withEventLogAppendLockSync, src/state/event-log/event-log.ts): the
|
|
54
|
+
// non-recursive mkdir IS the mutex; timeout + mtime-stale reclaim +
|
|
55
|
+
// dead-pid awareness + PID-guarded release. Deliberately module-local
|
|
56
|
+
// (not extracted next to the event-log lock): that file is owned by a
|
|
57
|
+
// parallel fix wave — dedup into a shared helper is a noted follow-up.
|
|
58
|
+
// Divergence from the event-log pattern (per spec): contention retries ONLY
|
|
59
|
+
// on EEXIST — any other IO error fails fast instead of spinning to timeout.
|
|
60
|
+
function withForegroundControlLock<T>(controlPath: string, fn: () => T): T {
|
|
61
|
+
const lockDir = `${controlPath}.lock`;
|
|
62
|
+
const pidFile = path.join(lockDir, "pid");
|
|
63
|
+
// Ensure the parent directory exists before attempting the lock (mirrors event-log).
|
|
64
|
+
fs.mkdirSync(path.dirname(controlPath), { recursive: true });
|
|
65
|
+
const start = Date.now();
|
|
66
|
+
let acquired = false;
|
|
67
|
+
while (true) {
|
|
68
|
+
try {
|
|
69
|
+
fs.mkdirSync(lockDir);
|
|
70
|
+
acquired = true;
|
|
71
|
+
try {
|
|
72
|
+
// Disposable pid file ("wx" open + plain write): the lock dir itself
|
|
73
|
+
// is the mutex; the pid file only feeds stale/dead detection and the
|
|
74
|
+
// guarded release below.
|
|
75
|
+
const pidFd = fs.openSync(pidFile, "wx");
|
|
76
|
+
try {
|
|
77
|
+
fs.writeSync(pidFd, String(process.pid));
|
|
78
|
+
} finally {
|
|
79
|
+
fs.closeSync(pidFd);
|
|
80
|
+
}
|
|
81
|
+
} catch {
|
|
82
|
+
/* best-effort — e.g. EEXIST under a re-taken dir; a missing pid is mtime-detected */
|
|
83
|
+
}
|
|
84
|
+
break;
|
|
85
|
+
} catch (err) {
|
|
86
|
+
// Contend ONLY on EEXIST; any other IO error (ENOTDIR, EACCES, EROFS…)
|
|
87
|
+
// fails fast with a clear error instead of spinning to the timeout.
|
|
88
|
+
if (!isLockContention(err)) {
|
|
89
|
+
const cause = err instanceof Error ? err : new Error(String(err));
|
|
90
|
+
const wrapped = new Error(`Foreground control lock acquire failed for ${controlPath}: ${cause.message}`, {
|
|
91
|
+
cause,
|
|
92
|
+
});
|
|
93
|
+
logInternalError("foreground-control.lock-acquire-failed", wrapped, `controlPath=${controlPath}`);
|
|
94
|
+
throw wrapped;
|
|
95
|
+
}
|
|
96
|
+
if (Date.now() - start > FOREGROUND_CONTROL_LOCK_TIMEOUT_MS) {
|
|
97
|
+
const timeoutErr = new Error(`Foreground control lock timeout for ${controlPath}`);
|
|
98
|
+
logInternalError("foreground-control.lock-timeout", timeoutErr, `controlPath=${controlPath}`);
|
|
99
|
+
throw timeoutErr;
|
|
100
|
+
}
|
|
101
|
+
// Mtime-stale reclaim FIRST, independent of the pid file: a holder
|
|
102
|
+
// that crashed between mkdir and the pid write has no pid file —
|
|
103
|
+
// only an aged mtime can reclaim its dir.
|
|
104
|
+
try {
|
|
105
|
+
const dirStat = fs.statSync(lockDir);
|
|
106
|
+
if (Date.now() - dirStat.mtimeMs > FOREGROUND_CONTROL_LOCK_STALE_MS) {
|
|
107
|
+
fs.rmSync(lockDir, { recursive: true, force: true });
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
} catch {
|
|
111
|
+
/* dir vanished — let the loop retry */
|
|
112
|
+
}
|
|
113
|
+
// Dead-pid secondary fast-path: a provably dead owner only means keep
|
|
114
|
+
// waiting — NEVER steal a fresh lock on a pid-lookup race; the
|
|
115
|
+
// aged-mtime branch above is the only reclaim.
|
|
116
|
+
try {
|
|
117
|
+
const raw = fs.readFileSync(pidFile, "utf-8").trim();
|
|
118
|
+
const ownerPid = Number.parseInt(raw, 10);
|
|
119
|
+
if (!Number.isNaN(ownerPid) && ownerPid !== process.pid) {
|
|
120
|
+
try {
|
|
121
|
+
process.kill(ownerPid, 0);
|
|
122
|
+
} catch {
|
|
123
|
+
/* dead — but mtime not stale yet, keep waiting */
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
} catch {
|
|
127
|
+
/* no pid file — the mtime check above already handles it */
|
|
128
|
+
}
|
|
129
|
+
// Brief sleep to avoid CPU spinning.
|
|
130
|
+
// WI-7.3 (M7 spec §5): kept sync. Making this async would force
|
|
131
|
+
// requestForegroundControl() + its caller chain async (10+
|
|
132
|
+
// callers in message-tool/lifecycle paths). 10ms spin per
|
|
133
|
+
// iteration is bounded by the 5s timeout above (max ~500 iters
|
|
134
|
+
// = 5s total worst-case block, matching the lock-timeout).
|
|
135
|
+
sleepSync(10);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
try {
|
|
139
|
+
return fn();
|
|
140
|
+
} finally {
|
|
141
|
+
if (acquired) {
|
|
142
|
+
// PID-guarded release: only remove the dir if the pid file still
|
|
143
|
+
// records OUR pid — if fn outlived staleMs and a stealer re-took the
|
|
144
|
+
// dir, the stealer owns it now.
|
|
145
|
+
try {
|
|
146
|
+
const currentPid = fs.readFileSync(pidFile, "utf-8").trim();
|
|
147
|
+
if (currentPid === String(process.pid)) {
|
|
148
|
+
fs.rmSync(lockDir, { recursive: true, force: true });
|
|
149
|
+
}
|
|
150
|
+
} catch {
|
|
151
|
+
/* lock stolen or already gone — do not touch */
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
34
157
|
export function foregroundControlPath(manifest: TeamRunManifest): string {
|
|
35
158
|
return path.join(manifest.stateRoot, "foreground-control.json");
|
|
36
159
|
}
|
|
@@ -70,105 +193,11 @@ export function writeForegroundInterruptRequest(
|
|
|
70
193
|
reason = "User requested foreground interrupt.",
|
|
71
194
|
): ForegroundControlRequest {
|
|
72
195
|
const controlPath = foregroundControlPath(manifest);
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
// Added stale lock detection similar to event-log.ts
|
|
79
|
-
const acquireLock = (): void => {
|
|
80
|
-
const timeout = 5000;
|
|
81
|
-
const staleMs = 10000;
|
|
82
|
-
const start = Date.now();
|
|
83
|
-
while (true) {
|
|
84
|
-
try {
|
|
85
|
-
fs.mkdirSync(lockDir, { recursive: true });
|
|
86
|
-
try {
|
|
87
|
-
atomicWriteFile(pidFile, String(process.pid));
|
|
88
|
-
} catch {
|
|
89
|
-
/* best-effort */
|
|
90
|
-
}
|
|
91
|
-
break;
|
|
92
|
-
} catch {
|
|
93
|
-
if (Date.now() - start > timeout) {
|
|
94
|
-
// Check if lock is stale before giving up
|
|
95
|
-
try {
|
|
96
|
-
const raw = fs.readFileSync(pidFile, "utf-8").trim();
|
|
97
|
-
const ownerPid = Number.parseInt(raw, 10);
|
|
98
|
-
if (!Number.isNaN(ownerPid) && ownerPid !== process.pid) {
|
|
99
|
-
let alive = false;
|
|
100
|
-
try {
|
|
101
|
-
process.kill(ownerPid, 0);
|
|
102
|
-
alive = true;
|
|
103
|
-
} catch {
|
|
104
|
-
/* dead */
|
|
105
|
-
}
|
|
106
|
-
if (!alive) {
|
|
107
|
-
// Lock is stale — clear it and retry
|
|
108
|
-
fs.rmSync(lockDir, {
|
|
109
|
-
recursive: true,
|
|
110
|
-
force: true,
|
|
111
|
-
});
|
|
112
|
-
continue;
|
|
113
|
-
}
|
|
114
|
-
// Lock held by live process — throw
|
|
115
|
-
const err = new Error(`Foreground control lock timeout for ${controlPath}`);
|
|
116
|
-
logInternalError("foreground-control.lock-timeout", err, `controlPath=${controlPath}`);
|
|
117
|
-
throw err;
|
|
118
|
-
}
|
|
119
|
-
} catch {
|
|
120
|
-
/* no pid file — continue to throw */
|
|
121
|
-
}
|
|
122
|
-
const err = new Error(`Foreground control lock timeout for ${controlPath}`);
|
|
123
|
-
logInternalError("foreground-control.lock-timeout", err, `controlPath=${controlPath}`);
|
|
124
|
-
throw err;
|
|
125
|
-
}
|
|
126
|
-
// Stale detection: if the owning process is dead, remove the stale lock
|
|
127
|
-
try {
|
|
128
|
-
const raw = fs.readFileSync(pidFile, "utf-8").trim();
|
|
129
|
-
const ownerPid = Number.parseInt(raw, 10);
|
|
130
|
-
if (!Number.isNaN(ownerPid) && ownerPid !== process.pid) {
|
|
131
|
-
let alive = false;
|
|
132
|
-
try {
|
|
133
|
-
process.kill(ownerPid, 0);
|
|
134
|
-
alive = true;
|
|
135
|
-
} catch {
|
|
136
|
-
/* dead */
|
|
137
|
-
}
|
|
138
|
-
if (!alive) {
|
|
139
|
-
const stat = fs.statSync(lockDir);
|
|
140
|
-
if (Date.now() - stat.mtimeMs > staleMs) {
|
|
141
|
-
fs.rmSync(lockDir, {
|
|
142
|
-
recursive: true,
|
|
143
|
-
force: true,
|
|
144
|
-
});
|
|
145
|
-
continue;
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
} catch {
|
|
150
|
-
/* no pid file — fall through to sleep */
|
|
151
|
-
}
|
|
152
|
-
// Brief sleep to avoid CPU spinning
|
|
153
|
-
// WI-7.3 (M7 spec §5): kept sync. Making this async would force
|
|
154
|
-
// requestForegroundControl() + its caller chain async (10+
|
|
155
|
-
// callers in message-tool/lifecycle paths). 10ms spin per
|
|
156
|
-
// iteration is bounded by the 5s timeout above (max ~500 iters
|
|
157
|
-
// = 5s total worst-case block, matching the lock-timeout).
|
|
158
|
-
sleepSync(10);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
};
|
|
162
|
-
const releaseLock = (): void => {
|
|
163
|
-
try {
|
|
164
|
-
fs.rmSync(lockDir, { recursive: true, force: true });
|
|
165
|
-
} catch {
|
|
166
|
-
/* best-effort */
|
|
167
|
-
}
|
|
168
|
-
};
|
|
169
|
-
|
|
170
|
-
acquireLock();
|
|
171
|
-
try {
|
|
196
|
+
// F2: this module-level lock is SHARED with the interrupt-guard consumer —
|
|
197
|
+
// the producer append and the consumer ack serialize on the same lock dir
|
|
198
|
+
// (`${controlPath}.lock`), closing the unlocked whole-file RMW race.
|
|
199
|
+
return withForegroundControlLock(controlPath, (): ForegroundControlRequest => {
|
|
200
|
+
let requests: ForegroundControlRequest[] = [];
|
|
172
201
|
if (fs.existsSync(controlPath)) {
|
|
173
202
|
try {
|
|
174
203
|
const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
|
|
@@ -201,9 +230,7 @@ export function writeForegroundInterruptRequest(
|
|
|
201
230
|
logInternalError("foreground_control.append", e, "type=foreground.interrupt_requested");
|
|
202
231
|
}
|
|
203
232
|
return request;
|
|
204
|
-
}
|
|
205
|
-
releaseLock();
|
|
206
|
-
}
|
|
233
|
+
});
|
|
207
234
|
}
|
|
208
235
|
|
|
209
236
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
@@ -217,12 +244,12 @@ export function writeForegroundInterruptRequest(
|
|
|
217
244
|
// (writeForegroundInterruptRequest above) so the control-file contract has
|
|
218
245
|
// one home.
|
|
219
246
|
//
|
|
220
|
-
// Polling/ack semantics carried over
|
|
221
|
-
//
|
|
222
|
-
//
|
|
223
|
-
//
|
|
224
|
-
// the ack write fails),
|
|
225
|
-
// interval.unref().
|
|
247
|
+
// Polling/ack semantics carried over from the previous inline guard in
|
|
248
|
+
// background-runner.ts: existsSync+readFileSync+JSON.Parse per tick, interval
|
|
249
|
+
// PI_CREW_INTERRUPT_GUARD_INTERVAL_MS (default 1000ms), synchronous atomic ack
|
|
250
|
+
// write (now under the producer's dir-lock — F2), once-per-request gate
|
|
251
|
+
// (defense-in-depth when the ack write fails),
|
|
252
|
+
// appendEventBuffered("async.interrupt_detected"), interval.unref().
|
|
226
253
|
//
|
|
227
254
|
// What an interrupt DOES is host-scoped via onInterrupt:
|
|
228
255
|
// - detached runner (background-runner.startInterruptGuard): process-wide
|
|
@@ -261,21 +288,42 @@ export function startRunInterruptGuard(opts: RunInterruptGuardOptions): () => vo
|
|
|
261
288
|
const interval = setInterval(() => {
|
|
262
289
|
try {
|
|
263
290
|
if (!fs.existsSync(controlPath)) return;
|
|
264
|
-
const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as
|
|
265
|
-
requests?: Array<{ type: string; acknowledged?: boolean }>;
|
|
266
|
-
};
|
|
291
|
+
const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as ForegroundControlFile;
|
|
267
292
|
const last = parsed.requests?.at(-1);
|
|
268
293
|
if (last?.type === "interrupt" && last?.acknowledged !== true) {
|
|
269
294
|
if (interruptHandled) return;
|
|
270
295
|
interruptHandled = true;
|
|
271
296
|
// RT-4 FIX: write acknowledged:true back SYNCHRONOUSLY so the next
|
|
272
297
|
// tick does not re-fire. Sync because a polling callback cannot await.
|
|
298
|
+
// F2 FIX: the ack is a whole-file read→mutate→write and now runs
|
|
299
|
+
// under the SAME module-level lock as the producer
|
|
300
|
+
// (withForegroundControlLock) with a FRESH read inside the critical
|
|
301
|
+
// section. The old path wrote back the stale tick snapshot
|
|
302
|
+
// unlocked — a producer append racing the tick was clobbered
|
|
303
|
+
// (lost interrupt request).
|
|
273
304
|
try {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
305
|
+
withForegroundControlLock(controlPath, () => {
|
|
306
|
+
if (!fs.existsSync(controlPath)) return;
|
|
307
|
+
let fresh: ForegroundControlFile;
|
|
308
|
+
try {
|
|
309
|
+
fresh = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as ForegroundControlFile;
|
|
310
|
+
} catch {
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
const reqs = fresh.requests;
|
|
314
|
+
if (!Array.isArray(reqs) || reqs.length === 0) return;
|
|
315
|
+
// Ack the exact request that fired (by id when present);
|
|
316
|
+
// fall back to the tail entry so id-less legacy files
|
|
317
|
+
// keep acking like before.
|
|
318
|
+
let target = last.id !== undefined ? reqs.find((r) => r.id === last.id) : undefined;
|
|
319
|
+
if (target === undefined) {
|
|
320
|
+
const tail = reqs[reqs.length - 1];
|
|
321
|
+
if (tail?.type === "interrupt" && tail.acknowledged !== true) target = tail;
|
|
322
|
+
}
|
|
323
|
+
if (target === undefined || target.type !== "interrupt" || target.acknowledged === true) return;
|
|
324
|
+
target.acknowledged = true;
|
|
325
|
+
atomicWriteFile(controlPath, JSON.stringify(fresh, null, 2));
|
|
326
|
+
});
|
|
279
327
|
} catch {
|
|
280
328
|
/* best-effort ack — interruptHandled gate prevents re-fire */
|
|
281
329
|
}
|
|
@@ -803,6 +803,12 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
|
|
|
803
803
|
// manifest's model routing snapshot onto each turn manifest too, so
|
|
804
804
|
// status/resume paths see the same routing the turn was dispatched with.
|
|
805
805
|
...(manifest.modelContext ? { modelContext: manifest.modelContext } : {}),
|
|
806
|
+
// FINDING B (2026-10-31): inherit the OUTER manifest's dispatcher
|
|
807
|
+
// (session) PUBLIC verify key — the session that launched the loop is
|
|
808
|
+
// the process that answers asks/steers these turn workers, and the
|
|
809
|
+
// turn manifests are created here in the DETACHED runner (whose own
|
|
810
|
+
// keypair is a different process than the responder).
|
|
811
|
+
...(manifest.dispatcherVerifyKey ? { dispatcherVerifyKey: manifest.dispatcherVerifyKey } : {}),
|
|
806
812
|
});
|
|
807
813
|
goal =
|
|
808
814
|
store.patch(
|