pi-crew 0.11.18 → 0.11.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/NOTICE.md +13 -0
- package/dist/index.mjs +569 -521
- package/package.json +3 -3
- package/skills/real-test-pi-crew/SKILL.md +4 -1
- package/src/config/env-allowlist-registry.ts +8 -0
- package/src/config/env-vars.ts +11 -0
- package/src/runtime/child-pi/mock-fixtures.ts +36 -0
- package/src/runtime/live-session/live-session-runtime.ts +17 -24
- package/src/runtime/output/pi-json-output.ts +12 -25
- package/src/runtime/recovery/crash-recovery.ts +4 -0
- package/src/runtime/recovery/retry-executor.ts +90 -0
- package/src/runtime/session-usage.ts +48 -5
- package/src/runtime/task-runner/child-executor.ts +7 -2
- package/src/runtime/task-runner/pre-execution.ts +1 -1
- package/src/runtime/task-runner.ts +136 -81
- package/src/runtime/verification/verify-gate.ts +164 -7
- package/src/state/atomic-write.ts +10 -3
- package/src/state/coordination/locks.ts +118 -2
- package/src/state/coordination/process-lock.ts +199 -0
- package/src/state/event-log/cursor.ts +292 -84
- package/src/state/event-log/event-log-rotation.ts +30 -1
- package/src/state/event-log/poison-marker.ts +251 -0
- package/src/state/event-log/worker-atomic-writer.ts +282 -41
- package/src/state/types.ts +14 -3
- package/src/utils/incremental-reader.ts +316 -36
|
@@ -160,92 +160,147 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
160
160
|
// to its prompt (it diagnoses; it does not re-derive exit codes).
|
|
161
161
|
// Disabled by PI_CREW_VERIFY_GATE=0; scaffold runtime never gates.
|
|
162
162
|
if (ctx.task.role === "verifier" && ctx.runtimeKind !== "scaffold" && resolveVerifyGateEnabled()) {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
163
|
+
// PHASE 4 (U13-ADJ): the verify-gate itself must NEVER fail a task —
|
|
164
|
+
// any unexpected error in the pre-gate block (engine bug, FS failure
|
|
165
|
+
// on an artifact write) degrades to the ungated verifier spawn
|
|
166
|
+
// (exactly the pre-U13 behavior) instead of throwing out of runTeamTask.
|
|
167
|
+
// FIX (review round): the SKIP-path finalizer call is deliberately NOT
|
|
168
|
+
// inside this try — it is hoisted below. When `return await
|
|
169
|
+
// finalizeTaskResult(...)` lived in the try, a finalizer throw was
|
|
170
|
+
// swallowed by the belt catch and execution FELL THROUGH to the
|
|
171
|
+
// verifier spawn AFTER the skip event was emitted and diagnostics
|
|
172
|
+
// mutated (double-finalize / divergent task state). A finalizer error
|
|
173
|
+
// must propagate out of runTeamTask, exactly like the normal path's
|
|
174
|
+
// finalizer (pre-belt semantics).
|
|
175
|
+
let skipFinalize: TaskExecutionResult | undefined;
|
|
176
|
+
try {
|
|
177
|
+
const preGate = await runVerifierGatePreGate({
|
|
178
|
+
cwd: ctx.task.cwd,
|
|
179
|
+
signal: input.signal,
|
|
180
|
+
classifier: {
|
|
181
|
+
enabled: resolveClassifierEnabled(input.runtimeConfig?.classifierEnabled),
|
|
182
|
+
modelRegistry: input.modelRegistry,
|
|
183
|
+
classifierModel: resolveClassifierModel(input.runtimeConfig?.classifierModel),
|
|
184
|
+
eventsPath: manifest.eventsPath,
|
|
185
|
+
runId: manifest.runId,
|
|
186
|
+
taskId: ctx.task.id,
|
|
187
|
+
},
|
|
188
|
+
metricRegistry: input.metricRegistry,
|
|
189
|
+
});
|
|
190
|
+
// Diagnostic event (task.verifier_pre_gate — U6B's event type; the
|
|
191
|
+
// TEAM_EVENT_TYPES registry entry stays report-only pending a
|
|
192
|
+
// src/state/** lane, same as classifier-pre-gate's own appends).
|
|
193
|
+
appendEventFireAndForget(manifest.eventsPath, {
|
|
194
|
+
type: "task.verifier_pre_gate",
|
|
171
195
|
runId: manifest.runId,
|
|
172
196
|
taskId: ctx.task.id,
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
taskId: ctx.task.id,
|
|
183
|
-
message: preGate.decision.skipVerifier
|
|
184
|
-
? `Deterministic verify_gate ${preGate.outcome.verdict} — LLM verifier spawn SKIPPED (${preGate.decision.reason})`
|
|
185
|
-
: `Deterministic verify_gate ${preGate.outcome.verdict} — LLM verifier spawns with gate context (${preGate.decision.reason})`,
|
|
186
|
-
data: {
|
|
187
|
-
verdict: preGate.outcome.verdict,
|
|
188
|
-
decision: preGate.decision,
|
|
189
|
-
evidenceLines: preGate.outcome.evidenceLines,
|
|
190
|
-
...(preGate.preGate ? { classifierAction: preGate.preGate.action } : {}),
|
|
191
|
-
},
|
|
192
|
-
});
|
|
193
|
-
if (preGate.decision.skipVerifier) {
|
|
194
|
-
// Terminalize WITHOUT spawning: synthesize the branch result (gate
|
|
195
|
-
// summary as the task result artifact + stdout) and reuse the full
|
|
196
|
-
// finalizer — events, hooks, persistence, verification evidence.
|
|
197
|
-
const summary = renderVerifyGateSummary(preGate.outcome, preGate.decision.reason);
|
|
198
|
-
const resultArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
199
|
-
kind: "result",
|
|
200
|
-
relativePath: `results/${ctx.task.id}.txt`,
|
|
201
|
-
content: `${summary}\n`,
|
|
202
|
-
producer: ctx.task.id,
|
|
203
|
-
});
|
|
204
|
-
ctx.task = {
|
|
205
|
-
...ctx.task,
|
|
206
|
-
// Provenance: this verdict came from the deterministic gate, not a
|
|
207
|
-
// model attempt (U6B mount note: evaluator/model provenance marker).
|
|
208
|
-
diagnostics: {
|
|
209
|
-
...(ctx.task.diagnostics ?? {}),
|
|
210
|
-
verifyGate: {
|
|
211
|
-
verdict: preGate.outcome.verdict,
|
|
212
|
-
reason: preGate.decision.reason,
|
|
213
|
-
evidenceLines: preGate.outcome.evidenceLines,
|
|
214
|
-
},
|
|
197
|
+
message: preGate.decision.skipVerifier
|
|
198
|
+
? `Deterministic verify_gate ${preGate.outcome.verdict} — LLM verifier spawn SKIPPED (${preGate.decision.reason})`
|
|
199
|
+
: `Deterministic verify_gate ${preGate.outcome.verdict} — LLM verifier spawns with gate context (${preGate.decision.reason})`,
|
|
200
|
+
data: {
|
|
201
|
+
verdict: preGate.outcome.verdict,
|
|
202
|
+
decision: preGate.decision,
|
|
203
|
+
evidenceLines: preGate.outcome.evidenceLines,
|
|
204
|
+
...(preGate.outcome.determinismNote ? { determinismNote: preGate.outcome.determinismNote } : {}),
|
|
205
|
+
...(preGate.preGate ? { classifierAction: preGate.preGate.action } : {}),
|
|
215
206
|
},
|
|
216
|
-
};
|
|
217
|
-
return await finalizeTaskResult(ctx, {
|
|
218
|
-
resultArtifact,
|
|
219
|
-
logArtifact: undefined,
|
|
220
|
-
transcriptArtifact: undefined,
|
|
221
|
-
exitCode: 0,
|
|
222
|
-
error: undefined,
|
|
223
|
-
modelAttempts: [],
|
|
224
|
-
parsedOutput: undefined,
|
|
225
|
-
finalStdout: summary,
|
|
226
|
-
transcriptPath: undefined,
|
|
227
|
-
terminalEvidence: [],
|
|
228
|
-
startupEvidence: ctx.startupEvidence,
|
|
229
207
|
});
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
})
|
|
208
|
+
if (preGate.decision.skipVerifier) {
|
|
209
|
+
// Terminalize WITHOUT spawning: synthesize the branch result (gate
|
|
210
|
+
// summary as the task result artifact + stdout) and reuse the full
|
|
211
|
+
// finalizer — events, hooks, persistence, verification evidence.
|
|
212
|
+
const summary = renderVerifyGateSummary(preGate.outcome, preGate.decision.reason);
|
|
213
|
+
// PHASE 4 gap 1 (U13-ADJ): the skip-path artifact write must NEVER
|
|
214
|
+
// fail the task — an FS error here (ENOSPC / EACCES / ENOTDIR)
|
|
215
|
+
// degrades to "no result artifact" (the finalizer treats it as
|
|
216
|
+
// optional) instead of throwing out of runTeamTask.
|
|
217
|
+
let resultArtifact: ArtifactDescriptor | undefined;
|
|
218
|
+
try {
|
|
219
|
+
resultArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
220
|
+
kind: "result",
|
|
221
|
+
relativePath: `results/${ctx.task.id}.txt`,
|
|
222
|
+
content: `${summary}\n`,
|
|
223
|
+
producer: ctx.task.id,
|
|
224
|
+
});
|
|
225
|
+
} catch (error) {
|
|
226
|
+
console.error(
|
|
227
|
+
`[task-runner] verify-gate skip-path writeArtifact failed — degrading (task continues without a result artifact): ${
|
|
228
|
+
error instanceof Error ? error.message : String(error)
|
|
229
|
+
}`,
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
ctx.task = {
|
|
233
|
+
...ctx.task,
|
|
234
|
+
// Provenance: this verdict came from the deterministic gate, not a
|
|
235
|
+
// model attempt (U6B mount note: evaluator/model provenance marker).
|
|
236
|
+
diagnostics: {
|
|
237
|
+
...(ctx.task.diagnostics ?? {}),
|
|
238
|
+
verifyGate: {
|
|
239
|
+
verdict: preGate.outcome.verdict,
|
|
240
|
+
reason: preGate.decision.reason,
|
|
241
|
+
evidenceLines: preGate.outcome.evidenceLines,
|
|
242
|
+
},
|
|
243
|
+
},
|
|
244
|
+
};
|
|
245
|
+
// FIX (review round): prepare the finalizer args here but DO NOT
|
|
246
|
+
// call finalizeTaskResult inside the belt try — see the hoisted
|
|
247
|
+
// call right after the catch below.
|
|
248
|
+
skipFinalize = {
|
|
249
|
+
resultArtifact,
|
|
250
|
+
logArtifact: undefined,
|
|
251
|
+
transcriptArtifact: undefined,
|
|
252
|
+
exitCode: 0,
|
|
253
|
+
error: undefined,
|
|
254
|
+
modelAttempts: [],
|
|
255
|
+
parsedOutput: undefined,
|
|
256
|
+
finalStdout: summary,
|
|
257
|
+
transcriptPath: undefined,
|
|
258
|
+
terminalEvidence: [],
|
|
259
|
+
startupEvidence: ctx.startupEvidence,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
if (preGate.promptContext) {
|
|
263
|
+
// Spawn proceeds — attach the gate results to the verifier prompt
|
|
264
|
+
// (both channels: full prompt + the user-message half). The prompt
|
|
265
|
+
// artifact was written in pre-execution with the PRE-gate prompt;
|
|
266
|
+
// rewrite it so the persisted artifact matches what the worker saw.
|
|
267
|
+
ctx.prompt = `${ctx.prompt}\n\n${preGate.promptContext}`;
|
|
268
|
+
if (ctx.userPrompt !== undefined) ctx.userPrompt = `${ctx.userPrompt}\n\n${preGate.promptContext}`;
|
|
269
|
+
if (ctx.promptArtifact) {
|
|
270
|
+
// PHASE 4 gap 1 (cont.): same never-fail-a-task degrade for the
|
|
271
|
+
// prompt-artifact rewrite — on FS failure keep the pre-gate
|
|
272
|
+
// artifact on disk (the worker still gets the extended prompt).
|
|
273
|
+
try {
|
|
274
|
+
ctx.promptArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
275
|
+
kind: "prompt",
|
|
276
|
+
// Same relative-path convention pre-execution used
|
|
277
|
+
// (prompts/<taskId>.md) — ArtifactDescriptor carries only the
|
|
278
|
+
// absolute path, so the rewrite re-derives the relative path.
|
|
279
|
+
relativePath: `prompts/${ctx.task.id}.md`,
|
|
280
|
+
content: `${ctx.prompt}\n`,
|
|
281
|
+
producer: ctx.task.id,
|
|
282
|
+
});
|
|
283
|
+
} catch (error) {
|
|
284
|
+
console.error(
|
|
285
|
+
`[task-runner] verify-gate prompt-artifact rewrite failed — degrading (persisted prompt keeps the pre-gate content): ${
|
|
286
|
+
error instanceof Error ? error.message : String(error)
|
|
287
|
+
}`,
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
248
291
|
}
|
|
292
|
+
} catch (error) {
|
|
293
|
+
console.error(
|
|
294
|
+
`[task-runner] verify-gate pre-gate error — degrading to the ungated verifier spawn: ${
|
|
295
|
+
error instanceof Error ? error.message : String(error)
|
|
296
|
+
}`,
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
// FIX (review round): skip-path finalizer — OUTSIDE the belt try so a
|
|
300
|
+
// finalizer throw propagates out of runTeamTask (pre-belt semantics)
|
|
301
|
+
// instead of being swallowed and falling through to the verifier spawn.
|
|
302
|
+
if (skipFinalize !== undefined) {
|
|
303
|
+
return await finalizeTaskResult(ctx, skipFinalize);
|
|
249
304
|
}
|
|
250
305
|
}
|
|
251
306
|
// Destructure mutable fields for branch closures + sync-back
|
|
@@ -32,6 +32,13 @@
|
|
|
32
32
|
* outcome escalates. A FAILED gate never reaches the classifier — a cheap
|
|
33
33
|
* classifier may never rubber-stamp a failure (U6B escalation policy),
|
|
34
34
|
* so consulting it there would only burn a call.
|
|
35
|
+
* - Layer 3 (PHASE 4, U13-ADJ determinism, DEFAULT OFF): when
|
|
36
|
+
* PI_CREW_VERIFY_DETERMINISM=1 and the default checks are all green, the
|
|
37
|
+
* most critical check (test-critical in the default preset) is RERUN
|
|
38
|
+
* once; a disagreement (flaky) or a rerun executor failure downgrades
|
|
39
|
+
* the verdict to INCONCLUSIVE — never FAILED, never a task failure. It
|
|
40
|
+
* never changes the FAILED↔spawn logic; flag off keeps the gate
|
|
41
|
+
* bit-for-bit identical to pre-U13-ADJ.
|
|
35
42
|
*
|
|
36
43
|
* SAFETY: a gate that cannot run (disabled by env, no checks resolved,
|
|
37
44
|
* executor failure) degrades to "spawn the LLM verifier as today" — the
|
|
@@ -86,6 +93,10 @@ export interface VerifyGateOutcome {
|
|
|
86
93
|
evidenceLines: string[];
|
|
87
94
|
allPassed: boolean;
|
|
88
95
|
totalDurationMs: number;
|
|
96
|
+
/** PHASE 4 (U13-ADJ): set ONLY by the layer-3 determinism rerun when it
|
|
97
|
+
* detected flakiness or a rerun executor failure — the note is also
|
|
98
|
+
* appended to evidenceLines and shapes the rendered context trailer. */
|
|
99
|
+
determinismNote?: string;
|
|
89
100
|
}
|
|
90
101
|
|
|
91
102
|
/** Injected command executor. The TOOL path wraps ctx.executeTool("bash");
|
|
@@ -125,6 +136,16 @@ export function resolveVerifyGateTimeoutMs(explicit?: number): number {
|
|
|
125
136
|
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_VERIFY_GATE_TIMEOUT_MS;
|
|
126
137
|
}
|
|
127
138
|
|
|
139
|
+
/** PHASE 4 (U13-ADJ, upgrade 2026-10-13) layer-3 toggle:
|
|
140
|
+
* PI_CREW_VERIFY_DETERMINISM beats the explicit arg; default OFF (unset →
|
|
141
|
+
* explicit ?? false). Flag off ⇒ the gate runs exactly as pre-U13-ADJ
|
|
142
|
+
* (no third check, no rerun, no verdict downgrade — bit-for-bit). */
|
|
143
|
+
export function resolveVerifyDeterminismEnabled(explicit?: boolean): boolean {
|
|
144
|
+
const env = getCrewEnvBool("PI_CREW_VERIFY_DETERMINISM");
|
|
145
|
+
if (env !== undefined) return env;
|
|
146
|
+
return explicit ?? false;
|
|
147
|
+
}
|
|
148
|
+
|
|
128
149
|
/**
|
|
129
150
|
* Default check preset — the spec's "check cấu hình" set, resolved from the
|
|
130
151
|
* target repo's package.json scripts so the gate adapts instead of failing:
|
|
@@ -242,6 +263,106 @@ export function createSubprocessVerifyGateExecutor(): VerifyGateExecutor {
|
|
|
242
263
|
};
|
|
243
264
|
}
|
|
244
265
|
|
|
266
|
+
/** PHASE 4 (U13-ADJ) layer-3 result — one entry per pre-gate execution. */
|
|
267
|
+
export interface DeterminismLayerResult {
|
|
268
|
+
/** false → the layer did not run (budget rule / no eligible target). */
|
|
269
|
+
ran: boolean;
|
|
270
|
+
/** Metric label: "deterministic" | "flaky" | "rerun_error" when ran;
|
|
271
|
+
* the skip reason ("verdict_failed" | "verdict_inconclusive" |
|
|
272
|
+
* "no_critical_pass") otherwise. */
|
|
273
|
+
result: string;
|
|
274
|
+
/** The gate outcome AFTER the layer (the layer-1 outcome when !ran). */
|
|
275
|
+
outcome: VerifyGateOutcome;
|
|
276
|
+
/** Non-empty iff the rerun disagreed (flaky) or its executor failed. */
|
|
277
|
+
note: string;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/** Pick the layer-3 rerun target: the MOST critical check — the LAST check
|
|
281
|
+
* with critical !== false that ran and PASSED (default preset order puts
|
|
282
|
+
* test-critical last: tests are the strongest determinism signal). Checks
|
|
283
|
+
* that failed are never rerun (pointless — a failure already escalates). */
|
|
284
|
+
function pickDeterminismRerunTarget(
|
|
285
|
+
checks: VerifyGateCheck[],
|
|
286
|
+
results: VerifyGateCheckResult[],
|
|
287
|
+
): { check: VerifyGateCheck; result: VerifyGateCheckResult } | undefined {
|
|
288
|
+
let target: { check: VerifyGateCheck; result: VerifyGateCheckResult } | undefined;
|
|
289
|
+
for (let i = 0; i < checks.length; i++) {
|
|
290
|
+
const check = checks[i];
|
|
291
|
+
const result = results[i];
|
|
292
|
+
if (!check || !result || result.skipped || !result.passed) continue;
|
|
293
|
+
if (check.critical === false) continue; // critical undefined = critical
|
|
294
|
+
target = { check, result };
|
|
295
|
+
}
|
|
296
|
+
return target;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** PHASE 4 (U13-ADJ) layer 3 — the determinism rerun. Runs ONLY when the
|
|
300
|
+
* layer-1 verdict is PASS (budget rule: never rerun an already-FAILED gate)
|
|
301
|
+
* AND at least one critical check passed. Semantics:
|
|
302
|
+
* - rerun exit 0 (agrees with run 1) → verdict stays PASS, the rerun is
|
|
303
|
+
* recorded as a passing `determinism:<name>` check for evidence;
|
|
304
|
+
* - rerun exit ≠ 0 (DISAGREES — flaky) → verdict INCONCLUSIVE (never
|
|
305
|
+
* FAILED) + note "deterministic-check: flaky <name>";
|
|
306
|
+
* - rerun executor failure (exitCode null) → INCONCLUSIVE (soft-degrade,
|
|
307
|
+
* never FAILED) + note "deterministic-check: rerun executor error".
|
|
308
|
+
* The layer can ONLY downgrade its own rerun check to INCONCLUSIVE — it
|
|
309
|
+
* never changes the FAILED↔spawn decision logic. NEVER rejects. */
|
|
310
|
+
export async function applyVerifyGateDeterminismLayer(
|
|
311
|
+
checks: VerifyGateCheck[],
|
|
312
|
+
outcome: VerifyGateOutcome,
|
|
313
|
+
exec: VerifyGateExecutor,
|
|
314
|
+
opts: { cwd: string; signal?: AbortSignal; timeoutMs?: number },
|
|
315
|
+
): Promise<DeterminismLayerResult> {
|
|
316
|
+
// Budget rule: no pointless rerun when the gate already failed / resolved
|
|
317
|
+
// nothing — layer 3 only re-tests a GREEN gate for hidden nondeterminism.
|
|
318
|
+
if (outcome.verdict !== "PASS") {
|
|
319
|
+
return { ran: false, result: `verdict_${outcome.verdict.toLowerCase()}`, outcome, note: "" };
|
|
320
|
+
}
|
|
321
|
+
const target = pickDeterminismRerunTarget(checks, outcome.checks);
|
|
322
|
+
if (!target) {
|
|
323
|
+
return { ran: false, result: "no_critical_pass", outcome, note: "" };
|
|
324
|
+
}
|
|
325
|
+
const timeoutMs = resolveVerifyGateTimeoutMs(opts.timeoutMs);
|
|
326
|
+
const start = Date.now();
|
|
327
|
+
let rerun: VerifyGateExecutorResult;
|
|
328
|
+
try {
|
|
329
|
+
rerun = await exec(target.check.command, { cwd: opts.cwd, timeoutMs, signal: opts.signal });
|
|
330
|
+
} catch (error) {
|
|
331
|
+
// Executor contract says never-reject; defend anyway (soft-degrade).
|
|
332
|
+
rerun = { exitCode: null, output: "", error: error instanceof Error ? error.message : String(error) };
|
|
333
|
+
}
|
|
334
|
+
const rerunCheck: VerifyGateCheckResult = {
|
|
335
|
+
name: `determinism:${target.check.name}`,
|
|
336
|
+
command: target.check.command,
|
|
337
|
+
exitCode: rerun.exitCode,
|
|
338
|
+
passed: rerun.exitCode === 0,
|
|
339
|
+
durationMs: Date.now() - start,
|
|
340
|
+
outputTail: tailOutput(rerun.output ?? ""),
|
|
341
|
+
...(rerun.error ? { error: rerun.error } : {}),
|
|
342
|
+
};
|
|
343
|
+
const base = evaluateVerifyGateResults([...outcome.checks, rerunCheck]);
|
|
344
|
+
if (rerun.exitCode === 0) {
|
|
345
|
+
return { ran: true, result: "deterministic", outcome: base, note: "" };
|
|
346
|
+
}
|
|
347
|
+
const note =
|
|
348
|
+
rerun.exitCode === null
|
|
349
|
+
? `deterministic-check: rerun executor error (${target.check.name})`
|
|
350
|
+
: `deterministic-check: flaky ${target.check.name}`;
|
|
351
|
+
return {
|
|
352
|
+
ran: true,
|
|
353
|
+
result: rerun.exitCode === null ? "rerun_error" : "flaky",
|
|
354
|
+
// The rerun check's failure would otherwise read as gate FAILED — the
|
|
355
|
+
// ONLY thing layer 3 may do is downgrade ITS OWN rerun to INCONCLUSIVE.
|
|
356
|
+
outcome: {
|
|
357
|
+
...base,
|
|
358
|
+
verdict: "INCONCLUSIVE",
|
|
359
|
+
evidenceLines: [...base.evidenceLines, note],
|
|
360
|
+
determinismNote: note,
|
|
361
|
+
},
|
|
362
|
+
note,
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
|
|
245
366
|
export interface VerifierSpawnDecision {
|
|
246
367
|
/** true → do NOT spawn the LLM verifier; the gate verdict stands. */
|
|
247
368
|
skipVerifier: boolean;
|
|
@@ -286,10 +407,19 @@ export function renderVerifyGateContext(outcome: VerifyGateOutcome): string {
|
|
|
286
407
|
lines.push(`### ${check.name} output tail`, "```", tail, "```");
|
|
287
408
|
}
|
|
288
409
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
410
|
+
// PHASE 4 gap 2 (U13-ADJ): distinguish "executor died" (exit=err — spawn /
|
|
411
|
+
// timer failure, NO exit code) from a real exit≠0. An executor death is
|
|
412
|
+
// NOT evidence the build is broken — asserting it as ground truth would
|
|
413
|
+
// bias the LLM verifier against a possibly-fine build. Real exit≠0 keeps
|
|
414
|
+
// the original wording bit-for-bit; a flaky rerun (layer 3) gets its own
|
|
415
|
+
// trailer because "do NOT re-run / ground truth" would be provably wrong.
|
|
416
|
+
const executorDeath = outcome.checks.some((c) => !c.skipped && c.exitCode === null);
|
|
417
|
+
const trailer = executorDeath
|
|
418
|
+
? "A verification command could not run (executor error — exit=err). Treat the build/test state as UNKNOWN; run the build/tests yourself before concluding."
|
|
419
|
+
: outcome.determinismNote
|
|
420
|
+
? "The gate rerun detected NONDETERMINISM (a check passed once and failed on rerun). These gate results are NOT reliable — re-run the affected command yourself and treat the build/test state as UNKNOWN."
|
|
421
|
+
: "The deterministic gate already ran these commands — do NOT re-run them; verify the WORK (diff, files, acceptance), using these results as ground truth for build/test state.";
|
|
422
|
+
lines.push("", trailer);
|
|
293
423
|
return lines.join("\n");
|
|
294
424
|
}
|
|
295
425
|
|
|
@@ -314,6 +444,9 @@ export interface VerifierGatePreGateInput {
|
|
|
314
444
|
taskId?: string;
|
|
315
445
|
};
|
|
316
446
|
metricRegistry?: MetricRegistry;
|
|
447
|
+
/** PHASE 4 (U13-ADJ): explicit layer-3 determinism toggle — env
|
|
448
|
+
* PI_CREW_VERIFY_DETERMINISM beats this; both default OFF. */
|
|
449
|
+
determinism?: boolean;
|
|
317
450
|
}
|
|
318
451
|
|
|
319
452
|
export interface VerifierGatePreGateResult {
|
|
@@ -333,18 +466,41 @@ function incMetric(registry: MetricRegistry | undefined, name: string, help: str
|
|
|
333
466
|
}
|
|
334
467
|
|
|
335
468
|
/**
|
|
336
|
-
* Spawn-site orchestration: run the deterministic gate,
|
|
469
|
+
* Spawn-site orchestration: run the deterministic gate, optionally the U13-ADJ
|
|
470
|
+
* layer-3 determinism rerun (flag-gated, default OFF), consult the U6B
|
|
337
471
|
* classifier layer when inconclusive, resolve the two-layer spawn decision,
|
|
338
472
|
* and emit the acceptance metrics. NEVER rejects — executor failures degrade
|
|
339
473
|
* to INCONCLUSIVE checks, which escalate to the LLM verifier.
|
|
340
474
|
*/
|
|
341
475
|
export async function runVerifierGatePreGate(input: VerifierGatePreGateInput): Promise<VerifierGatePreGateResult> {
|
|
342
476
|
const checks = input.checks ?? defaultVerifyGateChecks(input.cwd);
|
|
343
|
-
const
|
|
477
|
+
const exec = createSubprocessVerifyGateExecutor();
|
|
478
|
+
let outcome = await runVerifyGate(checks, exec, {
|
|
344
479
|
cwd: input.cwd,
|
|
345
480
|
signal: input.signal,
|
|
346
481
|
timeoutMs: input.timeoutMs,
|
|
347
482
|
});
|
|
483
|
+
// ── PHASE 4 (U13-ADJ) layer 3: determinism rerun (DEFAULT OFF) ──────────
|
|
484
|
+
// Only re-tests a GREEN gate: a flaky critical check or a rerun executor
|
|
485
|
+
// failure downgrades PASS→INCONCLUSIVE (never FAILED), so a nondeterministic
|
|
486
|
+
// green gate can no longer skip the LLM verifier. Flag off ⇒ skipped
|
|
487
|
+
// entirely (bit-for-bit pre-U13-ADJ behavior).
|
|
488
|
+
let determinismNote = "";
|
|
489
|
+
if (resolveVerifyDeterminismEnabled(input.determinism)) {
|
|
490
|
+
const layer = await applyVerifyGateDeterminismLayer(checks, outcome, exec, {
|
|
491
|
+
cwd: input.cwd,
|
|
492
|
+
signal: input.signal,
|
|
493
|
+
timeoutMs: input.timeoutMs,
|
|
494
|
+
});
|
|
495
|
+
outcome = layer.outcome;
|
|
496
|
+
determinismNote = layer.note;
|
|
497
|
+
incMetric(
|
|
498
|
+
input.metricRegistry,
|
|
499
|
+
"crew.verification.verify_gate_determinism_runs_total",
|
|
500
|
+
"Layer-3 determinism reruns (U13-ADJ) by result",
|
|
501
|
+
{ result: layer.result },
|
|
502
|
+
);
|
|
503
|
+
}
|
|
348
504
|
let preGate: VerifierPreGateResult | undefined;
|
|
349
505
|
if (outcome.verdict === "INCONCLUSIVE" && input.classifier?.enabled) {
|
|
350
506
|
try {
|
|
@@ -389,6 +545,7 @@ export async function runVerifierGatePreGate(input: VerifierGatePreGateInput): P
|
|
|
389
545
|
outcome,
|
|
390
546
|
decision,
|
|
391
547
|
...(preGate ? { preGate } : {}),
|
|
392
|
-
promptContext:
|
|
548
|
+
promptContext:
|
|
549
|
+
decision.attachGateContext || (determinismNote !== "" && !decision.skipVerifier) ? renderVerifyGateContext(outcome) : "",
|
|
393
550
|
};
|
|
394
551
|
}
|
|
@@ -828,16 +828,23 @@ export function atomicWriteFile(filePath: string, content: string, options?: Ato
|
|
|
828
828
|
export async function atomicWriteFileAsync(filePath: string, content: string, options?: AtomicWriteOptions): Promise<void> {
|
|
829
829
|
cancelPendingCoalescedWrite(filePath);
|
|
830
830
|
const { durability, mode } = normalizeOptions(options);
|
|
831
|
+
// PHASE 0 P3 (U22 bước-0): run the symlink guard BEFORE dispatching to the
|
|
832
|
+
// worker. Previously the worker branch sat ABOVE this guard, so an unsafe
|
|
833
|
+
// target was written by the worker without the main-side check, and the
|
|
834
|
+
// normalized options were dropped entirely (worker hardcoded 0o600).
|
|
835
|
+
if (!isSymlinkSafeDirCached(filePath))
|
|
836
|
+
throw new Error(`Refusing to write: target is a symlink or inside untrusted directory: ${filePath}`);
|
|
831
837
|
// Phase 1.5 (RFC 15): when the worker-thread atomic writer is enabled
|
|
832
838
|
// (PI_CREW_WORKER_ATOMIC_WRITER=1), dispatch to a dedicated worker thread
|
|
833
839
|
// that performs SYNC fs ops with no internal yields. Mitigates the
|
|
834
840
|
// non-deterministic V8/libuv crash during event-loop yields in multi-step
|
|
835
841
|
// goal-wrapped workflows.
|
|
836
842
|
if (isWorkerAtomicWriterEnabled()) {
|
|
837
|
-
|
|
843
|
+
// PHASE 0 P3: forward `mode` through the message protocol. Durability is
|
|
844
|
+
// not forwarded — the worker now ALWAYS fsyncs data + parent dir (defect
|
|
845
|
+
// (b) in worker-atomic-writer.ts), i.e. durability:"full" semantics.
|
|
846
|
+
return atomicWriteFileViaWorker(filePath, content, mode);
|
|
838
847
|
}
|
|
839
|
-
if (!isSymlinkSafeDirCached(filePath))
|
|
840
|
-
throw new Error(`Refusing to write: target is a symlink or inside untrusted directory: ${filePath}`);
|
|
841
848
|
// PERF (2026-08-24): shared dir memo — a sync mkdir on an existing dir is one
|
|
842
849
|
// cheap syscall; replacing `await fs.promises.mkdir` is fine because the
|
|
843
850
|
// memoized hit does not throw and does not yield the event loop.
|
|
@@ -3,10 +3,12 @@ import { randomUUID, timingSafeEqual } from "node:crypto";
|
|
|
3
3
|
import * as fs from "node:fs";
|
|
4
4
|
import * as path from "node:path";
|
|
5
5
|
import { DEFAULT_LOCKS } from "../../config/defaults.ts";
|
|
6
|
+
import { getCrewEnvBool } from "../../config/env-vars.ts";
|
|
6
7
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
7
8
|
import { sleepSync } from "../../utils/sleep.ts";
|
|
8
9
|
import { ensureDirSync, isSymlinkSafeDirCached, isSymlinkSafePath } from "../atomic-write.ts";
|
|
9
10
|
import type { TeamRunManifest } from "../types.ts";
|
|
11
|
+
import { acquireProcessLock, isProcessLockHeldError, type ProcessLockHandle } from "./process-lock.ts";
|
|
10
12
|
|
|
11
13
|
export interface RunLockOptions {
|
|
12
14
|
staleMs?: number;
|
|
@@ -305,6 +307,13 @@ function timingSafeTokenMatch(a: string, b: string): boolean {
|
|
|
305
307
|
* never swept, no matter how old.
|
|
306
308
|
*
|
|
307
309
|
* Returns the lock files it removed. Idempotent: a second sweep finds nothing.
|
|
310
|
+
*
|
|
311
|
+
* @deprecated-after-window U21 (2026-10-13): async run-lock holders now also
|
|
312
|
+
* own a kernel process socket (locks.ts `withRunLock`) which needs NO sweep —
|
|
313
|
+
* it dies with its process. This file-lock sweep still covers the SYNC path
|
|
314
|
+
* (`withRunLockSync`, file-only by design) and legacy/foreign holders during
|
|
315
|
+
* the migration window; remove it after the file-lock migration window
|
|
316
|
+
* closes.
|
|
308
317
|
*/
|
|
309
318
|
export function sweepStaleLocks(lockFiles: readonly string[], options: RunLockOptions = {}): { removed: string[] } {
|
|
310
319
|
const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
|
|
@@ -327,6 +336,10 @@ export function sweepStaleLocks(lockFiles: readonly string[], options: RunLockOp
|
|
|
327
336
|
/**
|
|
328
337
|
* US-002: discover every `run.lock` under a runs root
|
|
329
338
|
* (`<runsRoot>/<runId>/run.lock`), bounded, for sweepStaleLocks. Missing → empty.
|
|
339
|
+
*
|
|
340
|
+
* @deprecated-after-window U21: alongside `sweepStaleLocks` — socket-guarded
|
|
341
|
+
* async holders need no sweep; remove after the file-lock migration window
|
|
342
|
+
* closes (sync path + legacy holders keep needing it until then).
|
|
330
343
|
*/
|
|
331
344
|
export function discoverRunLockFiles(runsRoot: string, maxDirs = 500): string[] {
|
|
332
345
|
try {
|
|
@@ -550,6 +563,79 @@ async function acquireLockWithRetryAsync(filePath: string, staleMs: number, kind
|
|
|
550
563
|
}
|
|
551
564
|
}
|
|
552
565
|
|
|
566
|
+
/**
|
|
567
|
+
* U21 (2026-10-13): process-socket supplement toggle for the ASYNC run-lock
|
|
568
|
+
* path. Default ON; `'0'`/`'false'`/`'no'` restores the pre-U21 file-lock-only
|
|
569
|
+
* behaviour bit-for-bit (dual-holder migration window escape hatch). Read via
|
|
570
|
+
* the crew env registry — this is a HOST-process read, never passed to children
|
|
571
|
+
* (the lock name is derived from the run dir path; nothing is inherited).
|
|
572
|
+
*/
|
|
573
|
+
function runLockSocketSupplementEnabled(): boolean {
|
|
574
|
+
return getCrewEnvBool("PI_CREW_RUN_LOCK_SOCKET") ?? true;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
/**
|
|
578
|
+
* U21: the kernel-socket lock name for a run-lock file path. Derived from the
|
|
579
|
+
* ABSOLUTE run dir path so every process agrees on the name for the same run
|
|
580
|
+
* regardless of cwd/relative-path spelling.
|
|
581
|
+
*/
|
|
582
|
+
export function runLockSocketName(filePath: string): string {
|
|
583
|
+
return `run:${path.resolve(filePath)}`;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* U21 dual-guard acquire for the async run-lock: file lock FIRST (existing
|
|
588
|
+
* WAIT/backoff admission of D1 / RR-011 completely unchanged), THEN the kernel
|
|
589
|
+
* process socket (process-lock.ts, vendored from chord).
|
|
590
|
+
*
|
|
591
|
+
* If the socket is held (`reason: "locked"`) we treat it exactly like file
|
|
592
|
+
* contention: drop OUR file token (token-guarded releaseOwnLock) and keep
|
|
593
|
+
* WAITing — same `staleMs * 2` deadline budget the file-acquire loop uses —
|
|
594
|
+
* then retry the whole file+socket acquisition. When BOTH are held the caller
|
|
595
|
+
* owns the run. Non-contention socket failures propagate AFTER the file token
|
|
596
|
+
* is dropped (we never keep the file without the guard that promised it).
|
|
597
|
+
*
|
|
598
|
+
* The socket exists to harden async holders against dual-process entry where
|
|
599
|
+
* O_EXCL file locks are weak (NFS and similar); it intentionally does NOT
|
|
600
|
+
* touch the six file-lock semantics (re-entrance, token release, live WAIT,
|
|
601
|
+
* stale-steal + WARN, EPERM, ENOENT) — those all live in the file tier.
|
|
602
|
+
*/
|
|
603
|
+
async function acquireRunLockGuarded(filePath: string, staleMs: number): Promise<{ token: string; socket: ProcessLockHandle | undefined }> {
|
|
604
|
+
if (!runLockSocketSupplementEnabled()) {
|
|
605
|
+
// Supplement OFF (PI_CREW_RUN_LOCK_SOCKET=0): pre-U21 behaviour, bit-for-bit.
|
|
606
|
+
return { token: await acquireLockWithRetryAsync(filePath, staleMs, "run"), socket: undefined };
|
|
607
|
+
}
|
|
608
|
+
const deadline = Date.now() + staleMs * 2; // same budget as the file-acquire loop
|
|
609
|
+
let attempt = 0;
|
|
610
|
+
for (;;) {
|
|
611
|
+
const token = await acquireLockWithRetryAsync(filePath, staleMs, "run");
|
|
612
|
+
// RR-011 (F02) extended to the dual-guard window: the token must be
|
|
613
|
+
// visible as a LIVE in-process hold from the moment the file tier is
|
|
614
|
+
// ours — the socket listen below is a macrotask and can be delayed by
|
|
615
|
+
// event-loop pressure, which would otherwise make concurrent
|
|
616
|
+
// same-process acquirers misread our fresh token as a stealable corpse.
|
|
617
|
+
runLockHeldTokens.add(token);
|
|
618
|
+
try {
|
|
619
|
+
const socket = await acquireProcessLock(runLockSocketName(filePath));
|
|
620
|
+
return { token, socket };
|
|
621
|
+
} catch (error) {
|
|
622
|
+
// Never keep the file token without the socket guard that follows it.
|
|
623
|
+
runLockHeldTokens.delete(token);
|
|
624
|
+
releaseOwnLock(filePath, token);
|
|
625
|
+
if (!isProcessLockHeldError(error)) throw error;
|
|
626
|
+
// Socket contention — WAIT (timer-based, never sleepSync: the live
|
|
627
|
+
// socket holder may be another async context of THIS process, and
|
|
628
|
+
// blocking the event loop would starve it — the v0.9.26 deadlock
|
|
629
|
+
// class) and retry within the same staleMs*2 deadline.
|
|
630
|
+
if (Date.now() > deadline) {
|
|
631
|
+
throw new Error(`Run '${path.basename(filePath)}' is locked by another operation.`);
|
|
632
|
+
}
|
|
633
|
+
await sleep(Math.min(250, 25 * 2 ** attempt));
|
|
634
|
+
attempt++;
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
553
639
|
/**
|
|
554
640
|
* General-purpose file lock for arbitrary file paths.
|
|
555
641
|
* Uses the same O_EXCL atomic create strategy as run locks.
|
|
@@ -762,6 +848,13 @@ export async function withFileLockAsync<T>(filePath: string, fn: () => Promise<T
|
|
|
762
848
|
}
|
|
763
849
|
}
|
|
764
850
|
|
|
851
|
+
/**
|
|
852
|
+
* SYNC run-lock — FILE-LOCK-ONLY by design (U21): a socket listen cannot be
|
|
853
|
+
* awaited from a synchronous path, so the process-socket supplement of
|
|
854
|
+
* `withRunLock` does not apply here. Mutual exclusion between sync and async
|
|
855
|
+
* holders is still guaranteed by the SHARED file lock (an async holder always
|
|
856
|
+
* takes the file lock first, before its socket supplement).
|
|
857
|
+
*/
|
|
765
858
|
export function withRunLockSync<T>(manifest: TeamRunManifest, fn: () => T, options: RunLockOptions = {}): T {
|
|
766
859
|
const filePath = lockPath(manifest);
|
|
767
860
|
const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
|
|
@@ -798,6 +891,19 @@ export function withRunLockSync<T>(manifest: TeamRunManifest, fn: () => T, optio
|
|
|
798
891
|
});
|
|
799
892
|
}
|
|
800
893
|
|
|
894
|
+
/**
|
|
895
|
+
* ASYNC run-lock — U21 (2026-10-13) DUAL GUARD: the existing file lock
|
|
896
|
+
* (acquired FIRST — WAIT/backoff admission of D1 / RR-011 unchanged) is
|
|
897
|
+
* SUPPLEMENTED by a kernel process socket owned for the holder's process
|
|
898
|
+
* lifetime (see `acquireRunLockGuarded`). The supplement guards against
|
|
899
|
+
* dual-process entry where O_EXCL file locks are weak (NFS etc.); it is
|
|
900
|
+
* toggled by PI_CREW_RUN_LOCK_SOCKET (default ON, `'0'` = pre-U21 behaviour
|
|
901
|
+
* bit-for-bit). All six file-lock semantics (re-entrance, token release,
|
|
902
|
+
* live-in-process WAIT, stale-steal + WARN, EPERM, ENOENT) are unchanged and
|
|
903
|
+
* live in the file tier; the re-entrance bypass below returns BEFORE any
|
|
904
|
+
* socket acquisition, so nested same-context calls never self-deadlock on
|
|
905
|
+
* the kernel socket. Release order: socket FIRST, file token LAST.
|
|
906
|
+
*/
|
|
801
907
|
export async function withRunLock<T>(manifest: TeamRunManifest, fn: () => Promise<T>, options: RunLockOptions = {}): Promise<T> {
|
|
802
908
|
const filePath = lockPath(manifest);
|
|
803
909
|
const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
|
|
@@ -806,7 +912,7 @@ export async function withRunLock<T>(manifest: TeamRunManifest, fn: () => Promis
|
|
|
806
912
|
return await fn();
|
|
807
913
|
}
|
|
808
914
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
809
|
-
const token = await
|
|
915
|
+
const { token, socket } = await acquireRunLockGuarded(filePath, staleMs);
|
|
810
916
|
// RR-011 (F02): register the hold IMMEDIATELY (no await between the acquire
|
|
811
917
|
// returning and this add — otherwise a concurrent contender could observe the
|
|
812
918
|
// fresh on-disk token as a stealable corpse). Deleted in the finally below.
|
|
@@ -824,7 +930,17 @@ export async function withRunLock<T>(manifest: TeamRunManifest, fn: () => Promis
|
|
|
824
930
|
// RR-011 (F02): see withRunLockSync above — unregister the live token
|
|
825
931
|
// BEFORE the token-guarded releaseOwnLock (runLockHeldTokens invariant).
|
|
826
932
|
runLockHeldTokens.delete(token);
|
|
827
|
-
|
|
933
|
+
// U21: release the socket BEFORE the file token — a contender whose
|
|
934
|
+
// file-acquire already succeeded on the weak-file tier then sees the
|
|
935
|
+
// socket free only after BOTH guards of the previous holder are gone;
|
|
936
|
+
// the file stays held (by us) until the socket is fully released.
|
|
937
|
+
// FIX (review round): if socket.release() rejects, releaseOwnLock must
|
|
938
|
+
// still run — otherwise our own file token leaks on disk as a live lock.
|
|
939
|
+
try {
|
|
940
|
+
if (socket) await socket.release();
|
|
941
|
+
} finally {
|
|
942
|
+
releaseOwnLock(filePath, token);
|
|
943
|
+
}
|
|
828
944
|
}
|
|
829
945
|
});
|
|
830
946
|
}
|