pi-crew 0.11.5 → 0.11.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +165 -0
- package/README.md +1 -1
- package/dist/index.mjs +503 -493
- package/docs/dynamic-workflows.md +37 -0
- package/package.json +13 -9
- package/scripts/analyze-run.mjs +95 -11
- package/skills/model-routing-context/SKILL.md +4 -3
- package/skills/real-test-pi-crew/SKILL.md +9 -6
- package/src/config/config-validation.ts +28 -0
- package/src/config/env-vars.ts +29 -0
- package/src/config/types.ts +47 -0
- package/src/extension/crew-vibes/index.ts +29 -10
- package/src/extension/pi-api.ts +1 -0
- package/src/extension/registration/lifecycle-handlers.ts +65 -3
- package/src/extension/team-tool/respond.ts +54 -0
- package/src/extension/team-tool/status.ts +10 -0
- package/src/extension/team-tool.ts +393 -3
- package/src/extension/webhook-notify.ts +23 -1
- package/src/observability/exporters/otlp-exporter.ts +93 -11
- package/src/prompt/prompt-runtime.ts +46 -8
- package/src/runtime/agent-control.ts +1 -15
- package/src/runtime/background-runner.ts +3 -0
- package/src/runtime/broker/crew-broker.ts +12 -103
- package/src/runtime/broker/protocol/request-parsers.ts +9 -1
- package/src/runtime/broker/protocol/wait-resolve.ts +168 -0
- package/src/runtime/child-pi/child-pi-spawn.ts +12 -4
- package/src/runtime/child-pi/child-pi.ts +93 -4
- package/src/runtime/child-pi/session-recovery.ts +334 -0
- package/src/runtime/dispatch-batch.ts +3 -1
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +111 -12
- package/src/runtime/goal-workflow/dynamic-workflow-runner.ts +19 -0
- package/src/runtime/live-session/live-agent-manager.ts +35 -0
- package/src/runtime/live-session/live-session-runtime.ts +34 -5
- package/src/runtime/mcp-proxy.ts +110 -7
- package/src/runtime/model/model-scope.ts +141 -21
- package/src/runtime/model/pi-args.ts +115 -11
- package/src/runtime/model/provider-extensions.ts +67 -1
- package/src/runtime/peer-dep.ts +24 -0
- package/src/runtime/rpc/README.md +100 -0
- package/src/runtime/rpc/frame-client.ts +328 -0
- package/src/runtime/rpc/rpc-worker.ts +151 -0
- package/src/runtime/rpc/ui-request-policy.ts +112 -0
- package/src/runtime/run-worker.ts +14 -0
- package/src/runtime/scheduling/run-coalesced-task-group.ts +89 -6
- package/src/runtime/stale-reconciler.ts +47 -0
- package/src/runtime/surface/surface-spawn.ts +8 -1
- package/src/runtime/task-output-context.ts +191 -32
- package/src/runtime/task-packet.ts +3 -0
- package/src/runtime/task-runner/child-executor.ts +170 -6
- package/src/runtime/task-runner/output-splitter.ts +48 -5
- package/src/runtime/task-runner/post-execution.ts +5 -5
- package/src/runtime/task-runner/pre-execution.ts +32 -4
- package/src/runtime/task-runner/prompt-builder.ts +10 -2
- package/src/runtime/task-runner/retrieval-orchestrator.ts +28 -2
- package/src/schema/config-schema.ts +25 -0
- package/src/schema/team-tool-schema.ts +35 -0
- package/src/state/contracts.ts +16 -0
- package/src/state/crew-init.ts +114 -0
- package/src/state/stores/active-run-registry.ts +13 -6
- package/src/state/types.ts +0 -8
- package/src/ui/goal-flag.ts +44 -0
- package/src/ui/live-run-sidebar.ts +5 -1
- package/src/ui/pi-ui-compat.ts +37 -11
- package/src/ui/run-dashboard.ts +2 -1
- package/src/ui/widget/index.ts +0 -1
- package/src/ui/widget/widget-model.ts +6 -2
- package/src/utils/ip-literal.ts +145 -0
- package/src/workflows/workflow-config.ts +6 -0
|
@@ -27,6 +27,16 @@ import { ChildPiLineObserver } from "./child-pi-streams.ts";
|
|
|
27
27
|
// Phase 2.3: the six timer constructs moved to child-pi-timers.ts (pure motion).
|
|
28
28
|
import { createChildPiTimers } from "./child-pi-timers.ts";
|
|
29
29
|
import { runMockChildPi } from "./mock-fixtures.ts";
|
|
30
|
+
// W2 (session-file recovery): deterministic worker session flags + crash-path
|
|
31
|
+
// tail-replay of the session JSONL (see session-recovery.ts header).
|
|
32
|
+
import {
|
|
33
|
+
appendWorkerSessionArgs,
|
|
34
|
+
deriveSessionPaths,
|
|
35
|
+
recoverLastAssistantFromSession,
|
|
36
|
+
resolveSessionRecoveryEnabled,
|
|
37
|
+
type SessionRecoveryInfo,
|
|
38
|
+
shouldAttemptSessionRecovery,
|
|
39
|
+
} from "./session-recovery.ts";
|
|
30
40
|
|
|
31
41
|
// ── Re-exports from child-pi-kill.ts (H-7 decomposition step 2) ──
|
|
32
42
|
// killProcessTree is internal (not previously exported) — keep that invariant.
|
|
@@ -185,8 +195,18 @@ export interface ChildPiRunInput {
|
|
|
185
195
|
inheritContext?: boolean;
|
|
186
196
|
/** Pass to pi to mark certain commands as context-excluded. Default: false */
|
|
187
197
|
excludeContextBash?: boolean;
|
|
188
|
-
/** pi session ID for session naming (aligns with pi-crew run ID)
|
|
198
|
+
/** pi session ID for session naming (aligns with pi-crew run ID). W2: now
|
|
199
|
+
* WIRED — flows to `--session-id` (deterministic worker session files for
|
|
200
|
+
* crash-path tail-recovery; see session-recovery.ts). */
|
|
189
201
|
sessionId?: string;
|
|
202
|
+
/** W2 (session-file recovery): explicit session directory override. When
|
|
203
|
+
* absent, derived per-worker as `<artifactsRoot>/sessions/<taskId>/` so
|
|
204
|
+
* session JSONL is cleaned up together with run artifacts. */
|
|
205
|
+
sessionDir?: string;
|
|
206
|
+
/** W2 (session-file recovery): explicit on/off (runtime.sessionRecovery
|
|
207
|
+
* config). Env PI_CREW_SESSION_RECOVERY overrides in either direction;
|
|
208
|
+
* default ON. Only consulted on crash-ish settles (exitCode null / killed). */
|
|
209
|
+
sessionRecovery?: boolean;
|
|
190
210
|
/** Path to steering JSONL file for real-time steer injection. */
|
|
191
211
|
steeringFile?: string;
|
|
192
212
|
/** Run ID for cleanup tracking */
|
|
@@ -197,6 +217,15 @@ export interface ChildPiRunInput {
|
|
|
197
217
|
role?: string;
|
|
198
218
|
/** Team-role thinking override (takes precedence over agent.thinking). */
|
|
199
219
|
thinkingOverride?: string;
|
|
220
|
+
/** R3-1: run-static worker header for the --append-system-prompt channel
|
|
221
|
+
* (renderTaskPrompt().stablePrefix). Threaded by child-executor so the
|
|
222
|
+
* Protocol/mailbox/workspace/runtime-context block survives compaction
|
|
223
|
+
* instead of living in the summarizable user-message span. */
|
|
224
|
+
systemPromptAppend?: string;
|
|
225
|
+
/** R3-19/D5 (hermetic worker spawns): thread runtime.hermeticWorkers to
|
|
226
|
+
* buildPiWorkerArgs (--no-extensions). Env PI_CREW_HERMETIC_WORKERS
|
|
227
|
+
* overrides either way; default TRUE. */
|
|
228
|
+
hermeticWorkers?: boolean;
|
|
200
229
|
/** Root directory for artifacts (used to validate transcriptPath). */
|
|
201
230
|
artifactsRoot?: string;
|
|
202
231
|
/** I5: run events JSONL path — threaded to the worker so its scratchpad
|
|
@@ -291,6 +320,13 @@ export interface ChildPiRunResult {
|
|
|
291
320
|
* text) still produce a non-empty result. Consumers should prefer rawFinalText
|
|
292
321
|
* first — this is a last-resort fallback. */
|
|
293
322
|
intermediateFindings?: string;
|
|
323
|
+
/** W2 (session-file recovery): present when the worker died WITHOUT a
|
|
324
|
+
* final assistant event (exitCode null / killed) and a COMPLETE assistant
|
|
325
|
+
* record was tail-recovered from the worker's session JSONL (deterministic
|
|
326
|
+
* `--session-id`/`--session-dir` under the run artifacts root). Consumers
|
|
327
|
+
* treat it as a crash-path supplement to rawFinalText — never a replacement
|
|
328
|
+
* for clean-path output, and manifest polling is untouched. */
|
|
329
|
+
recoveredFromSession?: SessionRecoveryInfo;
|
|
294
330
|
/**
|
|
295
331
|
* MuxSurface A1: present ONLY when the worker booted inside a multiplexer
|
|
296
332
|
* pane instead of a stdio pipe (spec §13.1). Downstream (T9 EventLogTailSource,
|
|
@@ -371,6 +407,17 @@ function safeLoadSurfaceConfig(cwd: string): PiTeamsConfig {
|
|
|
371
407
|
}
|
|
372
408
|
}
|
|
373
409
|
|
|
410
|
+
/**
|
|
411
|
+
* D1/D5: flags that apply to HEADLESS worker spawns only. The surface TUI
|
|
412
|
+
* branch strips them before booting a pane — a pane hosts a full interactive
|
|
413
|
+
* pi session where the interactive trust prompt must stay available and the
|
|
414
|
+
* ambient extension stack is part of the human-visible session (task mandate:
|
|
415
|
+
* worker spawn CHỈ, KHÔNG đụng surface TUI spawn).
|
|
416
|
+
*/
|
|
417
|
+
function stripHeadlessOnlyFlags(args: string[]): string[] {
|
|
418
|
+
return args.filter((arg) => arg !== "--no-approve" && arg !== "--no-extensions");
|
|
419
|
+
}
|
|
420
|
+
|
|
374
421
|
/**
|
|
375
422
|
* MuxSurface A1 spawn branch (spec §13.1). Returns a ChildPiRunResult when the
|
|
376
423
|
* worker was booted INSIDE a multiplexer pane (no stdio process is spawned and
|
|
@@ -413,7 +460,7 @@ async function trySurfaceBranch(
|
|
|
413
460
|
livePaneCount: degradeController ? degradeController.livePaneCount() : (surfaceOpts?.livePaneCount ?? 0),
|
|
414
461
|
taskId,
|
|
415
462
|
cwd: input.cwd,
|
|
416
|
-
piArgs: builtArgs,
|
|
463
|
+
piArgs: stripHeadlessOnlyFlags(builtArgs),
|
|
417
464
|
stateRoot: input.eventsPath ? path.dirname(input.eventsPath) : "",
|
|
418
465
|
baseDir: surfaceOpts?.baseDir,
|
|
419
466
|
deps: preResolved ? { provider: preResolved } : {},
|
|
@@ -684,6 +731,24 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
|
|
|
684
731
|
const spawnPrep = prepareSpawnContext(brokerSpawn ? { ...input, brokerSpawn } : input, effectiveTask, depthEnv);
|
|
685
732
|
if (spawnPrep.kind === "aborted") return spawnPrep.result;
|
|
686
733
|
const { spawnSpec, mergedEnv, tempDir, builtEnv, builtArgs } = spawnPrep.ctx;
|
|
734
|
+
// W2 (session-file recovery): resolve the worker's deterministic session
|
|
735
|
+
// identity (per-worker dir under the run artifacts root) and arm it on BOTH
|
|
736
|
+
// argv views — the headless spawnSpec.args ([script, ...builtArgs]) and the
|
|
737
|
+
// raw builtArgs (also consumed by the surface branch). prepareSpawnContext
|
|
738
|
+
// does not forward session identity to the arg builder (ownership boundary
|
|
739
|
+
// with child-pi-spawn.ts — see the lane report), so this is the in-zone
|
|
740
|
+
// activation point; appendWorkerSessionArgs is idempotent, making the move
|
|
741
|
+
// to builder-side forwarding later a zero-diff no-op.
|
|
742
|
+
const workerSession = deriveSessionPaths(input);
|
|
743
|
+
const sessionRecoveryEnabled = resolveSessionRecoveryEnabled(input.sessionRecovery);
|
|
744
|
+
if (workerSession) {
|
|
745
|
+
try {
|
|
746
|
+
fs.mkdirSync(workerSession.sessionDir, { recursive: true });
|
|
747
|
+
} catch {
|
|
748
|
+
// Unwritable target → spawn proceeds; recovery simply finds no files.
|
|
749
|
+
}
|
|
750
|
+
appendWorkerSessionArgs(spawnSpec.args, builtArgs, workerSession);
|
|
751
|
+
}
|
|
687
752
|
// MuxSurface A1 (spec §13.1): attempt booting the worker in a mux pane BEFORE
|
|
688
753
|
// spawning a stdio pipe. Returns a finished result in surface mode (pane exit
|
|
689
754
|
// awaited) or null → fall through to the classic headless spawn below.
|
|
@@ -869,6 +934,22 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
|
|
|
869
934
|
clearPostExitGuard();
|
|
870
935
|
};
|
|
871
936
|
|
|
937
|
+
// W2 (session-file recovery): crash-path augment — when the worker died
|
|
938
|
+
// WITHOUT a final assistant event (exitCode null / killed), tail-replay
|
|
939
|
+
// its session JSONL and surface the last COMPLETE assistant record on the
|
|
940
|
+
// resolved result. Best-effort and deadline-bounded (session-recovery.ts)
|
|
941
|
+
// so settle can never hang on pathological IO. This AUGMENTS the result —
|
|
942
|
+
// manifest polling (heartbeat-watcher / detached-run-results) is untouched.
|
|
943
|
+
const recoverForSettle = async (settleResult: ChildPiRunResult): Promise<SessionRecoveryInfo | undefined> => {
|
|
944
|
+
if (!workerSession || !sessionRecoveryEnabled) return undefined;
|
|
945
|
+
if (!shouldAttemptSessionRecovery(settleResult, hardKilled)) return undefined;
|
|
946
|
+
try {
|
|
947
|
+
return (await recoverLastAssistantFromSession(workerSession.sessionDir, workerSession.sessionId)) ?? undefined;
|
|
948
|
+
} catch {
|
|
949
|
+
return undefined;
|
|
950
|
+
}
|
|
951
|
+
};
|
|
952
|
+
|
|
872
953
|
const settle = (result: ChildPiRunResult): Promise<void> => {
|
|
873
954
|
if (settled) return Promise.resolve();
|
|
874
955
|
settled = true;
|
|
@@ -885,7 +966,7 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
|
|
|
885
966
|
// durable transcript on disk.
|
|
886
967
|
return lineObserver
|
|
887
968
|
.flush()
|
|
888
|
-
.then(() => {
|
|
969
|
+
.then(async () => {
|
|
889
970
|
input.signal?.removeEventListener("abort", abort);
|
|
890
971
|
input.signal?.removeEventListener("abort", onParentAbort);
|
|
891
972
|
try {
|
|
@@ -893,12 +974,16 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
|
|
|
893
974
|
} catch (error) {
|
|
894
975
|
cleanupErrors.push(error instanceof Error ? error.message : String(error));
|
|
895
976
|
}
|
|
977
|
+
// W2: crash-path session tail-recovery (await BEFORE resolve so the
|
|
978
|
+
// recovered record rides the same settled result object).
|
|
979
|
+
const recoveredFromSession = await recoverForSettle(result);
|
|
896
980
|
// Catch all errors from settle to prevent unhandled rejection from propagating
|
|
897
981
|
try {
|
|
898
982
|
resolve({
|
|
899
983
|
...result,
|
|
900
984
|
rawFinalText: lineObserver.getRawFinalText(),
|
|
901
985
|
intermediateFindings: lineObserver.getIntermediateFindings(),
|
|
986
|
+
...(recoveredFromSession ? { recoveredFromSession } : {}),
|
|
902
987
|
exitStatus: result.exitStatus ?? {
|
|
903
988
|
exitCode: result.exitCode,
|
|
904
989
|
cancelled: abortRequested,
|
|
@@ -926,7 +1011,7 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
|
|
|
926
1011
|
);
|
|
927
1012
|
}
|
|
928
1013
|
})
|
|
929
|
-
.catch((flushError) => {
|
|
1014
|
+
.catch(async (flushError) => {
|
|
930
1015
|
// Drain failed — log and still resolve so runChildPi doesn't hang.
|
|
931
1016
|
logInternalError(
|
|
932
1017
|
"child-pi.settle-flush-failed",
|
|
@@ -940,11 +1025,15 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
|
|
|
940
1025
|
} catch (error) {
|
|
941
1026
|
cleanupErrors.push(error instanceof Error ? error.message : String(error));
|
|
942
1027
|
}
|
|
1028
|
+
// W2: crash-path session tail-recovery — same augment on the
|
|
1029
|
+
// drain-failed path (a torn flush is itself crash-adjacent).
|
|
1030
|
+
const recoveredFromSession = await recoverForSettle(result);
|
|
943
1031
|
try {
|
|
944
1032
|
resolve({
|
|
945
1033
|
...result,
|
|
946
1034
|
rawFinalText: lineObserver.getRawFinalText(),
|
|
947
1035
|
intermediateFindings: lineObserver.getIntermediateFindings(),
|
|
1036
|
+
...(recoveredFromSession ? { recoveredFromSession } : {}),
|
|
948
1037
|
exitStatus: result.exitStatus ?? {
|
|
949
1038
|
exitCode: result.exitCode,
|
|
950
1039
|
cancelled: abortRequested,
|
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* W2 (Pi 1.0.0 adoption — session-file recovery).
|
|
3
|
+
*
|
|
4
|
+
* Workers run with a DETERMINISTIC pi session identity: `--session-id` +
|
|
5
|
+
* `--session-dir` pointing under the run's artifacts root
|
|
6
|
+
* (`<artifactsRoot>/sessions/<safeTaskId>/`), so session JSONL lives and dies
|
|
7
|
+
* with run artifacts (auto-prune) instead of `~/.pi`. When a worker dies
|
|
8
|
+
* WITHOUT a final assistant event (exitCode null / killed — SIGKILL between
|
|
9
|
+
* turns), the session file still contains every COMPLETED turn (the SDK
|
|
10
|
+
* persists records at message_end; a kill can only tear the line being
|
|
11
|
+
* written). This module tail-replays that file and surfaces the last complete
|
|
12
|
+
* assistant record as `recoveredFromSession` on the run result.
|
|
13
|
+
*
|
|
14
|
+
* Contract (per verified packet):
|
|
15
|
+
* - Entry parsing is DELEGATED to the SDK (`parseSessionEntries` /
|
|
16
|
+
* `migrateSessionEntries`) — no hand-rolled parser. The only local IO
|
|
17
|
+
* defense is dropping a torn trailing fragment line (no final `\n`).
|
|
18
|
+
* - A trailing user/toolResult record with no assistant after it means the
|
|
19
|
+
* worker died mid-turn → the last COMPLETE assistant BEFORE it is recovered.
|
|
20
|
+
* - Recovery AUGMENTS the settle-time result; manifest polling is untouched.
|
|
21
|
+
* - Default ON; `PI_CREW_SESSION_RECOVERY` env overrides; explicit
|
|
22
|
+
* `sessionRecovery: false` (runtime config) is the opt-out below env.
|
|
23
|
+
*
|
|
24
|
+
* SDK import is a LAZY dynamic import (first value-import of the SDK in
|
|
25
|
+
* src/runtime): recovery only runs on crash paths, so module init must never
|
|
26
|
+
* pay the SDK load cost.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import * as fs from "node:fs";
|
|
30
|
+
import * as path from "node:path";
|
|
31
|
+
import { getCrewEnvBool } from "../../config/env-vars.ts";
|
|
32
|
+
import { isSafePathId } from "../../utils/safe-paths.ts";
|
|
33
|
+
|
|
34
|
+
/** Recovered last-complete-assistant payload attached to the run result. */
|
|
35
|
+
export interface SessionRecoveryInfo {
|
|
36
|
+
/** The session id the worker ran with (`--session-id`). */
|
|
37
|
+
sessionId: string;
|
|
38
|
+
/** Absolute path of the session JSONL the text was recovered from. */
|
|
39
|
+
sessionFile: string;
|
|
40
|
+
/** Text content of the last complete assistant message (text parts joined). */
|
|
41
|
+
text: string;
|
|
42
|
+
/** Timestamp of the recovered session entry (ISO). */
|
|
43
|
+
timestamp: string;
|
|
44
|
+
/** Stop reason of the recovered assistant message (e.g. "stop", "toolUse"). */
|
|
45
|
+
stopReason?: string;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Per-worker session identity resolved from the run input. */
|
|
49
|
+
export interface WorkerSessionContext {
|
|
50
|
+
sessionId: string;
|
|
51
|
+
sessionDir: string;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Identity fields read off `ChildPiRunInput` (kept structural for tests). */
|
|
55
|
+
export interface SessionIdentityInput {
|
|
56
|
+
sessionId?: string;
|
|
57
|
+
sessionDir?: string;
|
|
58
|
+
agentId?: string;
|
|
59
|
+
artifactsRoot?: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Sanitize a task/session id the same way the state store does
|
|
64
|
+
* (crew-agent-records.ts safeAgentTaskId — module-private there, mirrored
|
|
65
|
+
* here): delegate ids "parent:child" collapse to the last segment, then the
|
|
66
|
+
* result must be path-safe. Returns null when unsafe — callers fail closed
|
|
67
|
+
* (no session flags, no recovery) rather than spawning with a broken id.
|
|
68
|
+
* [A-Za-z0-9_-]+ is a strict subset of the SDK's session-id charset.
|
|
69
|
+
*/
|
|
70
|
+
function sanitizeSessionIdentity(rawId: string): string | null {
|
|
71
|
+
const leaf = rawId.includes(":") ? (rawId.split(":").pop() ?? "") : rawId;
|
|
72
|
+
return isSafePathId(leaf) ? leaf : null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Derive the worker's deterministic session identity.
|
|
77
|
+
*
|
|
78
|
+
* Layout (packet §R2.2 + fact pack §3): session files land in a PER-WORKER
|
|
79
|
+
* directory `<artifactsRoot>/sessions/<safeTaskId>/`; the session id rides
|
|
80
|
+
* `--session-id` (explicit `sessionId` — e.g. manifest.sessionId already
|
|
81
|
+
* threaded by child-executor — preferred, else the safe task id). Same id
|
|
82
|
+
* across retries appends to one file = cross-attempt resume for free.
|
|
83
|
+
*
|
|
84
|
+
* Returns null (→ no session flags, recovery silently impossible) when the
|
|
85
|
+
* artifacts root is unknown or no sanitizable identity exists.
|
|
86
|
+
*/
|
|
87
|
+
export function deriveSessionPaths(input: SessionIdentityInput): WorkerSessionContext | null {
|
|
88
|
+
const safeAgent = input.agentId ? sanitizeSessionIdentity(input.agentId) : null;
|
|
89
|
+
const safeSession = input.sessionId ? sanitizeSessionIdentity(input.sessionId) : null;
|
|
90
|
+
// File id: explicit session id (manifest.sessionId) preferred, else task id.
|
|
91
|
+
const sessionId = safeSession ?? safeAgent;
|
|
92
|
+
// Dir leaf: PER-WORKER (safe task id) so same-run siblings never share a
|
|
93
|
+
// session file; fall back to the session id for agent-id-less custom spawns.
|
|
94
|
+
const dirLeaf = safeAgent ?? safeSession;
|
|
95
|
+
if (!sessionId || !dirLeaf) return null;
|
|
96
|
+
if (input.sessionDir) return { sessionId, sessionDir: input.sessionDir };
|
|
97
|
+
if (!input.artifactsRoot) return null;
|
|
98
|
+
return { sessionId, sessionDir: path.join(input.artifactsRoot, "sessions", dirLeaf) };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Append the session flags to the two argv views produced by
|
|
103
|
+
* `prepareSpawnContext`: `builtArgs` (raw worker argv — also consumed by the
|
|
104
|
+
* surface branch) and the headless `spawnArgs` ([script, ...builtArgs]).
|
|
105
|
+
* Flags are inserted right after the `--mode json -p` cluster so they never
|
|
106
|
+
* trail the task-file positional. Idempotent: if the flags are already
|
|
107
|
+
* present (builder forwarding, future wiring), this is a no-op.
|
|
108
|
+
*/
|
|
109
|
+
export function appendWorkerSessionArgs(spawnArgs: string[], builtArgs: string[], ctx: WorkerSessionContext): boolean {
|
|
110
|
+
// PER-FLAG idempotency (battery fix 2026-10-04): the builder may forward a
|
|
111
|
+
// PARTIAL identity (e.g. manifest sessionId without sessionDir — the
|
|
112
|
+
// child-executor runWorker call passes sessionId only). The old
|
|
113
|
+
// both-or-nothing guard ("if --session-id present, skip everything") let
|
|
114
|
+
// that partial arming self-disarm the full activation: --session-id went
|
|
115
|
+
// out, --session-dir never did, and worker session files silently landed
|
|
116
|
+
// in the cwd-keyed default dir instead of the per-worker artifacts dir —
|
|
117
|
+
// breaking crash recovery in the live path. Fill in each missing flag.
|
|
118
|
+
let changed = false;
|
|
119
|
+
const missing: string[] = [];
|
|
120
|
+
if (!builtArgs.includes("--session-id")) missing.push("--session-id", ctx.sessionId);
|
|
121
|
+
if (!builtArgs.includes("--session-dir")) missing.push("--session-dir", ctx.sessionDir);
|
|
122
|
+
if (missing.length > 0) {
|
|
123
|
+
insertAfterHeadlessCluster(builtArgs, missing);
|
|
124
|
+
changed = true;
|
|
125
|
+
}
|
|
126
|
+
// Mirror onto the headless spawnArgs view (independent array) with the
|
|
127
|
+
// same per-flag check.
|
|
128
|
+
const missingSpawn: string[] = [];
|
|
129
|
+
if (!spawnArgs.includes("--session-id")) missingSpawn.push("--session-id", ctx.sessionId);
|
|
130
|
+
if (!spawnArgs.includes("--session-dir")) missingSpawn.push("--session-dir", ctx.sessionDir);
|
|
131
|
+
if (missingSpawn.length > 0) {
|
|
132
|
+
insertAfterHeadlessCluster(spawnArgs, missingSpawn);
|
|
133
|
+
changed = true;
|
|
134
|
+
}
|
|
135
|
+
return changed;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function insertAfterHeadlessCluster(arr: string[], flags: string[]): void {
|
|
139
|
+
const idx = arr.indexOf("--mode");
|
|
140
|
+
if (idx !== -1 && arr[idx + 1] === "json" && arr[idx + 2] === "-p") {
|
|
141
|
+
arr.splice(idx + 3, 0, ...flags);
|
|
142
|
+
} else {
|
|
143
|
+
arr.push(...flags);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Recovery gate: explicit flag (runtime.sessionRecovery → ChildPiRunInput)
|
|
149
|
+
* with env override. Default ON (W2 scope). The env var is the operator
|
|
150
|
+
* kill-switch and wins in EITHER direction.
|
|
151
|
+
*/
|
|
152
|
+
export function resolveSessionRecoveryEnabled(explicit?: boolean): boolean {
|
|
153
|
+
const env = getCrewEnvBool("PI_CREW_SESSION_RECOVERY");
|
|
154
|
+
if (env !== undefined) return env;
|
|
155
|
+
return explicit ?? true;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Crash-path predicate (packet: exitCode === null OR killed). Structural so
|
|
160
|
+
* both the pre-resolve result and a materialized WorkerExitStatus feed it.
|
|
161
|
+
*/
|
|
162
|
+
export function shouldAttemptSessionRecovery(
|
|
163
|
+
result: { exitCode: number | null; exitStatus?: { exitCode?: number | null; killed?: boolean } },
|
|
164
|
+
killed: boolean,
|
|
165
|
+
): boolean {
|
|
166
|
+
// NB: explicit `exitCode: null` in exitStatus IS the signal-death signal —
|
|
167
|
+
// `??` would collapse it onto the outer exitCode, so branch on presence.
|
|
168
|
+
const status = result.exitStatus;
|
|
169
|
+
const exitCode = status && status.exitCode !== undefined ? status.exitCode : result.exitCode;
|
|
170
|
+
const wasKilled = status && status.killed !== undefined ? status.killed : killed;
|
|
171
|
+
return exitCode === null || wasKilled === true;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Bounded crash-path work: never let a pathological read hang task settle. */
|
|
175
|
+
const RECOVERY_DEADLINE_MS = 5_000;
|
|
176
|
+
/** Scan at most this many candidate files (id-matched first, then newest). */
|
|
177
|
+
const RECOVERY_MAX_FILES = 3;
|
|
178
|
+
/** Skip absurdly large session files rather than blocking settle for them. */
|
|
179
|
+
const RECOVERY_MAX_FILE_BYTES = 64 * 1024 * 1024;
|
|
180
|
+
|
|
181
|
+
interface SessionParserModule {
|
|
182
|
+
parseSessionEntries(content: string): unknown[];
|
|
183
|
+
migrateSessionEntries(entries: unknown[]): void;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
let sessionParserCache: SessionParserModule | null | undefined;
|
|
187
|
+
|
|
188
|
+
async function loadSessionParser(): Promise<SessionParserModule | null> {
|
|
189
|
+
if (sessionParserCache !== undefined) return sessionParserCache;
|
|
190
|
+
try {
|
|
191
|
+
// First value-import of the SDK inside src/runtime — recovery is a
|
|
192
|
+
// crash-path concern; child-pi module init must never pay SDK load cost.
|
|
193
|
+
// LAZY: SDK session parsers (parseSessionEntries/migrateSessionEntries)
|
|
194
|
+
const mod = (await import("@earendil-works/pi-coding-agent")) as unknown as SessionParserModule;
|
|
195
|
+
sessionParserCache = typeof mod.parseSessionEntries === "function" && typeof mod.migrateSessionEntries === "function" ? mod : null;
|
|
196
|
+
} catch {
|
|
197
|
+
sessionParserCache = null;
|
|
198
|
+
}
|
|
199
|
+
return sessionParserCache;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Candidate session files, best-first: id-suffixed matches
|
|
204
|
+
* (`<ISO-ts>_<sessionId>.jsonl`) newest-mtime first, then any other .jsonl
|
|
205
|
+
* (fallback for id-less derivation). Missing/unreadable dir → [].
|
|
206
|
+
*/
|
|
207
|
+
function listSessionCandidateFiles(sessionDir: string, sessionId?: string): string[] {
|
|
208
|
+
let names: string[];
|
|
209
|
+
try {
|
|
210
|
+
names = fs.readdirSync(sessionDir);
|
|
211
|
+
} catch {
|
|
212
|
+
return [];
|
|
213
|
+
}
|
|
214
|
+
const jsonl = names.filter((n) => n.endsWith(".jsonl"));
|
|
215
|
+
const withMtime = (list: string[]): { name: string; mtimeMs: number }[] => {
|
|
216
|
+
const out: { name: string; mtimeMs: number }[] = [];
|
|
217
|
+
for (const name of list) {
|
|
218
|
+
try {
|
|
219
|
+
out.push({ name, mtimeMs: fs.statSync(path.join(sessionDir, name)).mtimeMs });
|
|
220
|
+
} catch {
|
|
221
|
+
/* raced away — ignore */
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return out.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
225
|
+
};
|
|
226
|
+
const matched = sessionId ? withMtime(jsonl.filter((n) => n.endsWith(`_${sessionId}.jsonl`))) : [];
|
|
227
|
+
const matchedSet = new Set(matched.map((x) => x.name));
|
|
228
|
+
const rest = withMtime(jsonl.filter((n) => !matchedSet.has(n)));
|
|
229
|
+
return [...matched, ...rest].map((x) => path.join(sessionDir, x.name));
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
interface AssistantMessageLike {
|
|
233
|
+
role?: unknown;
|
|
234
|
+
content?: unknown;
|
|
235
|
+
stopReason?: unknown;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Join the text parts of an assistant message's content array. */
|
|
239
|
+
function extractAssistantText(content: unknown): string {
|
|
240
|
+
if (!Array.isArray(content)) return typeof content === "string" ? content : "";
|
|
241
|
+
const parts: string[] = [];
|
|
242
|
+
for (const part of content) {
|
|
243
|
+
if (part && typeof part === "object" && (part as { type?: unknown }).type === "text") {
|
|
244
|
+
const text = (part as { text?: unknown }).text;
|
|
245
|
+
if (typeof text === "string" && text.length > 0) parts.push(text);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return parts.join("\n");
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
async function readLastCompleteAssistant(sessionFile: string, sessionId: string): Promise<SessionRecoveryInfo | null> {
|
|
252
|
+
let size: number;
|
|
253
|
+
try {
|
|
254
|
+
size = fs.statSync(sessionFile).size;
|
|
255
|
+
} catch {
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
if (size <= 0 || size > RECOVERY_MAX_FILE_BYTES) return null;
|
|
259
|
+
let content: string;
|
|
260
|
+
try {
|
|
261
|
+
content = fs.readFileSync(sessionFile, "utf8");
|
|
262
|
+
} catch {
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
// Defensive torn-tail: a SIGKILL can interrupt the final line mid-write
|
|
266
|
+
// (no trailing newline). parseSessionEntries already skips malformed
|
|
267
|
+
// lines, but drop the unterminated fragment explicitly so it can never be
|
|
268
|
+
// resurrected as a partial record by migration.
|
|
269
|
+
if (content.length > 0 && !content.endsWith("\n")) {
|
|
270
|
+
content = content.slice(0, content.lastIndexOf("\n") + 1);
|
|
271
|
+
}
|
|
272
|
+
const parser = await loadSessionParser();
|
|
273
|
+
if (!parser) return null;
|
|
274
|
+
let entries: unknown[];
|
|
275
|
+
try {
|
|
276
|
+
entries = parser.parseSessionEntries(content);
|
|
277
|
+
parser.migrateSessionEntries(entries);
|
|
278
|
+
} catch {
|
|
279
|
+
return null;
|
|
280
|
+
}
|
|
281
|
+
// Backward scan for the last COMPLETE assistant record: role assistant,
|
|
282
|
+
// stopReason present and not "pending" (dangling), and it must carry
|
|
283
|
+
// replayable text — tool-call-only turns recover nothing useful, keep
|
|
284
|
+
// scanning for the last turn that actually spoke.
|
|
285
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
286
|
+
const entry = entries[i];
|
|
287
|
+
if (!entry || typeof entry !== "object" || (entry as { type?: unknown }).type !== "message") continue;
|
|
288
|
+
const message = (entry as { message?: unknown }).message;
|
|
289
|
+
if (!message || typeof message !== "object") continue;
|
|
290
|
+
const assistant = message as AssistantMessageLike;
|
|
291
|
+
if (assistant.role !== "assistant") continue;
|
|
292
|
+
const stopReason = assistant.stopReason;
|
|
293
|
+
if (typeof stopReason !== "string" || stopReason === "pending") continue;
|
|
294
|
+
const text = extractAssistantText(assistant.content);
|
|
295
|
+
if (!text) continue;
|
|
296
|
+
return {
|
|
297
|
+
sessionId,
|
|
298
|
+
sessionFile,
|
|
299
|
+
text,
|
|
300
|
+
timestamp:
|
|
301
|
+
typeof (entry as { timestamp?: unknown }).timestamp === "string"
|
|
302
|
+
? ((entry as { timestamp?: unknown }).timestamp as string)
|
|
303
|
+
: "",
|
|
304
|
+
stopReason,
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
return null;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
async function doRecoverLastAssistant(sessionDir: string, sessionId?: string): Promise<SessionRecoveryInfo | null> {
|
|
311
|
+
for (const file of listSessionCandidateFiles(sessionDir, sessionId).slice(0, RECOVERY_MAX_FILES)) {
|
|
312
|
+
const info = await readLastCompleteAssistant(file, sessionId ?? path.basename(file).replace(/\.jsonl$/, ""));
|
|
313
|
+
if (info) return info;
|
|
314
|
+
}
|
|
315
|
+
return null;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Tail-replay a worker's session directory: recover the last COMPLETE
|
|
320
|
+
* assistant record from the best candidate session JSONL. Returns null when
|
|
321
|
+
* nothing recoverable exists (no dir, no files, no complete assistant turn,
|
|
322
|
+
* SDK unresolvable). Best-effort by contract — never throws, never hangs
|
|
323
|
+
* past RECOVERY_DEADLINE_MS.
|
|
324
|
+
*/
|
|
325
|
+
export async function recoverLastAssistantFromSession(sessionDir: string, sessionId?: string): Promise<SessionRecoveryInfo | null> {
|
|
326
|
+
const attempt = doRecoverLastAssistant(sessionDir, sessionId);
|
|
327
|
+
return await Promise.race([
|
|
328
|
+
attempt,
|
|
329
|
+
new Promise<SessionRecoveryInfo | null>((resolve) => {
|
|
330
|
+
const timer = setTimeout(() => resolve(null), RECOVERY_DEADLINE_MS);
|
|
331
|
+
timer.unref();
|
|
332
|
+
}),
|
|
333
|
+
]);
|
|
334
|
+
}
|
|
@@ -625,6 +625,8 @@ export async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBat
|
|
|
625
625
|
const { workflow, input, runtimeKind, runController } = ctx;
|
|
626
626
|
|
|
627
627
|
// 2.2 caller migration: batch progress is high-frequency informational (M7 wire).
|
|
628
|
+
// .catch REQUIRED — buffered-flush rejections reject queued promises (see
|
|
629
|
+
// child-executor.ts:530 note, CI 2026-10-03).
|
|
628
630
|
void appendEventBuffered(ctx.manifest.eventsPath, {
|
|
629
631
|
type: "task.progress",
|
|
630
632
|
runId: ctx.manifest.runId,
|
|
@@ -640,7 +642,7 @@ export async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBat
|
|
|
640
642
|
defaultConcurrency: concurrency.defaultConcurrency,
|
|
641
643
|
concurrencyReason: approvalPending ? `${concurrency.reason};plan-approval-read-only` : concurrency.reason,
|
|
642
644
|
},
|
|
643
|
-
});
|
|
645
|
+
}).catch((error) => logInternalError("dispatch-batch.ready-batch-progress", error, `runId=${ctx.manifest.runId}`));
|
|
644
646
|
// Execute before_task_start hooks for the batch — P1-10: run hooks in
|
|
645
647
|
// parallel (each may be a subprocess), then apply skip mutations in order.
|
|
646
648
|
const beforeTaskStartReports = await Promise.all(
|