pi-crew 0.11.21 → 0.11.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/index.mjs +537 -526
- package/package.json +2 -1
- package/src/errors.ts +14 -1
- package/src/extension/async-notifier.ts +42 -17
- package/src/extension/cross-extension-rpc.ts +42 -5
- package/src/extension/notification-router.ts +15 -2
- package/src/extension/registration/lifecycle-handlers.ts +36 -13
- package/src/extension/registration/runtime-cleanup.ts +17 -0
- package/src/extension/registration/wire-cross-extension.ts +26 -0
- package/src/extension/rpc-hmac.ts +53 -2
- package/src/extension/team-tool/cancel.ts +34 -21
- package/src/extension/team-tool/dispatch/index.ts +6 -5
- package/src/extension/team-tool/dispatch/status.ts +42 -2
- package/src/extension/team-tool.ts +24 -2
- package/src/prompt/inbox-poll.ts +77 -25
- package/src/prompt/message-tool.ts +62 -4
- package/src/prompt/prompt-runtime.ts +82 -32
- package/src/runtime/broker/crew-broker-client.ts +74 -16
- package/src/runtime/broker/crew-broker.ts +111 -77
- package/src/runtime/broker/protocol/msg-send-targets.ts +74 -0
- package/src/runtime/broker/wait-status-cache.ts +14 -5
- package/src/runtime/child-pi/child-pi-spawn.ts +47 -9
- package/src/runtime/child-pi/child-pi.ts +20 -2
- package/src/runtime/child-pi/mock-fixtures.ts +23 -0
- package/src/runtime/compaction/compact-stages/truncation-stage.ts +45 -3
- package/src/runtime/compaction/important-line-classifier.ts +85 -11
- package/src/runtime/compaction/tool-output-pruner.ts +35 -11
- package/src/runtime/crew-agent-records.ts +7 -1
- package/src/runtime/delivery-coordinator.ts +30 -0
- package/src/runtime/dwf-state-store.ts +41 -7
- package/src/runtime/finalize-run.ts +13 -2
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +192 -64
- package/src/runtime/goal-workflow/goal-loop-runner.ts +130 -3
- package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
- package/src/runtime/live-session/embedded-tier.ts +3 -1
- package/src/runtime/live-session/live-session-runtime.ts +54 -1
- package/src/runtime/model/model-fallback.ts +23 -1
- package/src/runtime/output/pi-json-output.ts +26 -1
- package/src/runtime/policy-engine.ts +18 -0
- package/src/runtime/process/zombie-scanner.ts +78 -14
- package/src/runtime/process-status.ts +6 -1
- package/src/runtime/recovery/crash-recovery.ts +170 -28
- package/src/runtime/rpc/rpc-worker.ts +44 -1
- package/src/runtime/scheduling/run-coalesced-task-group.ts +64 -3
- package/src/runtime/scheduling/scheduler.ts +74 -11
- package/src/runtime/scratchpad/engine.ts +26 -3
- package/src/runtime/settings-store.ts +22 -2
- package/src/runtime/surface/tmux-provider.ts +24 -3
- package/src/runtime/task-health.ts +16 -1
- package/src/runtime/task-output-context.ts +143 -71
- package/src/runtime/task-packet.ts +70 -17
- package/src/runtime/task-runner/child-executor.ts +27 -0
- package/src/runtime/task-runner/output-splitter.ts +79 -16
- package/src/runtime/task-runner/post-execution.ts +75 -15
- package/src/runtime/task-runner/state-helpers.ts +92 -2
- package/src/runtime/usage-tracker.ts +11 -0
- package/src/runtime/verification/green-contract.ts +17 -0
- package/src/runtime/verification/verification-gates.ts +24 -6
- package/src/runtime/workspace-lock.ts +53 -0
- package/src/schema/team-tool-schema.ts +10 -4
- package/src/state/coordination/mailbox.ts +133 -38
- package/src/state/decision-ledger.ts +56 -8
- package/src/state/event-log/event-log-rotation.ts +83 -5
- package/src/state/event-log/event-log.ts +181 -12
- package/src/state/stores/active-run-registry.ts +10 -3
- package/src/state/types.ts +9 -1
- package/src/ui/agents-jobs-browser.ts +22 -8
- package/src/ui/dashboard-panes/agents-pane.ts +5 -2
- package/src/ui/events-state-source.ts +67 -28
- package/src/ui/inline-panel/agent-pane.ts +13 -6
- package/src/ui/inline-panel/agent-transcript.ts +33 -21
- package/src/ui/inline-panel/agent-view-overlay.ts +18 -3
- package/src/ui/inline-panel/index.ts +21 -1
- package/src/ui/live-run-sidebar.ts +14 -3
- package/src/ui/overlays/mailbox-compose-overlay.ts +10 -4
- package/src/ui/powerbar-publisher.ts +6 -3
- package/src/ui/rail.ts +7 -1
- package/src/ui/render-scheduler.ts +53 -1
- package/src/ui/run-action-dispatcher.ts +68 -20
- package/src/ui/status-colors.ts +9 -0
- package/src/utils/env-filter.ts +8 -0
|
@@ -33,15 +33,16 @@ import type { TeamRunManifest } from "../../state/types.ts";
|
|
|
33
33
|
import type { TeamConfig } from "../../teams/team-config.ts";
|
|
34
34
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
35
35
|
import { safeAbort } from "../../utils/safe-abort.ts";
|
|
36
|
+
import { countTokens } from "../../utils/token-counter.ts";
|
|
36
37
|
import { cleanupAgentWorktreeAsync, prepareAgentWorktreeAsync } from "../../worktree/worktree-manager.ts";
|
|
37
|
-
import type {
|
|
38
|
-
import {
|
|
38
|
+
import type { ChildPiRunResult } from "../child-pi/child-pi.ts";
|
|
39
|
+
import { DWF_CHECKPOINT_VERSION, type DwfCheckpointState } from "../dwf-state-store.ts";
|
|
40
|
+
import { type ParsedPiJsonOutput, parsePiJsonOutput } from "../output/pi-json-output.ts";
|
|
39
41
|
import { extractStructuredResult } from "../output/result-extractor.ts";
|
|
40
42
|
import { executeWithRetry } from "../recovery/retry-executor.ts";
|
|
41
43
|
import { runWorker } from "../run-worker.ts";
|
|
42
44
|
import { mapConcurrent } from "../scheduling/parallel-utils.ts";
|
|
43
45
|
import { Semaphore } from "../scheduling/semaphore.ts";
|
|
44
|
-
import { estimateTokens } from "../task-runner/prompt-builder.ts";
|
|
45
46
|
import { renderPlanTemplate } from "./plan-templates.ts";
|
|
46
47
|
|
|
47
48
|
export interface AgentCallOpts {
|
|
@@ -132,6 +133,45 @@ export class DwfAgentCallCapError extends Error {
|
|
|
132
133
|
}
|
|
133
134
|
}
|
|
134
135
|
|
|
136
|
+
/** R6-10 (DR-03 round-6, 2026-10-16): typed bridge error thrown INSIDE ctx.retry()
|
|
137
|
+
* when ctx.agent() resolves ok:false. executeWithRetry retries THROWN errors
|
|
138
|
+
* only, so an ok:false result previously flowed straight out of ctx.retry —
|
|
139
|
+
* a 1-attempt "retry" for every failure class except DwfAgentCallCapError.
|
|
140
|
+
* Throwing this wrapper makes the failure retryable; ctx.retry catches it after
|
|
141
|
+
* exhaustion and returns the CARRIED AgentResult, so ctx.retry's public return
|
|
142
|
+
* type/shape is unchanged. DwfAgentCallCapError keeps its passthrough semantics
|
|
143
|
+
* (the cap must terminate the workflow, never be retried into silence). */
|
|
144
|
+
export class DwfRetryFailure extends Error {
|
|
145
|
+
/** The ok:false AgentResult the failed attempt produced. */
|
|
146
|
+
readonly result: AgentResult;
|
|
147
|
+
constructor(result: AgentResult) {
|
|
148
|
+
super(result.error ?? "dynamic workflow retry: agent call returned ok:false");
|
|
149
|
+
this.name = "DwfRetryFailure";
|
|
150
|
+
this.result = result;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** R6-7 (DR-03 round-6, 2026-10-16): DWF empty-settle gate WITH the sibling
|
|
155
|
+
* exemptions the task-runner's attemptErrorFor (child-executor.ts:428-431) has
|
|
156
|
+
* but ctx.agent()'s inline gate lacked. Returns the structured 429-style error
|
|
157
|
+
* string when the worker settled without a final answer AND the exemptions do
|
|
158
|
+
* NOT apply, else undefined (gate passes):
|
|
159
|
+
* - patches are real work: coder roles may legitimately end patch-only
|
|
160
|
+
* (empty final assistant message is fine when the diff already landed);
|
|
161
|
+
* - cancelled/aborted attempts are excluded so an abort never surfaces as an
|
|
162
|
+
* upstream-rate-limit failure (and never burns fallback/retry budget on it).
|
|
163
|
+
* Mirrors child-executor.ts exactly — extracted so the exemptions are unit-testable
|
|
164
|
+
* at this seam (the mock-fixture layer cannot synthesise patch/cancelled shapes). */
|
|
165
|
+
export function dwfSettledWithoutAnswerFailure(
|
|
166
|
+
childResult: Pick<ChildPiRunResult, "exitStatus">,
|
|
167
|
+
parsed: Pick<ParsedPiJsonOutput, "settledWithoutAnswer" | "patches">,
|
|
168
|
+
): string | undefined {
|
|
169
|
+
if (parsed.settledWithoutAnswer !== true) return undefined;
|
|
170
|
+
if ((parsed.patches?.length ?? 0) > 0) return undefined;
|
|
171
|
+
if (childResult.exitStatus?.cancelled) return undefined;
|
|
172
|
+
return "Model settled without a final answer (empty final assistant message; upstream rate-limit/429 exhaustion)";
|
|
173
|
+
}
|
|
174
|
+
|
|
135
175
|
export interface WorkflowCtx {
|
|
136
176
|
cwd: string;
|
|
137
177
|
runId: string;
|
|
@@ -313,6 +353,14 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
313
353
|
}
|
|
314
354
|
: { currentPhase: undefined, phases: [] };
|
|
315
355
|
let phaseCapWarned = false;
|
|
356
|
+
// R6-12 (DR-03 round-6): phases already recorded in the hydrated checkpoint.
|
|
357
|
+
// ctx.phase() does NOT re-emit phase_started/phase_completed for these on a
|
|
358
|
+
// resumed run — the script re-executes from the top and re-walks its phase
|
|
359
|
+
// history; without this guard every replayed phase title re-emitted its events
|
|
360
|
+
// (duplicated phase boundaries in events.jsonl). Empty on a fresh run, so the
|
|
361
|
+
// guard is resume-only: a fresh script may still legitimately re-enter a title
|
|
362
|
+
// later (e.g. phase("review") rounds) and get its events.
|
|
363
|
+
const hydratedPhases = new Set<string>(opts.resumedState?.phases ?? []);
|
|
316
364
|
// round-14 P1-2/P1-3/P1-5: closure-scoped runtime state shared by budget/log/args.
|
|
317
365
|
// Mirrors the pi-dynamic-workflows RuntimeState pattern (workflow.ts:state).
|
|
318
366
|
// round-18 P2-3: hydrate spent/logs from a resumed checkpoint (backward compatible when unset).
|
|
@@ -323,9 +371,10 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
323
371
|
};
|
|
324
372
|
// PERS-1: per-agent-call idempotency cache. Hydrated from resumedState so a DWF
|
|
325
373
|
// resume skips already-completed agent calls (avoids duplicating artifacts/mailbox/tokens).
|
|
326
|
-
// Uses Record (not Map) for JSON serialization safety.
|
|
327
|
-
|
|
328
|
-
|
|
374
|
+
// Uses Record (not Map) for JSON serialization safety. R6-3 (DR-03 round-6):
|
|
375
|
+
// values are the FULL AgentResult (text + usage + structured + artifactPath +
|
|
376
|
+
// actual durationMs) so replay does not drop structured output / artifact linkage.
|
|
377
|
+
const completedAgentCalls: Record<string, AgentResult> = opts.resumedState?.completedAgentCalls ?? {};
|
|
329
378
|
// round-14 P1-2: frozen budget surface. The closures read wfState.spent so the
|
|
330
379
|
// object stays live after Object.freeze(ctx). total is a snapshot primitive.
|
|
331
380
|
const budget = Object.freeze({
|
|
@@ -348,17 +397,19 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
348
397
|
let worktreePath: string | undefined;
|
|
349
398
|
let worktreeBranch: string | undefined;
|
|
350
399
|
// BDG-2 + G14 (SDD-3): content-based reserve estimate — replaces the flat
|
|
351
|
-
// ESTIMATE=4096.
|
|
352
|
-
// from
|
|
353
|
-
//
|
|
354
|
-
//
|
|
355
|
-
//
|
|
356
|
-
//
|
|
357
|
-
//
|
|
358
|
-
//
|
|
359
|
-
//
|
|
360
|
-
//
|
|
361
|
-
|
|
400
|
+
// ESTIMATE=4096. countTokens() (R5-7, DR-02 round-5: the in-tree
|
|
401
|
+
// CJK-aware estimator from utils/token-counter.ts — replaces the chars/4
|
|
402
|
+
// estimateTokens call, which under-counted CJK/Vietnamese-diacritic
|
|
403
|
+
// prompts ~2-3x and could let N concurrent calls blow past the budget)
|
|
404
|
+
// runs over the call's prompt + optional systemPrompt override, floored
|
|
405
|
+
// at 512 tokens for the fixed per-call overhead (schema/JSON directive,
|
|
406
|
+
// artifact headers) not present in the prompt string. Short calls stop
|
|
407
|
+
// over-reserving; very long prompts reserve proportionally so N
|
|
408
|
+
// concurrent calls cannot blow past the budget between reserve and the
|
|
409
|
+
// post-run adjust. Every BDG-2 site below (check/reserve/un-reserve/
|
|
410
|
+
// adjust/refund) uses THIS value, so refund always matches what was
|
|
411
|
+
// reserved.
|
|
412
|
+
const estimate = Math.max(512, countTokens(`${call.prompt ?? ""}${call.systemPrompt ?? ""}`));
|
|
362
413
|
let reserved = false;
|
|
363
414
|
try {
|
|
364
415
|
// SDD-3 W-C G13: agent-call cap — the structured bound for no-budget runs.
|
|
@@ -393,21 +444,20 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
393
444
|
// from the call arguments and check if this call was already completed
|
|
394
445
|
// (e.g. during a previous run before a crash). If so, return the cached
|
|
395
446
|
// result without re-spawning the agent.
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
447
|
+
// R6-3 (DR-03 round-6, HIGH): the key now covers ALL call-shaping fields
|
|
448
|
+
// (inputs/systemPrompt/disableTools/maxTurns/graceTurns/worktree/skill/…
|
|
449
|
+
// and the schema BY VALUE) — previously only {role,agent,prompt,model,
|
|
450
|
+
// schema:"yes"/"no"}, so two DIFFERENT calls sharing those 5 fields hit the
|
|
451
|
+
// same cache entry on EVERY call (e.g. same prompt with different inputs
|
|
452
|
+
// returned the first call's result — inputs provably alter the dispatched
|
|
453
|
+
// task via composeAgentTask). See computeCallId for the normalization.
|
|
454
|
+
const callId = computeCallId(call);
|
|
403
455
|
const cached = completedAgentCalls[callId];
|
|
404
456
|
if (cached) {
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
durationMs: 0,
|
|
410
|
-
};
|
|
457
|
+
// Full-result replay (R6-3): structured + artifactPath survive the
|
|
458
|
+
// round-trip through the checkpoint. ok is forced true — only successful
|
|
459
|
+
// results are cached, and legacy (pre-R6-3) hydrated records lack the field.
|
|
460
|
+
return { ...cached, ok: true };
|
|
411
461
|
}
|
|
412
462
|
|
|
413
463
|
// BDG-2: reserve-then-adjust budget. Before spawning, estimate the cost and
|
|
@@ -490,18 +540,32 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
490
540
|
runId: manifest.runId,
|
|
491
541
|
role: call.role ?? call.agent,
|
|
492
542
|
});
|
|
493
|
-
|
|
543
|
+
const parsed = parsePiJsonOutput(childResult.stdout);
|
|
544
|
+
// DR-01 P0 completeness (DWF): the attemptErrorFor empty-settle branch
|
|
545
|
+
// lives in the task-runner attempt loop, which ctx.agent() bypasses
|
|
546
|
+
// (runWorker is called directly here). Without this gate a
|
|
547
|
+
// 429-exhausted worker (exit 0, empty final assistant message) passes
|
|
548
|
+
// the exitCode/error check and its rawFinalText/finalText — an early
|
|
549
|
+
// opening sentence — becomes the step's "result": a false-green DWF
|
|
550
|
+
// step. See run team_20261014082741 for the live stdio shape.
|
|
551
|
+
// R6-7 (DR-03 round-6): the settledWithoutAnswer term now carries the
|
|
552
|
+
// sibling exemptions (patches = real work; cancelled ≠ 429) via
|
|
553
|
+
// dwfSettledWithoutAnswerFailure — mirroring child-executor.ts:428-431
|
|
554
|
+
// exactly, so DWF and the task-runner classify the same transcript the
|
|
555
|
+
// same way (a patch-only coder step or an aborted attempt no longer fails
|
|
556
|
+
// as rate-limit exhaustion here while passing there).
|
|
557
|
+
const settledFailure = dwfSettledWithoutAnswerFailure(childResult, parsed);
|
|
558
|
+
if (childResult.exitCode !== 0 || childResult.error || settledFailure !== undefined) {
|
|
494
559
|
// BDG-2: un-reserve the estimate on spawn failure.
|
|
495
560
|
wfState.spent -= estimate;
|
|
496
561
|
reserved = false;
|
|
497
562
|
return {
|
|
498
563
|
ok: false,
|
|
499
564
|
text: "",
|
|
500
|
-
error: childResult.error ?? `exit ${childResult.exitCode}`,
|
|
565
|
+
error: childResult.error ?? settledFailure ?? `exit ${childResult.exitCode}`,
|
|
501
566
|
durationMs: Date.now() - started,
|
|
502
567
|
};
|
|
503
568
|
}
|
|
504
|
-
const parsed = parsePiJsonOutput(childResult.stdout);
|
|
505
569
|
// round-14 P1-2: accumulate this run's token usage into the workflow budget.
|
|
506
570
|
// BDG-2: adjust the reserve — subtract the estimate, add the actual usage.
|
|
507
571
|
// This correctly reduces spent when actualUsage < estimate.
|
|
@@ -539,10 +603,16 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
539
603
|
durationMs: Date.now() - started,
|
|
540
604
|
};
|
|
541
605
|
}
|
|
542
|
-
// PERS-1: cache the successful result for idempotency on resume
|
|
606
|
+
// PERS-1: cache the FULL successful result for idempotency on resume
|
|
607
|
+
// (R6-3: structured + artifactPath + actual durationMs included, so
|
|
608
|
+
// replay returns the same payload a live call would have).
|
|
543
609
|
completedAgentCalls[callId] = {
|
|
610
|
+
ok: true,
|
|
544
611
|
text,
|
|
545
|
-
|
|
612
|
+
structured: extracted.structured ? extracted.data : undefined,
|
|
613
|
+
usage: parsed.usage,
|
|
614
|
+
artifactPath: artifact.path,
|
|
615
|
+
durationMs: Date.now() - started,
|
|
546
616
|
};
|
|
547
617
|
return {
|
|
548
618
|
ok: true,
|
|
@@ -567,23 +637,17 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
567
637
|
durationMs: Date.now() - started,
|
|
568
638
|
};
|
|
569
639
|
} finally {
|
|
570
|
-
//
|
|
571
|
-
//
|
|
572
|
-
//
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
} catch (cleanupError) {
|
|
577
|
-
logInternalError("dynamic-workflow-context.worktree-cleanup", cleanupError, `worktreePath=${worktreePath}`);
|
|
578
|
-
}
|
|
579
|
-
}
|
|
580
|
-
// round-18 P2-3: checkpoint AFTER the agent completes (success or fail) so a
|
|
581
|
-
// crash between agent calls leaves durable state to resume from. The counter is
|
|
582
|
-
// incremented here (after the call) so the checkpoint reflects the call that ran.
|
|
640
|
+
// R6-12 (DR-03 round-6): CHECKPOINT BEFORE CLEANUP. agentCount++ and
|
|
641
|
+
// onCheckpoint must land BEFORE the worktree-cleanup await — previously
|
|
642
|
+
// the checkpoint ran after it, so a crash/hang during cleanup lost the
|
|
643
|
+
// just-completed call from the durable checkpoint and resume re-spawned
|
|
644
|
+
// the same agent (duplicate work/tokens). Cleanup is best-effort and
|
|
645
|
+
// idempotent-ish (a leaked worktree is recoverable; a lost call is not).
|
|
583
646
|
agentCount++;
|
|
584
647
|
if (opts.onCheckpoint) {
|
|
585
648
|
try {
|
|
586
649
|
opts.onCheckpoint({
|
|
650
|
+
version: DWF_CHECKPOINT_VERSION,
|
|
587
651
|
runId: manifest.runId,
|
|
588
652
|
vars: ctx.vars,
|
|
589
653
|
phases: phaseState.phases,
|
|
@@ -598,6 +662,17 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
598
662
|
logInternalError("dynamic-workflow-context.checkpoint", checkpointError, `runId=${manifest.runId}`);
|
|
599
663
|
}
|
|
600
664
|
}
|
|
665
|
+
// round-17 P2-4: clean up the worktree after the agent completes (success
|
|
666
|
+
// OR failure). Captures the diff as an artifact before removal. Best-effort
|
|
667
|
+
// — a leak must never crash the workflow. R6-12: runs AFTER the checkpoint
|
|
668
|
+
// (see above) — ordering is the fix.
|
|
669
|
+
if (worktreePath) {
|
|
670
|
+
try {
|
|
671
|
+
await cleanupAgentWorktreeAsync(manifest, worktreePath, worktreeBranch);
|
|
672
|
+
} catch (cleanupError) {
|
|
673
|
+
logInternalError("dynamic-workflow-context.worktree-cleanup", cleanupError, `worktreePath=${worktreePath}`);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
601
676
|
semaphore.release();
|
|
602
677
|
}
|
|
603
678
|
},
|
|
@@ -718,19 +793,35 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
718
793
|
};
|
|
719
794
|
},
|
|
720
795
|
async retry(taskId: string, retryOpts?: { feedback?: string }): Promise<AgentResult> {
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
796
|
+
// R6-10 (DR-03 round-6): executeWithRetry retries THROWN errors only, but
|
|
797
|
+
// ctx.agent() returns {ok:false} for every failure class except
|
|
798
|
+
// DwfAgentCallCapError — so the previous pass-through never retried an
|
|
799
|
+
// ok:false attempt (a 1-attempt "retry"). Bridge: throw DwfRetryFailure
|
|
800
|
+
// (typed, carries the AgentResult) on !ok so executeWithRetry retries it;
|
|
801
|
+
// after exhaustion catch it and return the CARRIED result (public return
|
|
802
|
+
// type/shape unchanged). DwfAgentCallCapError is not wrapped — it keeps
|
|
803
|
+
// its terminate-the-workflow semantics and propagates untouched.
|
|
804
|
+
try {
|
|
805
|
+
return await executeWithRetry(
|
|
806
|
+
async () => {
|
|
807
|
+
const res = await ctx.agent({
|
|
808
|
+
role: "executor",
|
|
809
|
+
prompt: `Re-do task '${taskId}'.${retryOpts?.feedback ? ` Feedback: ${retryOpts.feedback}` : ""}`,
|
|
810
|
+
});
|
|
811
|
+
if (!res.ok) throw new DwfRetryFailure(res);
|
|
812
|
+
return res;
|
|
813
|
+
},
|
|
814
|
+
{
|
|
815
|
+
maxAttempts: 3,
|
|
816
|
+
backoffMs: 0,
|
|
817
|
+
jitterRatio: 0,
|
|
818
|
+
exponentialFactor: 1,
|
|
819
|
+
},
|
|
820
|
+
);
|
|
821
|
+
} catch (error) {
|
|
822
|
+
if (error instanceof DwfRetryFailure) return error.result;
|
|
823
|
+
throw error;
|
|
824
|
+
}
|
|
734
825
|
},
|
|
735
826
|
mail(
|
|
736
827
|
to: string,
|
|
@@ -778,6 +869,13 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
778
869
|
}
|
|
779
870
|
// Idempotency: same phase title → no event, no state change.
|
|
780
871
|
if (title === phaseState.currentPhase) return;
|
|
872
|
+
// R6-12 (DR-03 round-6): phase no-dup on resume. A phase already recorded
|
|
873
|
+
// in the hydrated checkpoint is NOT re-emitted — the resumed script
|
|
874
|
+
// re-walks its phase history and would otherwise duplicate phase_started/
|
|
875
|
+
// phase_completed events in events.jsonl. Resume-only guard: hydratedPhases
|
|
876
|
+
// is empty on a fresh run, so legitimate same-title re-entry later in a
|
|
877
|
+
// fresh script still emits (see the fresh-run counter-pin test).
|
|
878
|
+
if (hydratedPhases.has(title)) return;
|
|
781
879
|
// Close out the previous open phase BEFORE the new one opens.
|
|
782
880
|
// U2 (2026-10-10): kept SYNC — ctx closures are exercised DIRECTLY by
|
|
783
881
|
// unit tests that readEvents() synchronously right after the call
|
|
@@ -921,6 +1019,8 @@ export function getWorkflowCheckpoint(ctx: WorkflowCtx): DwfCheckpointState {
|
|
|
921
1019
|
const phaseState = getWorkflowPhaseState(ctx);
|
|
922
1020
|
const logs = getWorkflowLogs(ctx);
|
|
923
1021
|
return {
|
|
1022
|
+
// R6-12: snapshot carries the format version (mirrors the onCheckpoint payload).
|
|
1023
|
+
version: DWF_CHECKPOINT_VERSION,
|
|
924
1024
|
runId: ctx.runId,
|
|
925
1025
|
vars: ctx.vars,
|
|
926
1026
|
phases: phaseState?.phases ?? [],
|
|
@@ -928,13 +1028,41 @@ export function getWorkflowCheckpoint(ctx: WorkflowCtx): DwfCheckpointState {
|
|
|
928
1028
|
logs: logs ?? [],
|
|
929
1029
|
spent: ctx.budget.spent(),
|
|
930
1030
|
agentCount: (ctx as unknown as { __agentCount?: number }).__agentCount ?? 0,
|
|
931
|
-
completedAgentCalls:
|
|
932
|
-
(ctx as unknown as { __completedAgentCalls?: Record<string, { text: string; usage?: { input: number; output: number } }> })
|
|
933
|
-
.__completedAgentCalls ?? {},
|
|
1031
|
+
completedAgentCalls: (ctx as unknown as { __completedAgentCalls?: Record<string, AgentResult> }).__completedAgentCalls ?? {},
|
|
934
1032
|
updatedAt: new Date().toISOString(),
|
|
935
1033
|
};
|
|
936
1034
|
}
|
|
937
1035
|
|
|
1036
|
+
/**
|
|
1037
|
+
* R6-3 (DR-03 round-6, HIGH): deterministic PERS-1 cache key over ALL
|
|
1038
|
+
* call-shaping AgentCallOpts fields — the JSON of the resolved call object
|
|
1039
|
+
* with stable ordering:
|
|
1040
|
+
* - object keys sorted (two identical calls built with different literal key
|
|
1041
|
+
* orders still share a cache entry);
|
|
1042
|
+
* - top-level array fields (inputs, skill) keyed as a SORTED copy — the same
|
|
1043
|
+
* set in a different order is the same call;
|
|
1044
|
+
* - undefined fields dropped;
|
|
1045
|
+
* - the schema is included BY VALUE (a structural change to the expected
|
|
1046
|
+
* output shape changes the dispatched task and must miss the cache — the
|
|
1047
|
+
* old "yes"/"no" placeholder collided different schemas).
|
|
1048
|
+
* Previously the key was {role, agent, prompt, model, schema:"yes"/"no"} only,
|
|
1049
|
+
* omitting inputs/systemPrompt/disableTools/maxTurns/graceTurns/worktree/skill
|
|
1050
|
+
* — e.g. the same prompt with DIFFERENT inputs (which provably alter the
|
|
1051
|
+
* dispatched task via composeAgentTask) hit the first call's cache entry on
|
|
1052
|
+
* EVERY call and returned the wrong call's result (lead-verified :398-412).
|
|
1053
|
+
* Note: fields that do NOT shape the call (description) are included anyway —
|
|
1054
|
+
* over-keying only costs a cache miss (a fresh spawn), never correctness.
|
|
1055
|
+
*/
|
|
1056
|
+
function computeCallId(call: AgentCallOpts): string {
|
|
1057
|
+
const normalized: Record<string, unknown> = {};
|
|
1058
|
+
for (const key of Object.keys(call).sort()) {
|
|
1059
|
+
const value: unknown = call[key as keyof AgentCallOpts];
|
|
1060
|
+
if (value === undefined) continue;
|
|
1061
|
+
normalized[key] = Array.isArray(value) ? [...(value as unknown[])].sort() : value;
|
|
1062
|
+
}
|
|
1063
|
+
return JSON.stringify(normalized);
|
|
1064
|
+
}
|
|
1065
|
+
|
|
938
1066
|
/** Compose the agent task: prompt + optional dependency-input context block. */
|
|
939
1067
|
function composeAgentTask(call: AgentCallOpts): string {
|
|
940
1068
|
let base = call.prompt;
|
|
@@ -20,6 +20,8 @@ import { createRequire } from "node:module";
|
|
|
20
20
|
import type { AgentConfig } from "../../agents/agent-config.ts";
|
|
21
21
|
import { loadConfig } from "../../config/config.ts";
|
|
22
22
|
import { effectiveRunConfig } from "../../extension/team-tool/config-patch.ts";
|
|
23
|
+
import { isTerminalTaskStatus } from "../../state/contracts.ts";
|
|
24
|
+
import { withRunLockSync } from "../../state/coordination/locks.ts";
|
|
23
25
|
import { appendEventBuffered } from "../../state/event-log/event-log.ts";
|
|
24
26
|
import { registerActiveRun, unregisterActiveRun } from "../../state/stores/active-run-registry.ts";
|
|
25
27
|
import { collectRunMetrics } from "../../state/stores/run-metrics.ts";
|
|
@@ -28,14 +30,52 @@ import type { GoalLoopState, GoalLoopStatus, GoalVerdict, TeamRunManifest, TeamT
|
|
|
28
30
|
import type { TeamConfig } from "../../teams/team-config.ts";
|
|
29
31
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
30
32
|
import type { WorkflowConfig } from "../../workflows/workflow-config.ts";
|
|
33
|
+
import { isRunLockBusyError } from "../broker/protocol/lock-busy.ts";
|
|
31
34
|
import { resolveCrewRuntime } from "../model/runtime-resolver.ts";
|
|
32
35
|
import { registryFromModelContext } from "../model/session-model.ts";
|
|
33
36
|
import { executeTeamRun } from "../team-runner.ts";
|
|
34
37
|
import { compareSnapshot, snapshotManifests } from "../verification/verification-integrity.ts";
|
|
35
|
-
import { acquireWorkspaceLock, type WorkspaceLockHandle } from "../workspace-lock.ts";
|
|
38
|
+
import { acquireWorkspaceLock, refreshWorkspaceLockHeartbeat, type WorkspaceLockHandle } from "../workspace-lock.ts";
|
|
36
39
|
import { bundleEvidence, evaluateGoal } from "./goal-evaluator.ts";
|
|
37
40
|
import { GoalStore } from "./goal-state-store.ts";
|
|
38
41
|
|
|
42
|
+
/**
|
|
43
|
+
* D5 (DR-02 Wave 2 Cluster C): merge the turn's FINAL task states onto a
|
|
44
|
+
* FRESH tasks.json read. The turn owns exactly the terminal-evidence fields
|
|
45
|
+
* its executor wrote (status/finishedAt/resultArtifact/error/exitCode);
|
|
46
|
+
* everything else on the fresh task (heartbeat/lastSeenAt, claim, delegate
|
|
47
|
+
* state, usage…) may be NEWER than the turn's in-memory snapshot and must
|
|
48
|
+
* never be regressed by the post-turn persist.
|
|
49
|
+
*
|
|
50
|
+
* Terminal-preserve: a task that is ALREADY terminal in the fresh read (an
|
|
51
|
+
* external decision — operator cancel, crash-recovery, the async-died
|
|
52
|
+
* notifier) keeps its fresh state entirely; the turn must not resurrect or
|
|
53
|
+
* re-decide it (same guard class as heartbeat.ts R13-S1). Turn tasks missing
|
|
54
|
+
* from the fresh read are appended — the turn created them and the
|
|
55
|
+
* budget/audit reads this persist exists for expect them on disk.
|
|
56
|
+
*/
|
|
57
|
+
export function mergeTurnFinalTasksOntoFresh(freshTasks: readonly TeamTaskState[], turnTasks: readonly TeamTaskState[]): TeamTaskState[] {
|
|
58
|
+
const freshById = new Map(freshTasks.map((task) => [task.id, task]));
|
|
59
|
+
const turnById = new Map(turnTasks.map((task) => [task.id, task]));
|
|
60
|
+
const merged = freshTasks.map((freshTask) => {
|
|
61
|
+
const turnTask = turnById.get(freshTask.id);
|
|
62
|
+
if (!turnTask) return freshTask;
|
|
63
|
+
if (isTerminalTaskStatus(freshTask.status)) return freshTask;
|
|
64
|
+
return {
|
|
65
|
+
...freshTask,
|
|
66
|
+
status: turnTask.status,
|
|
67
|
+
...(turnTask.finishedAt !== undefined ? { finishedAt: turnTask.finishedAt } : {}),
|
|
68
|
+
...(turnTask.resultArtifact !== undefined ? { resultArtifact: turnTask.resultArtifact } : {}),
|
|
69
|
+
...(turnTask.error !== undefined ? { error: turnTask.error } : {}),
|
|
70
|
+
...(turnTask.exitCode !== undefined ? { exitCode: turnTask.exitCode } : {}),
|
|
71
|
+
};
|
|
72
|
+
});
|
|
73
|
+
for (const turnTask of turnTasks) {
|
|
74
|
+
if (!freshById.has(turnTask.id)) merged.push(turnTask);
|
|
75
|
+
}
|
|
76
|
+
return merged;
|
|
77
|
+
}
|
|
78
|
+
|
|
39
79
|
/** Required minimal shape for the worker + agents discovery (P0 uses the goal's workerAgent). */
|
|
40
80
|
export interface GoalLoopRuntimeDeps {
|
|
41
81
|
/** Resolve the agent configs reachable from cwd (used for executeTeamRun's agents arg). */
|
|
@@ -538,6 +578,48 @@ export function deriveTranscriptPath(artifactsRoot: string, tasks: import("../..
|
|
|
538
578
|
return undefined;
|
|
539
579
|
}
|
|
540
580
|
|
|
581
|
+
/** Interval (ms) between workspace-lock heartbeat refreshes while a goal loop runs (WI-2 / DR-02 R4-3). 15s = 4x margin under the 60s staleness threshold. */
|
|
582
|
+
export const WORKSPACE_LOCK_HEARTBEAT_REFRESH_MS = 15_000;
|
|
583
|
+
|
|
584
|
+
export interface StartWorkspaceLockHeartbeatRefreshOptions {
|
|
585
|
+
/** Test injection: override the refresh interval (ms). */
|
|
586
|
+
refreshMs?: number;
|
|
587
|
+
/** Test injection: override the refresh implementation (default: refreshWorkspaceLockHeartbeat). */
|
|
588
|
+
refresh?: (lockPath: string, goalId: string) => boolean;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
/**
|
|
592
|
+
* Start the periodic workspace-lock heartbeat refresh for a lock acquired by
|
|
593
|
+
* runGoalLoop (WI-2 / DR-02 R4-3).
|
|
594
|
+
*
|
|
595
|
+
* Without refresh, workspace-lock isLockStale clause (b) marks the lock stale
|
|
596
|
+
* after 60s REGARDLESS of holder pid liveness — a goal whose turn takes >60s
|
|
597
|
+
* gets its workspace lock force-stolen by a second goal on the same cwd
|
|
598
|
+
* (multi-goal clobber). This timer keeps the heartbeat young for the whole
|
|
599
|
+
* loop lifetime. It is unref()-ed (must not keep the background process
|
|
600
|
+
* alive) and the returned stop() MUST be called in the same finally that
|
|
601
|
+
* releases the lock. Refresh failures are logged as warn and swallowed — a
|
|
602
|
+
* heartbeat miss must never throw into the goal loop.
|
|
603
|
+
*/
|
|
604
|
+
export function startWorkspaceLockHeartbeatRefresh(
|
|
605
|
+
handle: Pick<WorkspaceLockHandle, "lockPath" | "goalId">,
|
|
606
|
+
opts: StartWorkspaceLockHeartbeatRefreshOptions = {},
|
|
607
|
+
): { stop: () => void } {
|
|
608
|
+
const refreshMs = opts.refreshMs ?? WORKSPACE_LOCK_HEARTBEAT_REFRESH_MS;
|
|
609
|
+
const refresh = opts.refresh ?? refreshWorkspaceLockHeartbeat;
|
|
610
|
+
const timer = setInterval(() => {
|
|
611
|
+
try {
|
|
612
|
+
refresh(handle.lockPath, handle.goalId);
|
|
613
|
+
} catch (error) {
|
|
614
|
+
// Never throw into the loop (interval callbacks crashing node:test aside,
|
|
615
|
+
// an escaping throw here would kill the background goal process).
|
|
616
|
+
logInternalError("goal-loop.workspaceLockHeartbeat", error, `goalId=${handle.goalId} lockPath=${handle.lockPath}`, "warn");
|
|
617
|
+
}
|
|
618
|
+
}, refreshMs);
|
|
619
|
+
timer.unref(); // the refresh timer must not keep the detached goal-loop process alive
|
|
620
|
+
return { stop: () => clearInterval(timer) };
|
|
621
|
+
}
|
|
622
|
+
|
|
541
623
|
/**
|
|
542
624
|
* Run the autonomous goal loop. Returns {manifest, tasks, goalState} — the OUTER
|
|
543
625
|
* manifest is the synthetic goal-loop manifest (runKind:"goal-loop"); per-turn
|
|
@@ -575,10 +657,18 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
|
|
|
575
657
|
// the background-process boundary. `goal start` / `goal resume` pre-check via isWorkspaceBusy
|
|
576
658
|
// for a good error message; this acquisition is the authoritative claim.
|
|
577
659
|
let workspaceLock: WorkspaceLockHandle | undefined;
|
|
660
|
+
// WI-2 (DR-02 R4-3): periodic heartbeat refresh for the lock above — stopped
|
|
661
|
+
// in the same finally that releases it (see startWorkspaceLockHeartbeatRefresh).
|
|
662
|
+
let lockHeartbeat: { stop: () => void } | undefined;
|
|
578
663
|
try {
|
|
579
664
|
workspaceLock = await acquireWorkspaceLock(goal.cwd, goal.goalId, {
|
|
580
665
|
signal,
|
|
581
666
|
});
|
|
667
|
+
// WI-2 (DR-02 R4-3): keep the heartbeat young for the whole loop lifetime —
|
|
668
|
+
// without this, isLockStale clause (b) force-steals the lock after 60s even
|
|
669
|
+
// though this pid is alive (multi-goal clobber). Started immediately after
|
|
670
|
+
// acquire; the timer is unref()-ed and never throws into the loop.
|
|
671
|
+
lockHeartbeat = startWorkspaceLockHeartbeatRefresh(workspaceLock);
|
|
582
672
|
} catch (error) {
|
|
583
673
|
logInternalError("goal-loop.workspaceLock", error, `goalId=${goal.goalId} cwd=${goal.cwd}`);
|
|
584
674
|
goal = safeSetStatus(store, goal.goalId, "blocked", goal, eventsPath);
|
|
@@ -753,10 +843,41 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
|
|
|
753
843
|
}
|
|
754
844
|
|
|
755
845
|
// Persist final task states for budget/audit reads.
|
|
846
|
+
// D5 (DR-02 Wave 2 Cluster C): the bare saveRunTasks(turnResult.tasks) was
|
|
847
|
+
// an UNLOCKED write of the executor's in-memory snapshot (goal-loop-runner.ts
|
|
848
|
+
// ":757") — it clobbered any NEWER state written under the run lock between
|
|
849
|
+
// the executor's last persist and this save (worker heartbeats/lastSeenAt,
|
|
850
|
+
// task claims, delegate statuses). Now a run-locked RMW: fresh re-read inside
|
|
851
|
+
// the lock, then a field-level merge — the turn re-decides only the fields it
|
|
852
|
+
// owns and never regresses fresh heartbeat/claim/delegate data.
|
|
756
853
|
try {
|
|
757
|
-
|
|
854
|
+
withRunLockSync(turnResult.manifest, () => {
|
|
855
|
+
const fresh = loadRunManifestById(turnResult.manifest.cwd, turnResult.manifest.runId);
|
|
856
|
+
if (!fresh) {
|
|
857
|
+
logInternalError(
|
|
858
|
+
"goal-loop.saveTurnTasks",
|
|
859
|
+
new Error(`turn run '${turnResult.manifest.runId}' vanished before the final task persist — skipped`),
|
|
860
|
+
"best-effort",
|
|
861
|
+
"warn",
|
|
862
|
+
);
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
865
|
+
saveRunTasks(fresh.manifest, mergeTurnFinalTasksOntoFresh(fresh.tasks, turnResult.tasks));
|
|
866
|
+
});
|
|
758
867
|
} catch (error) {
|
|
759
|
-
|
|
868
|
+
if (isRunLockBusyError(error)) {
|
|
869
|
+
// Best-effort persist: the executor already persisted terminal task
|
|
870
|
+
// states during the turn — this merge is belt-and-suspenders for
|
|
871
|
+
// budget/audit reads, so contention skips it with a warn.
|
|
872
|
+
logInternalError(
|
|
873
|
+
"goal-loop.saveTurnTasks",
|
|
874
|
+
error,
|
|
875
|
+
`turnRunId=${created.manifest.runId} run lock busy — skipped final task merge`,
|
|
876
|
+
"warn",
|
|
877
|
+
);
|
|
878
|
+
} else {
|
|
879
|
+
logInternalError("goal-loop.saveTurnTasks", error, `turnRunId=${created.manifest.runId}`);
|
|
880
|
+
}
|
|
760
881
|
}
|
|
761
882
|
|
|
762
883
|
// GL-1: if the turn ended in blocked/failed status, stop the loop without spawning the judge.
|
|
@@ -908,6 +1029,12 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
|
|
|
908
1029
|
}).catch((e) => logInternalError("goal-loop.buffered", e, "type=goal.loop_error"));
|
|
909
1030
|
goal = safeSetStatus(store, goal.goalId, "blocked", goal, eventsPath);
|
|
910
1031
|
} finally {
|
|
1032
|
+
// WI-2: stop the heartbeat refresh BEFORE releasing the lock it was keeping alive.
|
|
1033
|
+
try {
|
|
1034
|
+
lockHeartbeat?.stop();
|
|
1035
|
+
} catch {
|
|
1036
|
+
/* best-effort */
|
|
1037
|
+
}
|
|
911
1038
|
// P1g: release the workspace lock (held since loop start).
|
|
912
1039
|
try {
|
|
913
1040
|
workspaceLock?.release();
|
|
@@ -231,6 +231,11 @@ export class HeartbeatWatcher {
|
|
|
231
231
|
if (!activeKeys.has(key) && ts < cutoff) {
|
|
232
232
|
this.lastLevel.delete(key);
|
|
233
233
|
this.consecutiveDead.delete(key);
|
|
234
|
+
// M-8 (2026-10-16): also drop the deadletter cooldown marker —
|
|
235
|
+
// before this the map leaked one entry per vanished run:taskId key
|
|
236
|
+
// and a stale cooldown timestamp could suppress a legitimate
|
|
237
|
+
// re-trigger after the same task reappeared.
|
|
238
|
+
this.lastDeadletterTriggerAt.delete(key);
|
|
234
239
|
this.lastSeen.delete(key);
|
|
235
240
|
}
|
|
236
241
|
}
|
|
@@ -242,5 +247,7 @@ export class HeartbeatWatcher {
|
|
|
242
247
|
this.lastLevel.clear();
|
|
243
248
|
this.consecutiveDead.clear();
|
|
244
249
|
this.lastSeen.clear();
|
|
250
|
+
// M-8: clear the deadletter cooldown map too (was leaked before).
|
|
251
|
+
this.lastDeadletterTriggerAt.clear();
|
|
245
252
|
}
|
|
246
253
|
}
|
|
@@ -908,7 +908,9 @@ export function admitEmbeddedSession(input: {
|
|
|
908
908
|
});
|
|
909
909
|
// Raced against the prompt; a pre-race rejection must not surface as an
|
|
910
910
|
// unhandled rejection (which would trip our own process guard).
|
|
911
|
-
escapeSignal.catch(() => {
|
|
911
|
+
escapeSignal.catch(() => {
|
|
912
|
+
/* intentional UR-guard — see comment above */
|
|
913
|
+
});
|
|
912
914
|
let tornDown = false;
|
|
913
915
|
registerEmbeddedSessionGuard(
|
|
914
916
|
input.agentId,
|