pi-crew 0.11.1 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -9
- package/README.md +161 -1036
- package/agents/verifier.md +18 -7
- package/dist/index.mjs +742 -91462
- package/docs/README.md +57 -46
- package/docs/architecture.md +87 -33
- package/docs/commands-reference.md +9 -5
- package/docs/troubleshooting.md +3 -2
- package/package.json +5 -4
- package/schema.json +29 -0
- package/skills/real-test-pi-crew/SKILL.md +206 -41
- package/src/agents/agent-config.ts +1 -1
- package/src/agents/discover-agents.ts +1 -1
- package/src/config/config-validation.ts +15 -0
- package/src/config/config.ts +47 -13
- package/src/config/env-vars.ts +35 -0
- package/src/config/types.ts +19 -0
- package/src/errors.ts +2 -2
- package/src/extension/async-notifier.ts +55 -8
- package/src/extension/crew-input-router.ts +9 -2
- package/src/extension/help.ts +21 -10
- package/src/extension/knowledge-injection.ts +2 -1
- package/src/extension/management.ts +8 -3
- package/src/extension/notification-sink.ts +17 -0
- package/src/extension/plan-orchestrate.ts +19 -3
- package/src/extension/registration/command-utils.ts +28 -2
- package/src/extension/registration/commands/dashboard.ts +11 -1
- package/src/extension/registration/commands/manage.ts +31 -15
- package/src/extension/registration/commands/run.ts +24 -2
- package/src/extension/registration/commands/shared.ts +34 -10
- package/src/extension/registration/commands/status.ts +25 -2
- package/src/extension/registration/context-builder.ts +8 -2
- package/src/extension/registration/foreground-run-controller.ts +2 -2
- package/src/extension/registration/health-notify-policy.ts +100 -0
- package/src/extension/registration/lazy-configurers.ts +38 -0
- package/src/extension/registration/lifecycle-handlers.ts +107 -40
- package/src/extension/registration/lifecycle.ts +77 -10
- package/src/extension/registration/observability.ts +125 -53
- package/src/extension/registration/registration-types.ts +7 -5
- package/src/extension/registration/runtime-cleanup.ts +9 -3
- package/src/extension/registration/subagent-helpers.ts +38 -0
- package/src/extension/registration/team-tool.ts +9 -2
- package/src/extension/registration/wire-cross-extension.ts +30 -0
- package/src/extension/run-compare.ts +220 -0
- package/src/extension/run-export.ts +37 -5
- package/src/extension/run-maintenance.ts +155 -5
- package/src/extension/team-manager-command.ts +3 -2
- package/src/extension/team-tool/dispatch/index.ts +3 -2
- package/src/extension/team-tool/dispatch/manage.ts +5 -2
- package/src/extension/team-tool/goal-wrap.ts +7 -0
- package/src/extension/team-tool/goal.ts +16 -1
- package/src/extension/team-tool/handle-settings.ts +19 -2
- package/src/extension/team-tool/health-monitor.ts +21 -7
- package/src/extension/team-tool/lifecycle-actions.ts +49 -1
- package/src/extension/team-tool/parallel-dispatch.ts +16 -1
- package/src/extension/team-tool/plan.ts +10 -0
- package/src/extension/team-tool/routing-hint.ts +63 -0
- package/src/extension/team-tool/run-intent.ts +1 -1
- package/src/extension/team-tool/run.ts +1 -1
- package/src/extension/team-tool/status.ts +4 -0
- package/src/extension/team-tool.ts +52 -6
- package/src/extension/webhook-notify.ts +382 -0
- package/src/observability/metric-sink.ts +12 -2
- package/src/prompt/prompt-runtime.ts +82 -31
- package/src/prompt/worker-events-channel.ts +12 -0
- package/src/runtime/README.md +1 -1
- package/src/runtime/async-runner.ts +123 -1
- package/src/runtime/background-runner.ts +313 -234
- package/src/runtime/broker/crew-broker.ts +76 -71
- package/src/runtime/broker/delegate/delegate-event.ts +12 -8
- package/src/runtime/broker/delegate/shadow-lifecycle.ts +92 -0
- package/src/runtime/broker/protocol/connection-state.ts +10 -0
- package/src/runtime/broker/protocol/lock-busy.ts +76 -0
- package/src/runtime/broker/wait-status-cache.ts +1 -1
- package/src/runtime/child-pi/child-pi-timers.ts +1 -1
- package/src/runtime/child-pi/mock-fixtures.ts +48 -0
- package/src/runtime/crew-agent-records.ts +337 -45
- package/src/runtime/deadletter.ts +43 -1
- package/src/runtime/delegate-spawn.ts +21 -3
- package/src/runtime/deterministic-ast.ts +14 -2
- package/src/runtime/dispatch-batch.ts +72 -5
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +78 -58
- package/src/runtime/goal-workflow/dynamic-workflow-runner.ts +14 -2
- package/src/runtime/goal-workflow/goal-loop-runner.ts +105 -4
- package/src/runtime/group-join.ts +11 -3
- package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
- package/src/runtime/live-session/live-session-runtime.ts +1 -1
- package/src/runtime/model/model-fallback.ts +21 -1
- package/src/runtime/model/pi-args.ts +8 -10
- package/src/runtime/model/runtime-warmup.ts +2 -0
- package/src/runtime/output/output-validator.ts +10 -5
- package/src/runtime/plan-approval.ts +14 -1
- package/src/runtime/recovery/crash-recovery.ts +96 -49
- package/src/runtime/run-worker.ts +12 -1
- package/src/runtime/scheduling/global-worker-cap.ts +13 -6
- package/src/runtime/scheduling/run-coalesced-task-group.ts +27 -1
- package/src/runtime/scheduling/scheduler.ts +49 -13
- package/src/runtime/scheduling/semaphore.ts +148 -20
- package/src/runtime/scratchpad/README.md +1 -1
- package/src/runtime/scratchpad/protocol.ts +1 -1
- package/src/runtime/settings-store.ts +1 -1
- package/src/runtime/skill-instructions.ts +22 -0
- package/src/runtime/stale-reconciler.ts +85 -13
- package/src/runtime/subagent-manager.ts +43 -13
- package/src/runtime/task-runner/child-executor.ts +17 -6
- package/src/runtime/task-runner/post-execution.ts +44 -14
- package/src/runtime/task-runner/pre-execution.ts +26 -2
- package/src/runtime/task-runner/prompt-builder.ts +150 -45
- package/src/runtime/task-runner/retrieval-orchestrator.ts +36 -14
- package/src/runtime/task-runner.ts +28 -1
- package/src/runtime/team-runner.ts +42 -2
- package/src/runtime/verification/verification-worktree.ts +3 -0
- package/src/runtime/workspace-lock.ts +4 -1
- package/src/schema/config-schema.ts +14 -0
- package/src/schema/team-tool-schema.ts +17 -0
- package/src/state/atomic-write.ts +53 -0
- package/src/state/contracts.ts +109 -0
- package/src/state/coordination/locks.ts +191 -33
- package/src/state/coordination/mailbox.ts +147 -17
- package/src/state/crew-init.ts +87 -12
- package/src/state/event-log/cursor.ts +37 -1
- package/src/state/event-log/event-log-rotation.ts +72 -7
- package/src/state/stores/active-run-registry.ts +13 -1
- package/src/state/stores/state-store.ts +120 -23
- package/src/state/types.ts +4 -0
- package/src/ui/dashboard-panes/agents-pane.ts +22 -10
- package/src/ui/heartbeat-aggregator.ts +34 -0
- package/src/ui/inline-panel/index.ts +1 -1
- package/src/ui/keybinding-map.ts +22 -4
- package/src/ui/live-conversation-overlay.ts +6 -3
- package/src/ui/render-diff.ts +19 -6
- package/src/ui/run-dashboard.ts +109 -6
- package/src/ui/run-snapshot-cache.ts +27 -4
- package/src/ui/spinner.ts +26 -2
- package/src/ui/tool-progress-formatter.ts +2 -1
- package/src/ui/tool-renderers/brief-mode.ts +2 -1
- package/src/ui/tool-renderers/index.ts +3 -3
- package/src/ui/widget/task-list.ts +8 -5
- package/src/ui/widget/widget-model.ts +3 -1
- package/src/utils/incremental-reader.ts +11 -3
- package/src/utils/paths.ts +94 -12
- package/src/utils/project-markers.ts +40 -0
- package/src/worktree/worktree-manager.ts +206 -26
- package/tsconfig.json +2 -0
- package/workflows/distill.workflow.md +3 -3
- package/workflows/fast-fix.workflow.md +1 -1
- package/workflows/plan-execute.workflow.md +1 -1
- package/workflows/review.workflow.md +1 -1
- package/workflows/strict-fast-fix.workflow.md +1 -1
- package/docs/migration-v0.4-v0.5.md +0 -208
- package/docs/runtime-flow.md +0 -148
- package/src/ui/card-colors.ts +0 -168
- package/src/utils/completion-dedupe.ts +0 -63
- package/src/utils/resolve-shell.ts +0 -36
|
@@ -123,10 +123,15 @@ export function validateWorkerOutput(role: string, output: string): OutputValida
|
|
|
123
123
|
*
|
|
124
124
|
* A trimmed non-empty line counts as log noise ONLY if it matches one of:
|
|
125
125
|
* 1. a bracket-tag extension log line: `[oc-go] ...`, `[pi-qwen-mm] ...`,
|
|
126
|
-
* `[pi-qwen-mm] [core] [stderr]
|
|
127
|
-
*
|
|
128
|
-
* `[
|
|
129
|
-
*
|
|
126
|
+
* `[pi-qwen-mm] [core] [stderr] ...`, and — Finding #2 (2026-09-29
|
|
127
|
+
* battery) — colon-bearing extension tags like
|
|
128
|
+
* `[pi-crew:crash-recovery.reconcileStaleRuns] ...` or
|
|
129
|
+
* `[pi-crew:crew-vibes.publish-quota-status] ...`. The leading tag
|
|
130
|
+
* starts with a lowercase char (deliberately excludes capitalized
|
|
131
|
+
* bracketed prose like `[Note] ...` / `[Note: ...]`); the rest of the tag
|
|
132
|
+
* may carry `:`, `-`, `_`, `.`, digits, and inner capitals (extension
|
|
133
|
+
* subsystem names are camelCase, e.g. `reconcileStaleRuns`), optionally
|
|
134
|
+
* followed by known sub-tags (core/mcp/stderr/stdout/warn/info/error/debug);
|
|
130
135
|
* 2. a Python `warnings.warn(` line (deprecation-warning continuation);
|
|
131
136
|
* 3. a timestamped logging line: `2026-08-15 21:49:02,986 WARNING ...`.
|
|
132
137
|
*
|
|
@@ -137,7 +142,7 @@ export function validateWorkerOutput(role: string, output: string): OutputValida
|
|
|
137
142
|
* (post-execution.ts) applies the two-gate rule (authoritative output
|
|
138
143
|
* sources empty AND artifact log-noise-only) before failing a task.
|
|
139
144
|
*/
|
|
140
|
-
const BRACKET_TAG_LOG_LINE = /^\[[a-z0-9][a-
|
|
145
|
+
const BRACKET_TAG_LOG_LINE = /^\[[a-z0-9][a-zA-Z0-9_.:\-]*\](?:\s*\[(?:core|mcp|stderr|stdout|warn|info|error|debug)\])*(?:\s.*)?$/;
|
|
141
146
|
const PYTHON_WARNING_LINE = /(?:^|\s)warnings\.warn\(/;
|
|
142
147
|
const TIMESTAMPED_LOG_LINE = /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:[.,]\d{1,6})?(?:Z|[+-]\d{2}:?\d{2})?\s+[A-Za-z]+\b/;
|
|
143
148
|
|
|
@@ -70,7 +70,20 @@ export async function ensurePlanApprovalRequested(manifest: TeamRunManifest, tas
|
|
|
70
70
|
const assessTask = tasks.find((task) => task.stepId === "assess" && task.status === "completed");
|
|
71
71
|
// ROADMAP T1.2: for non-adaptive workflows, fall back to the most recent
|
|
72
72
|
// completed read-only (planning) task as the plan reference.
|
|
73
|
-
|
|
73
|
+
// RR-021 WI-4.3e: allocation-free reverse scan — the old
|
|
74
|
+
// [...tasks].reverse().find() copied the whole array per call.
|
|
75
|
+
// (findLast needs an ES2023 lib bump, which breaks tsgo's RequestInfo
|
|
76
|
+
// ambient resolution — see WI-4.3 evidence note.)
|
|
77
|
+
let planTask = assessTask;
|
|
78
|
+
if (!planTask) {
|
|
79
|
+
for (let i = tasks.length - 1; i >= 0; i--) {
|
|
80
|
+
const t = tasks[i];
|
|
81
|
+
if (t?.status === "completed" && !isMutatingTask(t)) {
|
|
82
|
+
planTask = t;
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
74
87
|
const now = new Date().toISOString();
|
|
75
88
|
|
|
76
89
|
// T2/R4 (ADR-4 §6 producer 3 + §8 dual-write): make sure a PlanRecord
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
4
|
+
import { DEFAULT_PATHS } from "../../config/defaults.ts";
|
|
4
5
|
import { appendHookEvent, executeHook } from "../../hooks/registry.ts";
|
|
5
6
|
import type { MetricRegistry } from "../../observability/metric-registry.ts";
|
|
6
|
-
import { withRunLockSync } from "../../state/coordination/locks.ts";
|
|
7
|
+
import { discoverRunLockFiles, sweepStaleLocks, withRunLock, withRunLockSync } from "../../state/coordination/locks.ts";
|
|
7
8
|
import { appendEvent, scanSequence } from "../../state/event-log/event-log.ts";
|
|
8
9
|
import { readActiveRunRegistry, unregisterActiveRun } from "../../state/stores/active-run-registry.ts";
|
|
9
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
loadRunManifestById,
|
|
12
|
+
loadRunManifestByIdAsync,
|
|
13
|
+
saveRunManifest,
|
|
14
|
+
saveRunTasks,
|
|
15
|
+
updateRunStatus,
|
|
16
|
+
} from "../../state/stores/state-store.ts";
|
|
10
17
|
import type { TeamTaskState } from "../../state/types.ts";
|
|
11
18
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
12
19
|
import { projectCrewRoot, userCrewRoot } from "../../utils/paths.ts";
|
|
@@ -17,6 +24,7 @@ import { isWorkerHeartbeatStale } from "../heartbeat/worker-heartbeat.ts";
|
|
|
17
24
|
import { terminateLiveAgentsForRun } from "../live-session/live-agent-manager.ts";
|
|
18
25
|
import type { ManifestCache } from "../manifest-cache.ts";
|
|
19
26
|
import { checkProcessLiveness } from "../process-status.ts";
|
|
27
|
+
import { mapConcurrent } from "../scheduling/parallel-utils.ts";
|
|
20
28
|
import { isIntentionalWait, isPlanApprovalPendingEffective, type ReconcileResult, reconcileStaleRun } from "../stale-reconciler.ts";
|
|
21
29
|
|
|
22
30
|
export interface RecoveryPlan {
|
|
@@ -689,13 +697,12 @@ export function purgeStaleActiveRunIndex(
|
|
|
689
697
|
return { purged, kept };
|
|
690
698
|
}
|
|
691
699
|
|
|
692
|
-
export function reconcileAllStaleRuns(
|
|
700
|
+
export async function reconcileAllStaleRuns(
|
|
693
701
|
cwd: string,
|
|
694
702
|
manifestCache: ManifestCache,
|
|
695
703
|
now = Date.now(),
|
|
696
704
|
currentSessionId?: string,
|
|
697
|
-
): ReconcileResult[] {
|
|
698
|
-
const results: ReconcileResult[] = [];
|
|
705
|
+
): Promise<ReconcileResult[]> {
|
|
699
706
|
// Capture runIds to reconcile BEFORE acquiring locks — avoids TOCTOU between cache iteration and lock acquisition.
|
|
700
707
|
const runIds = manifestCache
|
|
701
708
|
.list(50)
|
|
@@ -707,55 +714,95 @@ export function reconcileAllStaleRuns(
|
|
|
707
714
|
return true;
|
|
708
715
|
})
|
|
709
716
|
.map((m) => m.runId);
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
//
|
|
723
|
-
if (
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
return;
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
if (
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
717
|
+
// RR-021 WI-1.5 (audit C3): serial reconcile → mapConcurrent bound 4.
|
|
718
|
+
// Each runId acquires its OWN run lock file (lockPath() is per-run), so the
|
|
719
|
+
// acquisitions are independent; bound 4 caps fs/CPU contention. Results keep
|
|
720
|
+
// input (snapshot) order — mapConcurrent indexes by item — and per-run
|
|
721
|
+
// semantics (re-read inside lock, plan-approval re-check) are unchanged.
|
|
722
|
+
const perRun = await mapConcurrent(runIds, 4, async (runId): Promise<ReconcileResult[]> => {
|
|
723
|
+
// RR-021 review remediation: per-run error isolation — one corrupt run
|
|
724
|
+
// must not reject the whole sweep (serial baseline threw too, but the
|
|
725
|
+
// SPEC's "independent per-run" intent requires continue-on-error).
|
|
726
|
+
try {
|
|
727
|
+
const cached = manifestCache.get(runId);
|
|
728
|
+
if (!cached) return [];
|
|
729
|
+
const loaded = await loadRunManifestByIdAsync(cwd, runId); // NOTE: best-effort only; concurrent writes may cause inconsistency
|
|
730
|
+
if (!loaded) return [];
|
|
731
|
+
const out: ReconcileResult[] = [];
|
|
732
|
+
// Use lock to prevent race with cancel/status handlers modifying the same run.
|
|
733
|
+
// Same v0.9.26 lock family as the old sync acquisition — interop-safe.
|
|
734
|
+
await withRunLock(loaded.manifest, async () => {
|
|
735
|
+
// Re-read inside lock to get freshest data
|
|
736
|
+
const fresh = await loadRunManifestByIdAsync(cwd, runId); // NOTE: inside withRunLock - consistent read
|
|
737
|
+
if (!fresh || (fresh.manifest.status !== "running" && fresh.manifest.status !== "blocked")) return;
|
|
738
|
+
// Belt-and-suspenders: reconcileStaleRun itself guards this, but the run
|
|
739
|
+
// may have flipped to blocked+plan-approval between cache-list and lock
|
|
740
|
+
// acquisition — re-check the freshest manifest under the lock.
|
|
741
|
+
if (isPlanApprovalPendingEffective(fresh.manifest)) {
|
|
742
|
+
out.push({
|
|
743
|
+
runId,
|
|
744
|
+
verdict: "blocked_awaiting_approval",
|
|
745
|
+
repaired: false,
|
|
746
|
+
detail: "Plan approval is pending; stale reconciliation skipped",
|
|
747
|
+
});
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
const result = reconcileStaleRun(fresh.manifest, fresh.tasks, now);
|
|
751
|
+
if (result.repaired || result.verdict === "result_exists") {
|
|
752
|
+
if (result.repairedTasks) {
|
|
753
|
+
saveRunTasks(fresh.manifest, result.repairedTasks);
|
|
754
|
+
for (const task of result.repairedTasks) {
|
|
755
|
+
try {
|
|
756
|
+
upsertCrewAgent(fresh.manifest, recordFromTask(fresh.manifest, task, "scaffold"));
|
|
757
|
+
} catch {
|
|
758
|
+
/* non-critical */
|
|
759
|
+
}
|
|
741
760
|
}
|
|
742
761
|
}
|
|
762
|
+
updateRunStatus(fresh.manifest, "failed", `Stale run reconciled: ${result.detail}`);
|
|
763
|
+
void terminateLiveAgentsForRun(fresh.manifest.runId, "failed", appendEvent, fresh.manifest.eventsPath).catch((error) =>
|
|
764
|
+
logInternalError("crash-recovery.reconcile.terminate", error, `runId=${fresh.manifest.runId}`, "warn"),
|
|
765
|
+
);
|
|
766
|
+
appendEvent(fresh.manifest.eventsPath, {
|
|
767
|
+
type: "crew.run.reconciled_stale",
|
|
768
|
+
runId,
|
|
769
|
+
message: result.detail,
|
|
770
|
+
data: { verdict: result.verdict },
|
|
771
|
+
});
|
|
772
|
+
}
|
|
773
|
+
if (result.verdict !== "healthy") {
|
|
774
|
+
out.push(result);
|
|
743
775
|
}
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
776
|
+
});
|
|
777
|
+
return out;
|
|
778
|
+
} catch (error) {
|
|
779
|
+
logInternalError("crash-recovery.reconcileStaleRuns", error, `runId=${runId}`, "warn");
|
|
780
|
+
return [];
|
|
781
|
+
}
|
|
782
|
+
});
|
|
783
|
+
const results = perRun.flat();
|
|
784
|
+
// US-002 (2026-09-22): structured stale-lock sweep. Runs after reconciliation
|
|
785
|
+
// so locks whose holder died (kill -9 / crash) do not linger. The sweep is
|
|
786
|
+
// strictly safer than acquire-time steal (stale AND holder-dead only), so a
|
|
787
|
+
// live holder is never disturbed. Best-effort.
|
|
788
|
+
try {
|
|
789
|
+
const runsRoots = [
|
|
790
|
+
path.join(projectCrewRoot(cwd), DEFAULT_PATHS.state.runsSubdir),
|
|
791
|
+
path.join(userCrewRoot(), DEFAULT_PATHS.state.runsSubdir),
|
|
792
|
+
];
|
|
793
|
+
for (const runsRoot of runsRoots) {
|
|
794
|
+
const { removed } = sweepStaleLocks(discoverRunLockFiles(runsRoot));
|
|
795
|
+
if (removed.length > 0) {
|
|
796
|
+
logInternalError(
|
|
797
|
+
"crash-recovery.sweep-stale-locks",
|
|
798
|
+
new Error(`swept ${removed.length} stale run lock(s)`),
|
|
799
|
+
removed.join(", "),
|
|
800
|
+
"warn",
|
|
747
801
|
);
|
|
748
|
-
appendEvent(fresh.manifest.eventsPath, {
|
|
749
|
-
type: "crew.run.reconciled_stale",
|
|
750
|
-
runId,
|
|
751
|
-
message: result.detail,
|
|
752
|
-
data: { verdict: result.verdict },
|
|
753
|
-
});
|
|
754
|
-
}
|
|
755
|
-
if (result.verdict !== "healthy") {
|
|
756
|
-
results.push(result);
|
|
757
802
|
}
|
|
758
|
-
}
|
|
803
|
+
}
|
|
804
|
+
} catch (error) {
|
|
805
|
+
logInternalError("crash-recovery.sweep-stale-locks", error);
|
|
759
806
|
}
|
|
760
807
|
return results;
|
|
761
808
|
}
|
|
@@ -72,7 +72,18 @@ export interface WorkerSpawnInput extends ChildPiRunInput {
|
|
|
72
72
|
export async function runWorker(input: WorkerSpawnInput): Promise<ChildPiRunResult> {
|
|
73
73
|
const { cap = true, ...childPiInput } = input;
|
|
74
74
|
if (cap) {
|
|
75
|
-
|
|
75
|
+
// RR-014 / F15: thread the caller's signal into the slot WAIT itself.
|
|
76
|
+
// Previously the signal lived only inside childPiInput and was read by
|
|
77
|
+
// runChildPi AFTER the acquire resolved — a cancelled task queued on a
|
|
78
|
+
// full pool stayed pending until the slot holder finished (and delayed
|
|
79
|
+
// every drainPendingUnits awaiting it). Two consistent outcomes:
|
|
80
|
+
// - abort wins the race → acquire rejects (SemaphoreAbortedError), no
|
|
81
|
+
// slot taken, fn never runs;
|
|
82
|
+
// - grant wins the race → acquire resolves, runChildPi's pre-spawn
|
|
83
|
+
// guard (child-pi-spawn.ts B5) sees signal.aborted and returns
|
|
84
|
+
// kind "aborted" WITHOUT spawning, and withWorkerSlot's finally
|
|
85
|
+
// releases the slot within a microtask.
|
|
86
|
+
return withWorkerSlot(() => runChildPi(childPiInput), childPiInput.signal);
|
|
76
87
|
}
|
|
77
88
|
return runChildPi(childPiInput);
|
|
78
89
|
}
|
|
@@ -64,11 +64,16 @@ export function __test_resetCap(testCapacity: number): void {
|
|
|
64
64
|
* Acquire a global worker slot. Resolves immediately if under cap, else queues
|
|
65
65
|
* (FIFO) until a slot frees. MUST be paired with releaseWorkerSlot().
|
|
66
66
|
*
|
|
67
|
+
* RR-014 / F15: an optional AbortSignal makes the WAIT itself cancellable — a
|
|
68
|
+
* waiter whose signal fires is removed from the queue eagerly and the acquire
|
|
69
|
+
* REJECTS (SemaphoreAbortedError) without consuming a slot. Without a signal
|
|
70
|
+
* the semantics are unchanged.
|
|
71
|
+
*
|
|
67
72
|
* Used to bound WORKER spawns only. Do NOT route the goal-judge through this
|
|
68
73
|
* (see the RFC MAJ#3 rationale in the module header).
|
|
69
74
|
*/
|
|
70
|
-
export async function acquireWorkerSlot(): Promise<void> {
|
|
71
|
-
await semaphore.acquire();
|
|
75
|
+
export async function acquireWorkerSlot(signal?: AbortSignal): Promise<void> {
|
|
76
|
+
await semaphore.acquire(signal);
|
|
72
77
|
}
|
|
73
78
|
|
|
74
79
|
/**
|
|
@@ -82,13 +87,15 @@ export function releaseWorkerSlot(): void {
|
|
|
82
87
|
/**
|
|
83
88
|
* Convenience: acquire a worker slot, run `fn`, and release on completion OR
|
|
84
89
|
* throw. The slot is ALWAYS released — including when `fn` rejects — so a
|
|
85
|
-
* throwing worker never leaks a slot (deadlock prevention).
|
|
90
|
+
* throwing worker never leaks a slot (deadlock prevention). If `signal` is
|
|
91
|
+
* provided and fires while WAITING for a slot, the acquire rejects and `fn`
|
|
92
|
+
* never runs (no slot was taken, so there is nothing to release).
|
|
86
93
|
*
|
|
87
94
|
* Example:
|
|
88
|
-
* const result = await withWorkerSlot(() => runChildPi(...));
|
|
95
|
+
* const result = await withWorkerSlot(() => runChildPi(...), input.signal);
|
|
89
96
|
*/
|
|
90
|
-
export async function withWorkerSlot<T>(fn: () => Promise<T
|
|
91
|
-
await acquireWorkerSlot();
|
|
97
|
+
export async function withWorkerSlot<T>(fn: () => Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
98
|
+
await acquireWorkerSlot(signal);
|
|
92
99
|
try {
|
|
93
100
|
return await fn();
|
|
94
101
|
} finally {
|
|
@@ -5,6 +5,7 @@ import { writeArtifact } from "../../state/stores/artifact-store.ts";
|
|
|
5
5
|
import { loadRunManifestById, saveRunTasksAsync, updateRunStatus } from "../../state/stores/state-store.ts";
|
|
6
6
|
import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
|
|
7
7
|
import type { WorkflowStep } from "../../workflows/workflow-config.ts";
|
|
8
|
+
import { appendDeadletter } from "../deadletter.ts";
|
|
8
9
|
import { createWorkerHeartbeat, touchWorkerHeartbeat } from "../heartbeat/worker-heartbeat.ts";
|
|
9
10
|
import type { CrewRuntimeMode } from "../model/runtime-resolver.ts";
|
|
10
11
|
import type { RetryPolicy } from "../recovery/retry-executor.ts";
|
|
@@ -196,7 +197,32 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
196
197
|
}
|
|
197
198
|
};
|
|
198
199
|
try {
|
|
199
|
-
const result = useRetry
|
|
200
|
+
const result = useRetry
|
|
201
|
+
? await executeWithRetry(runOnce, policy, {
|
|
202
|
+
signal,
|
|
203
|
+
// US-003 (2026-09-22): exhausted retries on the COALESCED path were never
|
|
204
|
+
// deadlettered — only the singleton path (dispatch-batch) wired
|
|
205
|
+
// onRetryGivenUp, so an M6 group that burned its whole retry budget left
|
|
206
|
+
// no post-mortem record once the run dir was pruned. The group shares ONE
|
|
207
|
+
// worker: the entry is recorded against the group's first task and names
|
|
208
|
+
// the whole group in the message. Aborts are NOT exhaustion — skip them
|
|
209
|
+
// (executeWithRetry also fires this hook on the cancelled exit).
|
|
210
|
+
onRetryGivenUp: (attempts, error) => {
|
|
211
|
+
if (signal?.aborted) return;
|
|
212
|
+
appendDeadletter(manifest, {
|
|
213
|
+
runId: manifest.runId,
|
|
214
|
+
taskId: firstTask.id,
|
|
215
|
+
reason: "max-retries",
|
|
216
|
+
attempts,
|
|
217
|
+
lastError: `coalesced group (${taskIds.length} task(s): ${taskIds.join(", ")}) — ${error.message}`,
|
|
218
|
+
timestamp: new Date().toISOString(),
|
|
219
|
+
agent: agent.name,
|
|
220
|
+
role: firstTask.role,
|
|
221
|
+
runStatus: manifest.status,
|
|
222
|
+
});
|
|
223
|
+
},
|
|
224
|
+
})
|
|
225
|
+
: await runOnce();
|
|
200
226
|
rawOutput = result.rawFinalText ?? result.stdout ?? "";
|
|
201
227
|
// RT-5 #1/#2: distinguish cancel from failure. Cancel = run-level
|
|
202
228
|
// signal aborted OR the child reports cooperative cancellation.
|
|
@@ -24,6 +24,7 @@ export type ScheduleChangeEvent =
|
|
|
24
24
|
| { type: "removed"; jobId: string; spawnedRunIds?: string[] }
|
|
25
25
|
| { type: "updated"; job: ScheduledJob }
|
|
26
26
|
| { type: "fired"; jobId: string; agentId: string; name: string }
|
|
27
|
+
| { type: "skipped"; jobId: string; reason: string }
|
|
27
28
|
| { type: "error"; jobId: string; error: string };
|
|
28
29
|
|
|
29
30
|
export interface CrewSchedulerOptions {
|
|
@@ -152,19 +153,29 @@ export class CrewScheduler {
|
|
|
152
153
|
private arm(job: ScheduledJob): void {
|
|
153
154
|
if (this.timers.has(job.id)) return;
|
|
154
155
|
if (job.scheduleType === "interval" && job.intervalMs) {
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
156
|
+
// 2026-09-22 spawn-storm root cause: this branch used
|
|
157
|
+
// setInterval(fire, intervalMs). Node timers are 32-bit — a LEGAL long
|
|
158
|
+
// interval (e.g. 90d = 7,776,000,000ms > 2^31-1) overflows and Node
|
|
159
|
+
// silently sets the delay to 1ms (TimeoutOverflowWarning): the job then
|
|
160
|
+
// fired every millisecond, and each
|
|
161
|
+
// tick dispatched another run (measured live: ~104 garbage runs + 50+
|
|
162
|
+
// node processes from ONE registered 90-day interval job). armCron()
|
|
163
|
+
// already clamps its hops below the same ceiling — the interval branch
|
|
164
|
+
// was missed. Reuse the identical chained-hop treatment: schedule toward
|
|
165
|
+
// now+intervalMs, clamped; cronTick fires on arrival (its
|
|
166
|
+
// advanceCronNextRun is a cron-only no-op here), and fire()'s internal
|
|
167
|
+
// update() → disarm→arm lifecycle re-arms the next interval.
|
|
168
|
+
const targetMs = this.now().getTime() + job.intervalMs;
|
|
169
|
+
this.setCronTimeout(job.id, targetMs);
|
|
158
170
|
} else if (job.scheduleType === "once") {
|
|
159
171
|
const target = new Date(job.schedule).getTime();
|
|
160
172
|
const delay = target - this.now().getTime();
|
|
161
173
|
if (delay > 0) {
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
this.timers.set(job.id, t);
|
|
174
|
+
// Same 32-bit ceiling as the interval branch above: a once job armed
|
|
175
|
+
// > 2^31-1 ms out (e.g. "+30d" — a LEGAL relative spec — or a far ISO
|
|
176
|
+
// timestamp) overflowed setTimeout to a 1ms PREMATURE fire. Chained hops
|
|
177
|
+
// instead; cronTick self-disables once-jobs on arrival (below).
|
|
178
|
+
this.setCronTimeout(job.id, target);
|
|
168
179
|
} else {
|
|
169
180
|
this.update(job.id, { enabled: false, lastStatus: "error" });
|
|
170
181
|
this.emit?.({
|
|
@@ -219,6 +230,9 @@ export class CrewScheduler {
|
|
|
219
230
|
// hop that just fired is dead, so no timer doubles up.
|
|
220
231
|
this.fire(jobId);
|
|
221
232
|
this.advanceCronNextRun(jobId);
|
|
233
|
+
// once-semantics (was inline in arm()'s removed setTimeout): consume the
|
|
234
|
+
// job after its single arrival fire so a re-arm cannot fire it again.
|
|
235
|
+
if (job.scheduleType === "once") this.update(jobId, { enabled: false });
|
|
222
236
|
}
|
|
223
237
|
|
|
224
238
|
/** After a cron fire, advance the persisted nextRun to the next occurrence
|
|
@@ -265,6 +279,18 @@ export class CrewScheduler {
|
|
|
265
279
|
const job = this.jobs.get(id);
|
|
266
280
|
if (!job || !this.executor) return;
|
|
267
281
|
if (!job.enabled && !force) return;
|
|
282
|
+
// In-flight guard (2026-09-22, storm follow-up): the executor dispatches
|
|
283
|
+
// asynchronously and resets lastStatus (success/error) only on completion,
|
|
284
|
+
// so `running` marks the in-flight window. Without backpressure here, any
|
|
285
|
+
// interval shorter than the run duration stacks one overlapping run per
|
|
286
|
+
// tick (the 1ms overflow loop was the extreme case; fixed below/above).
|
|
287
|
+
// Skips emit a "skipped" event (visible to pi.events consumers; the toast
|
|
288
|
+
// bridge stays silent — a slow executor + short interval would spam).
|
|
289
|
+
// `force` (run-now) is the explicit escape hatch.
|
|
290
|
+
if (job.lastStatus === "running" && !force) {
|
|
291
|
+
this.emit?.({ type: "skipped", jobId: id, reason: "previous dispatch still in flight" });
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
268
294
|
this.update(id, { lastStatus: "running" });
|
|
269
295
|
let agentId: string;
|
|
270
296
|
try {
|
|
@@ -297,10 +323,15 @@ export class CrewScheduler {
|
|
|
297
323
|
normalized: new Date(Date.now() + ms).toISOString(),
|
|
298
324
|
};
|
|
299
325
|
}
|
|
300
|
-
// Interval: 5m
|
|
301
|
-
|
|
326
|
+
// Interval: 5m — "ms" MUST be accepted: handle-schedule builds
|
|
327
|
+
// `${params.interval}ms` from the schema-level numeric `interval` param
|
|
328
|
+
// (ms number), so without an ms unit here every interval schedule died in
|
|
329
|
+
// the parser ("Invalid schedule …") — found live 2026-09-21 when
|
|
330
|
+
// `team action='schedule' interval=3600000` was impossible to satisfy.
|
|
331
|
+
const ivl = trimmed.match(/^(\d+)(ms|s|m|h|d)$/);
|
|
302
332
|
if (ivl) {
|
|
303
|
-
const ms =
|
|
333
|
+
const ms =
|
|
334
|
+
parseInt(ivl[1], 10) * { ms: 1, s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 }[ivl[2] as "ms" | "s" | "m" | "h" | "d"];
|
|
304
335
|
return { type: "interval", intervalMs: ms, normalized: trimmed };
|
|
305
336
|
}
|
|
306
337
|
// ISO timestamp
|
|
@@ -329,13 +360,18 @@ function parseIntervalMs(s: string): number | undefined {
|
|
|
329
360
|
let ms = 0;
|
|
330
361
|
let remaining = s;
|
|
331
362
|
const unitMs: Record<string, number> = {
|
|
363
|
+
// "ms" MUST be first in the regex alternation below and present here:
|
|
364
|
+
// handle-schedule builds `${params.interval}ms` from the numeric interval
|
|
365
|
+
// param, so without an ms unit every interval schedule failed to parse
|
|
366
|
+
// (found live 2026-09-21 — `interval=3600000` was unsatisfiable).
|
|
367
|
+
ms: 1,
|
|
332
368
|
s: 1000,
|
|
333
369
|
m: 60_000,
|
|
334
370
|
h: 3_600_000,
|
|
335
371
|
d: 86_400_000,
|
|
336
372
|
};
|
|
337
373
|
while (remaining.length > 0) {
|
|
338
|
-
const m = remaining.match(/^(\d+)(s|m|h|d)/);
|
|
374
|
+
const m = remaining.match(/^(\d+)(ms|s|m|h|d)/);
|
|
339
375
|
if (!m) return undefined;
|
|
340
376
|
ms += parseInt(m[1], 10) * unitMs[m[2]];
|
|
341
377
|
remaining = remaining.slice(m[0].length);
|