pi-crew 0.1.37 → 0.1.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +27 -0
- package/README.md +5 -0
- package/agents/analyst.md +11 -11
- package/agents/critic.md +11 -11
- package/agents/executor.md +11 -11
- package/agents/explorer.md +11 -11
- package/agents/planner.md +11 -11
- package/agents/reviewer.md +11 -11
- package/agents/security-reviewer.md +11 -11
- package/agents/test-engineer.md +11 -11
- package/agents/verifier.md +11 -11
- package/agents/writer.md +11 -11
- package/docs/refactor-tasks-phase3.md +394 -394
- package/docs/refactor-tasks-phase4.md +564 -564
- package/docs/refactor-tasks-phase5.md +402 -402
- package/docs/refactor-tasks-phase6.md +662 -662
- package/docs/research-extension-examples.md +297 -297
- package/docs/research-extension-system.md +324 -324
- package/docs/research-optimization-plan.md +548 -548
- package/docs/research-pi-coding-agent.md +357 -357
- package/docs/research-source-pi-crew-reference.md +174 -174
- package/docs/resource-formats.md +10 -8
- package/docs/runtime-flow.md +148 -148
- package/docs/source-runtime-refactor-map.md +83 -83
- package/docs/usage.md +6 -0
- package/index.ts +6 -6
- package/package.json +3 -3
- package/schema.json +2 -2
- package/src/agents/agent-serializer.ts +34 -34
- package/src/config/config.ts +8 -4
- package/src/extension/cross-extension-rpc.ts +82 -82
- package/src/extension/import-index.ts +18 -2
- package/src/extension/register.ts +11 -1
- package/src/extension/registration/compaction-guard.ts +125 -125
- package/src/extension/registration/subagent-helpers.ts +30 -6
- package/src/extension/registration/subagent-tools.ts +8 -3
- package/src/extension/result-watcher.ts +98 -98
- package/src/extension/run-import.ts +12 -2
- package/src/extension/run-index.ts +12 -2
- package/src/extension/run-maintenance.ts +24 -24
- package/src/extension/team-tool/api.ts +54 -14
- package/src/extension/team-tool/cancel.ts +31 -31
- package/src/extension/team-tool/doctor.ts +179 -179
- package/src/extension/team-tool/inspect.ts +41 -41
- package/src/extension/team-tool/lifecycle-actions.ts +79 -79
- package/src/extension/team-tool/plan.ts +19 -19
- package/src/extension/team-tool/status.ts +73 -73
- package/src/observability/correlation.ts +35 -35
- package/src/observability/event-to-metric.ts +54 -54
- package/src/observability/exporters/adapter.ts +24 -24
- package/src/observability/exporters/otlp-exporter.ts +65 -65
- package/src/observability/exporters/prometheus-exporter.ts +47 -47
- package/src/observability/metric-registry.ts +72 -72
- package/src/observability/metric-retention.ts +46 -46
- package/src/observability/metric-sink.ts +51 -51
- package/src/observability/metrics-primitives.ts +166 -166
- package/src/prompt/prompt-runtime.ts +68 -68
- package/src/runtime/agent-control.ts +64 -64
- package/src/runtime/agent-memory.ts +72 -72
- package/src/runtime/agent-observability.ts +114 -113
- package/src/runtime/async-marker.ts +26 -26
- package/src/runtime/background-runner.ts +53 -53
- package/src/runtime/crash-recovery.ts +56 -56
- package/src/runtime/crew-agent-records.ts +54 -9
- package/src/runtime/crew-agent-runtime.ts +58 -58
- package/src/runtime/deadletter.ts +36 -36
- package/src/runtime/direct-run.ts +35 -35
- package/src/runtime/foreground-control.ts +82 -82
- package/src/runtime/green-contract.ts +46 -46
- package/src/runtime/group-join.ts +88 -88
- package/src/runtime/heartbeat-gradient.ts +28 -28
- package/src/runtime/heartbeat-watcher.ts +80 -80
- package/src/runtime/live-agent-control.ts +87 -78
- package/src/runtime/live-agent-manager.ts +85 -85
- package/src/runtime/live-control-realtime.ts +36 -36
- package/src/runtime/live-session-runtime.ts +299 -299
- package/src/runtime/manifest-cache.ts +248 -212
- package/src/runtime/model-fallback.ts +261 -261
- package/src/runtime/parallel-research.ts +44 -44
- package/src/runtime/parallel-utils.ts +99 -99
- package/src/runtime/pi-json-output.ts +111 -111
- package/src/runtime/policy-engine.ts +78 -78
- package/src/runtime/post-exit-stdio-guard.ts +86 -86
- package/src/runtime/process-status.ts +56 -56
- package/src/runtime/progress-event-coalescer.ts +43 -43
- package/src/runtime/recovery-recipes.ts +74 -74
- package/src/runtime/retry-executor.ts +59 -59
- package/src/runtime/role-permission.ts +39 -39
- package/src/runtime/session-usage.ts +79 -79
- package/src/runtime/sidechain-output.ts +28 -28
- package/src/runtime/subagent-manager.ts +80 -12
- package/src/runtime/task-display.ts +38 -38
- package/src/runtime/task-output-context.ts +127 -106
- package/src/runtime/task-runner/live-executor.ts +98 -98
- package/src/runtime/task-runner/progress.ts +111 -111
- package/src/runtime/task-runner/result-utils.ts +14 -14
- package/src/runtime/task-runner/state-helpers.ts +22 -22
- package/src/runtime/team-runner.ts +1 -1
- package/src/runtime/worker-heartbeat.ts +21 -21
- package/src/runtime/worker-startup.ts +57 -57
- package/src/schema/config-schema.ts +21 -21
- package/src/schema/team-tool-schema.ts +100 -100
- package/src/state/artifact-store.ts +122 -108
- package/src/state/contracts.ts +105 -105
- package/src/state/jsonl-writer.ts +77 -77
- package/src/state/mailbox.ts +67 -22
- package/src/state/state-store.ts +36 -5
- package/src/state/task-claims.ts +42 -42
- package/src/state/usage.ts +29 -29
- package/src/subagents/async-entry.ts +1 -1
- package/src/subagents/index.ts +3 -3
- package/src/subagents/live/control.ts +1 -1
- package/src/subagents/live/manager.ts +1 -1
- package/src/subagents/live/realtime.ts +1 -1
- package/src/subagents/live/session-runtime.ts +1 -1
- package/src/subagents/manager.ts +1 -1
- package/src/subagents/spawn.ts +1 -1
- package/src/teams/discover-teams.ts +27 -5
- package/src/teams/team-serializer.ts +38 -36
- package/src/types/diff.d.ts +18 -18
- package/src/ui/crew-footer.ts +101 -101
- package/src/ui/crew-select-list.ts +111 -111
- package/src/ui/dashboard-panes/metrics-pane.ts +34 -34
- package/src/ui/dynamic-border.ts +25 -25
- package/src/ui/layout-primitives.ts +106 -106
- package/src/ui/loaders.ts +158 -158
- package/src/ui/mascot.ts +441 -441
- package/src/ui/render-diff.ts +119 -119
- package/src/ui/run-dashboard.ts +5 -2
- package/src/ui/run-snapshot-cache.ts +19 -8
- package/src/ui/spinner.ts +17 -17
- package/src/ui/status-colors.ts +54 -54
- package/src/ui/syntax-highlight.ts +116 -116
- package/src/ui/transcript-viewer.ts +15 -1
- package/src/utils/completion-dedupe.ts +63 -63
- package/src/utils/file-coalescer.ts +84 -84
- package/src/utils/frontmatter.ts +36 -36
- package/src/utils/fs-watch.ts +31 -31
- package/src/utils/git.ts +262 -262
- package/src/utils/ids.ts +12 -12
- package/src/utils/names.ts +26 -26
- package/src/utils/paths.ts +3 -2
- package/src/utils/safe-paths.ts +34 -0
- package/src/utils/sleep.ts +32 -32
- package/src/utils/timings.ts +31 -31
- package/src/utils/visual.ts +159 -159
- package/src/workflows/discover-workflows.ts +30 -3
- package/src/workflows/validate-workflow.ts +40 -40
- package/src/worktree/branch-freshness.ts +45 -45
- package/teams/default.team.md +12 -12
- package/teams/fast-fix.team.md +11 -11
- package/teams/implementation.team.md +18 -18
- package/teams/parallel-research.team.md +14 -14
- package/teams/research.team.md +11 -11
- package/teams/review.team.md +12 -12
- package/workflows/default.workflow.md +29 -29
- package/workflows/fast-fix.workflow.md +22 -22
- package/workflows/implementation.workflow.md +38 -38
- package/workflows/parallel-research.workflow.md +46 -46
- package/workflows/research.workflow.md +22 -22
- package/workflows/review.workflow.md +30 -30
|
@@ -1,86 +1,86 @@
|
|
|
1
|
-
import type { ChildProcess } from "node:child_process";
|
|
2
|
-
|
|
3
|
-
interface PostExitStdioGuardOptions {
|
|
4
|
-
idleMs: number;
|
|
5
|
-
hardMs: number;
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
export interface ChildWithPipedStdio {
|
|
9
|
-
stdout: ChildProcess["stdout"];
|
|
10
|
-
stderr: ChildProcess["stderr"];
|
|
11
|
-
on: ChildProcess["on"];
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export interface ChildWithKill {
|
|
15
|
-
kill(signal?: NodeJS.Signals | number): boolean;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
export function trySignalChild(child: ChildWithKill, signal: NodeJS.Signals): boolean {
|
|
19
|
-
try {
|
|
20
|
-
return child.kill(signal);
|
|
21
|
-
} catch {
|
|
22
|
-
return false;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
export function attachPostExitStdioGuard(child: ChildWithPipedStdio, options: PostExitStdioGuardOptions): () => void {
|
|
27
|
-
const { idleMs, hardMs } = options;
|
|
28
|
-
let exited = false;
|
|
29
|
-
let stdoutEnded = false;
|
|
30
|
-
let stderrEnded = false;
|
|
31
|
-
let idleTimer: ReturnType<typeof setTimeout> | undefined;
|
|
32
|
-
let hardTimer: ReturnType<typeof setTimeout> | undefined;
|
|
33
|
-
|
|
34
|
-
const destroyUnendedStdio = (): void => {
|
|
35
|
-
if (!stdoutEnded) {
|
|
36
|
-
try {
|
|
37
|
-
child.stdout?.destroy();
|
|
38
|
-
} catch {}
|
|
39
|
-
}
|
|
40
|
-
if (!stderrEnded) {
|
|
41
|
-
try {
|
|
42
|
-
child.stderr?.destroy();
|
|
43
|
-
} catch {}
|
|
44
|
-
}
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
const clearTimers = (): void => {
|
|
48
|
-
if (idleTimer) {
|
|
49
|
-
clearTimeout(idleTimer);
|
|
50
|
-
idleTimer = undefined;
|
|
51
|
-
}
|
|
52
|
-
if (hardTimer) {
|
|
53
|
-
clearTimeout(hardTimer);
|
|
54
|
-
hardTimer = undefined;
|
|
55
|
-
}
|
|
56
|
-
};
|
|
57
|
-
|
|
58
|
-
const armIdleTimer = () => {
|
|
59
|
-
if (!exited) return;
|
|
60
|
-
if (idleTimer) clearTimeout(idleTimer);
|
|
61
|
-
idleTimer = setTimeout(destroyUnendedStdio, idleMs);
|
|
62
|
-
idleTimer.unref?.();
|
|
63
|
-
};
|
|
64
|
-
|
|
65
|
-
child.stdout?.on("data", armIdleTimer);
|
|
66
|
-
child.stderr?.on("data", armIdleTimer);
|
|
67
|
-
child.stdout?.on("end", () => {
|
|
68
|
-
stdoutEnded = true;
|
|
69
|
-
if (stdoutEnded && stderrEnded) clearTimers();
|
|
70
|
-
});
|
|
71
|
-
child.stderr?.on("end", () => {
|
|
72
|
-
stderrEnded = true;
|
|
73
|
-
if (stdoutEnded && stderrEnded) clearTimers();
|
|
74
|
-
});
|
|
75
|
-
child.on("exit", () => {
|
|
76
|
-
exited = true;
|
|
77
|
-
armIdleTimer();
|
|
78
|
-
if (hardTimer) return;
|
|
79
|
-
hardTimer = setTimeout(destroyUnendedStdio, hardMs);
|
|
80
|
-
hardTimer.unref?.();
|
|
81
|
-
});
|
|
82
|
-
child.on("close", clearTimers);
|
|
83
|
-
child.on("error", clearTimers);
|
|
84
|
-
|
|
85
|
-
return clearTimers;
|
|
86
|
-
}
|
|
1
|
+
import type { ChildProcess } from "node:child_process";
|
|
2
|
+
|
|
3
|
+
interface PostExitStdioGuardOptions {
|
|
4
|
+
idleMs: number;
|
|
5
|
+
hardMs: number;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface ChildWithPipedStdio {
|
|
9
|
+
stdout: ChildProcess["stdout"];
|
|
10
|
+
stderr: ChildProcess["stderr"];
|
|
11
|
+
on: ChildProcess["on"];
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface ChildWithKill {
|
|
15
|
+
kill(signal?: NodeJS.Signals | number): boolean;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function trySignalChild(child: ChildWithKill, signal: NodeJS.Signals): boolean {
|
|
19
|
+
try {
|
|
20
|
+
return child.kill(signal);
|
|
21
|
+
} catch {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function attachPostExitStdioGuard(child: ChildWithPipedStdio, options: PostExitStdioGuardOptions): () => void {
|
|
27
|
+
const { idleMs, hardMs } = options;
|
|
28
|
+
let exited = false;
|
|
29
|
+
let stdoutEnded = false;
|
|
30
|
+
let stderrEnded = false;
|
|
31
|
+
let idleTimer: ReturnType<typeof setTimeout> | undefined;
|
|
32
|
+
let hardTimer: ReturnType<typeof setTimeout> | undefined;
|
|
33
|
+
|
|
34
|
+
const destroyUnendedStdio = (): void => {
|
|
35
|
+
if (!stdoutEnded) {
|
|
36
|
+
try {
|
|
37
|
+
child.stdout?.destroy();
|
|
38
|
+
} catch {}
|
|
39
|
+
}
|
|
40
|
+
if (!stderrEnded) {
|
|
41
|
+
try {
|
|
42
|
+
child.stderr?.destroy();
|
|
43
|
+
} catch {}
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const clearTimers = (): void => {
|
|
48
|
+
if (idleTimer) {
|
|
49
|
+
clearTimeout(idleTimer);
|
|
50
|
+
idleTimer = undefined;
|
|
51
|
+
}
|
|
52
|
+
if (hardTimer) {
|
|
53
|
+
clearTimeout(hardTimer);
|
|
54
|
+
hardTimer = undefined;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const armIdleTimer = () => {
|
|
59
|
+
if (!exited) return;
|
|
60
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
61
|
+
idleTimer = setTimeout(destroyUnendedStdio, idleMs);
|
|
62
|
+
idleTimer.unref?.();
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
child.stdout?.on("data", armIdleTimer);
|
|
66
|
+
child.stderr?.on("data", armIdleTimer);
|
|
67
|
+
child.stdout?.on("end", () => {
|
|
68
|
+
stdoutEnded = true;
|
|
69
|
+
if (stdoutEnded && stderrEnded) clearTimers();
|
|
70
|
+
});
|
|
71
|
+
child.stderr?.on("end", () => {
|
|
72
|
+
stderrEnded = true;
|
|
73
|
+
if (stdoutEnded && stderrEnded) clearTimers();
|
|
74
|
+
});
|
|
75
|
+
child.on("exit", () => {
|
|
76
|
+
exited = true;
|
|
77
|
+
armIdleTimer();
|
|
78
|
+
if (hardTimer) return;
|
|
79
|
+
hardTimer = setTimeout(destroyUnendedStdio, hardMs);
|
|
80
|
+
hardTimer.unref?.();
|
|
81
|
+
});
|
|
82
|
+
child.on("close", clearTimers);
|
|
83
|
+
child.on("error", clearTimers);
|
|
84
|
+
|
|
85
|
+
return clearTimers;
|
|
86
|
+
}
|
|
@@ -1,56 +1,56 @@
|
|
|
1
|
-
import type { CrewAgentRecord } from "./crew-agent-runtime.ts";
|
|
2
|
-
import type { TeamRunManifest } from "../state/types.ts";
|
|
3
|
-
export { hasAsyncStartMarker } from "./async-marker.ts";
|
|
4
|
-
|
|
5
|
-
export interface ProcessLiveness {
|
|
6
|
-
pid?: number;
|
|
7
|
-
alive: boolean;
|
|
8
|
-
detail: string;
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
const ORPHANED_ACTIVE_RUN_MS = 10 * 60 * 1000;
|
|
12
|
-
|
|
13
|
-
export function checkProcessLiveness(pid: number | undefined): ProcessLiveness {
|
|
14
|
-
if (pid === undefined || !Number.isInteger(pid) || pid <= 0) {
|
|
15
|
-
return { pid, alive: false, detail: "no pid recorded" };
|
|
16
|
-
}
|
|
17
|
-
try {
|
|
18
|
-
process.kill(pid, 0);
|
|
19
|
-
return { pid, alive: true, detail: "process is alive" };
|
|
20
|
-
} catch (error) {
|
|
21
|
-
const nodeError = error as NodeJS.ErrnoException;
|
|
22
|
-
if (nodeError.code === "EPERM") return { pid, alive: true, detail: "process exists but permission is denied" };
|
|
23
|
-
if (nodeError.code === "ESRCH") return { pid, alive: false, detail: "process does not exist" };
|
|
24
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
25
|
-
return { pid, alive: false, detail: message };
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export function isActiveRunStatus(status: string): boolean {
|
|
30
|
-
return status === "queued" || status === "planning" || status === "running";
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export function isLikelyOrphanedActiveRun(run: TeamRunManifest, agents: CrewAgentRecord[] = [], now = Date.now(), staleMs = ORPHANED_ACTIVE_RUN_MS): boolean {
|
|
34
|
-
if (!isActiveRunStatus(run.status)) return false;
|
|
35
|
-
if (run.async?.pid !== undefined) return false;
|
|
36
|
-
const updatedAt = new Date(run.updatedAt).getTime();
|
|
37
|
-
if (!Number.isFinite(updatedAt) || now - updatedAt < staleMs) return false;
|
|
38
|
-
if (agents.length === 0) return run.summary === "Creating workflow prompts and placeholder results.";
|
|
39
|
-
return agents.every((agent) => agent.status === "queued" && !agent.completedAt && !agent.progress);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
function hasDurableActiveAgentEvidence(agent: CrewAgentRecord): boolean {
|
|
43
|
-
if (agent.status !== "running" && agent.status !== "queued") return false;
|
|
44
|
-
return Boolean(agent.statusPath || agent.eventsPath || agent.outputPath || agent.progress || agent.toolUses || agent.jsonEvents);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
export function hasStaleAsyncProcess(run: TeamRunManifest): boolean {
|
|
48
|
-
if (!isActiveRunStatus(run.status) || !run.async) return false;
|
|
49
|
-
return !checkProcessLiveness(run.async.pid).alive;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export function isDisplayActiveRun(run: TeamRunManifest, agents: CrewAgentRecord[] = [], now = Date.now()): boolean {
|
|
53
|
-
if (!isActiveRunStatus(run.status) || hasStaleAsyncProcess(run) || isLikelyOrphanedActiveRun(run, agents, now)) return false;
|
|
54
|
-
if (agents.length === 0) return true;
|
|
55
|
-
return agents.some(hasDurableActiveAgentEvidence);
|
|
56
|
-
}
|
|
1
|
+
import type { CrewAgentRecord } from "./crew-agent-runtime.ts";
|
|
2
|
+
import type { TeamRunManifest } from "../state/types.ts";
|
|
3
|
+
export { hasAsyncStartMarker } from "./async-marker.ts";
|
|
4
|
+
|
|
5
|
+
export interface ProcessLiveness {
|
|
6
|
+
pid?: number;
|
|
7
|
+
alive: boolean;
|
|
8
|
+
detail: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
const ORPHANED_ACTIVE_RUN_MS = 10 * 60 * 1000;
|
|
12
|
+
|
|
13
|
+
export function checkProcessLiveness(pid: number | undefined): ProcessLiveness {
|
|
14
|
+
if (pid === undefined || !Number.isInteger(pid) || pid <= 0) {
|
|
15
|
+
return { pid, alive: false, detail: "no pid recorded" };
|
|
16
|
+
}
|
|
17
|
+
try {
|
|
18
|
+
process.kill(pid, 0);
|
|
19
|
+
return { pid, alive: true, detail: "process is alive" };
|
|
20
|
+
} catch (error) {
|
|
21
|
+
const nodeError = error as NodeJS.ErrnoException;
|
|
22
|
+
if (nodeError.code === "EPERM") return { pid, alive: true, detail: "process exists but permission is denied" };
|
|
23
|
+
if (nodeError.code === "ESRCH") return { pid, alive: false, detail: "process does not exist" };
|
|
24
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
25
|
+
return { pid, alive: false, detail: message };
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function isActiveRunStatus(status: string): boolean {
|
|
30
|
+
return status === "queued" || status === "planning" || status === "running";
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function isLikelyOrphanedActiveRun(run: TeamRunManifest, agents: CrewAgentRecord[] = [], now = Date.now(), staleMs = ORPHANED_ACTIVE_RUN_MS): boolean {
|
|
34
|
+
if (!isActiveRunStatus(run.status)) return false;
|
|
35
|
+
if (run.async?.pid !== undefined) return false;
|
|
36
|
+
const updatedAt = new Date(run.updatedAt).getTime();
|
|
37
|
+
if (!Number.isFinite(updatedAt) || now - updatedAt < staleMs) return false;
|
|
38
|
+
if (agents.length === 0) return run.summary === "Creating workflow prompts and placeholder results.";
|
|
39
|
+
return agents.every((agent) => agent.status === "queued" && !agent.completedAt && !agent.progress);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function hasDurableActiveAgentEvidence(agent: CrewAgentRecord): boolean {
|
|
43
|
+
if (agent.status !== "running" && agent.status !== "queued") return false;
|
|
44
|
+
return Boolean(agent.statusPath || agent.eventsPath || agent.outputPath || agent.progress || agent.toolUses || agent.jsonEvents);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function hasStaleAsyncProcess(run: TeamRunManifest): boolean {
|
|
48
|
+
if (!isActiveRunStatus(run.status) || !run.async) return false;
|
|
49
|
+
return !checkProcessLiveness(run.async.pid).alive;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function isDisplayActiveRun(run: TeamRunManifest, agents: CrewAgentRecord[] = [], now = Date.now()): boolean {
|
|
53
|
+
if (!isActiveRunStatus(run.status) || hasStaleAsyncProcess(run) || isLikelyOrphanedActiveRun(run, agents, now)) return false;
|
|
54
|
+
if (agents.length === 0) return true;
|
|
55
|
+
return agents.some(hasDurableActiveAgentEvidence);
|
|
56
|
+
}
|
|
@@ -1,43 +1,43 @@
|
|
|
1
|
-
export interface ProgressEventSummary {
|
|
2
|
-
eventType: string;
|
|
3
|
-
currentTool?: string;
|
|
4
|
-
toolCount?: number;
|
|
5
|
-
tokens?: number;
|
|
6
|
-
turns?: number;
|
|
7
|
-
activityState?: string;
|
|
8
|
-
lastActivityAt?: string;
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
export interface ProgressEventCoalesceDecision {
|
|
12
|
-
shouldAppend: boolean;
|
|
13
|
-
reason: string;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export interface ProgressEventCoalesceInput {
|
|
17
|
-
previous?: ProgressEventSummary;
|
|
18
|
-
next: ProgressEventSummary;
|
|
19
|
-
nowMs: number;
|
|
20
|
-
lastAppendMs?: number;
|
|
21
|
-
minIntervalMs: number;
|
|
22
|
-
force?: boolean;
|
|
23
|
-
tokenThreshold?: number;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
const DEFAULT_TOKEN_THRESHOLD = 256;
|
|
27
|
-
|
|
28
|
-
function numericIncrease(previous: number | undefined, next: number | undefined): number {
|
|
29
|
-
return next !== undefined && previous !== undefined ? next - previous : next !== undefined ? next : 0;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
export function shouldAppendProgressEventUpdate(input: ProgressEventCoalesceInput): ProgressEventCoalesceDecision {
|
|
33
|
-
if (input.force) return { shouldAppend: true, reason: "force" };
|
|
34
|
-
if (!input.previous) return { shouldAppend: true, reason: "first" };
|
|
35
|
-
if (input.previous.activityState !== input.next.activityState) return { shouldAppend: true, reason: "activity_changed" };
|
|
36
|
-
if (input.previous.currentTool !== input.next.currentTool) return { shouldAppend: true, reason: "tool_changed" };
|
|
37
|
-
if (numericIncrease(input.previous.toolCount, input.next.toolCount) > 0) return { shouldAppend: true, reason: "tool_count_increased" };
|
|
38
|
-
if (numericIncrease(input.previous.turns, input.next.turns) > 0) return { shouldAppend: true, reason: "turns_increased" };
|
|
39
|
-
const tokenIncrease = numericIncrease(input.previous.tokens, input.next.tokens);
|
|
40
|
-
if (tokenIncrease >= (input.tokenThreshold ?? DEFAULT_TOKEN_THRESHOLD)) return { shouldAppend: true, reason: "tokens_increased" };
|
|
41
|
-
if (input.lastAppendMs === undefined || input.nowMs - input.lastAppendMs >= input.minIntervalMs) return { shouldAppend: true, reason: "interval" };
|
|
42
|
-
return { shouldAppend: false, reason: "coalesced" };
|
|
43
|
-
}
|
|
1
|
+
export interface ProgressEventSummary {
|
|
2
|
+
eventType: string;
|
|
3
|
+
currentTool?: string;
|
|
4
|
+
toolCount?: number;
|
|
5
|
+
tokens?: number;
|
|
6
|
+
turns?: number;
|
|
7
|
+
activityState?: string;
|
|
8
|
+
lastActivityAt?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface ProgressEventCoalesceDecision {
|
|
12
|
+
shouldAppend: boolean;
|
|
13
|
+
reason: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface ProgressEventCoalesceInput {
|
|
17
|
+
previous?: ProgressEventSummary;
|
|
18
|
+
next: ProgressEventSummary;
|
|
19
|
+
nowMs: number;
|
|
20
|
+
lastAppendMs?: number;
|
|
21
|
+
minIntervalMs: number;
|
|
22
|
+
force?: boolean;
|
|
23
|
+
tokenThreshold?: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const DEFAULT_TOKEN_THRESHOLD = 256;
|
|
27
|
+
|
|
28
|
+
function numericIncrease(previous: number | undefined, next: number | undefined): number {
|
|
29
|
+
return next !== undefined && previous !== undefined ? next - previous : next !== undefined ? next : 0;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function shouldAppendProgressEventUpdate(input: ProgressEventCoalesceInput): ProgressEventCoalesceDecision {
|
|
33
|
+
if (input.force) return { shouldAppend: true, reason: "force" };
|
|
34
|
+
if (!input.previous) return { shouldAppend: true, reason: "first" };
|
|
35
|
+
if (input.previous.activityState !== input.next.activityState) return { shouldAppend: true, reason: "activity_changed" };
|
|
36
|
+
if (input.previous.currentTool !== input.next.currentTool) return { shouldAppend: true, reason: "tool_changed" };
|
|
37
|
+
if (numericIncrease(input.previous.toolCount, input.next.toolCount) > 0) return { shouldAppend: true, reason: "tool_count_increased" };
|
|
38
|
+
if (numericIncrease(input.previous.turns, input.next.turns) > 0) return { shouldAppend: true, reason: "turns_increased" };
|
|
39
|
+
const tokenIncrease = numericIncrease(input.previous.tokens, input.next.tokens);
|
|
40
|
+
if (tokenIncrease >= (input.tokenThreshold ?? DEFAULT_TOKEN_THRESHOLD)) return { shouldAppend: true, reason: "tokens_increased" };
|
|
41
|
+
if (input.lastAppendMs === undefined || input.nowMs - input.lastAppendMs >= input.minIntervalMs) return { shouldAppend: true, reason: "interval" };
|
|
42
|
+
return { shouldAppend: false, reason: "coalesced" };
|
|
43
|
+
}
|
|
@@ -1,74 +1,74 @@
|
|
|
1
|
-
import type { PolicyDecision, PolicyDecisionReason } from "../state/types.ts";
|
|
2
|
-
|
|
3
|
-
export type FailureScenario = "trust_prompt_unresolved" | "prompt_misdelivery" | "stale_branch" | "compile_red_cross_crate" | "mcp_handshake_failure" | "partial_plugin_startup" | "provider_failure" | "task_failed" | "worker_stale" | "green_unsatisfied";
|
|
4
|
-
export type RecoveryStep = "accept_trust_prompt" | "redirect_prompt_to_agent" | "rebase_branch" | "clean_build" | "retry_mcp_handshake" | "restart_plugin" | "restart_worker" | "rerun_task" | "collect_verification_evidence" | "escalate_to_human";
|
|
5
|
-
export type RecoveryResultState = "planned" | "skipped" | "escalation_required";
|
|
6
|
-
|
|
7
|
-
export interface RecoveryRecipe {
|
|
8
|
-
scenario: FailureScenario;
|
|
9
|
-
steps: RecoveryStep[];
|
|
10
|
-
maxAttempts: number;
|
|
11
|
-
escalationPolicy: "alert_human" | "log_and_continue" | "abort";
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export interface RecoveryLedgerEntry {
|
|
15
|
-
scenario: FailureScenario;
|
|
16
|
-
taskId?: string;
|
|
17
|
-
decisionReason: PolicyDecisionReason;
|
|
18
|
-
attempt: number;
|
|
19
|
-
state: RecoveryResultState;
|
|
20
|
-
steps: RecoveryStep[];
|
|
21
|
-
message: string;
|
|
22
|
-
createdAt: string;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
export interface RecoveryLedger {
|
|
26
|
-
entries: RecoveryLedgerEntry[];
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export function scenarioForPolicyReason(reason: PolicyDecisionReason): FailureScenario {
|
|
30
|
-
switch (reason) {
|
|
31
|
-
case "branch_stale": return "stale_branch";
|
|
32
|
-
case "worker_stale": return "worker_stale";
|
|
33
|
-
case "green_unsatisfied": return "green_unsatisfied";
|
|
34
|
-
case "task_failed": return "task_failed";
|
|
35
|
-
default: return "provider_failure";
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
export function recipeFor(scenario: FailureScenario): RecoveryRecipe {
|
|
40
|
-
switch (scenario) {
|
|
41
|
-
case "trust_prompt_unresolved": return { scenario, steps: ["accept_trust_prompt"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
42
|
-
case "prompt_misdelivery": return { scenario, steps: ["redirect_prompt_to_agent"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
43
|
-
case "stale_branch": return { scenario, steps: ["rebase_branch", "clean_build"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
44
|
-
case "compile_red_cross_crate": return { scenario, steps: ["clean_build"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
45
|
-
case "mcp_handshake_failure": return { scenario, steps: ["retry_mcp_handshake"], maxAttempts: 1, escalationPolicy: "abort" };
|
|
46
|
-
case "partial_plugin_startup": return { scenario, steps: ["restart_plugin", "retry_mcp_handshake"], maxAttempts: 1, escalationPolicy: "log_and_continue" };
|
|
47
|
-
case "worker_stale": return { scenario, steps: ["restart_worker"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
48
|
-
case "green_unsatisfied": return { scenario, steps: ["collect_verification_evidence"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
49
|
-
case "task_failed": return { scenario, steps: ["rerun_task"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
50
|
-
case "provider_failure": return { scenario, steps: ["restart_worker"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export function buildRecoveryLedger(decisions: PolicyDecision[], previous: RecoveryLedger = { entries: [] }): RecoveryLedger {
|
|
55
|
-
const entries = [...previous.entries];
|
|
56
|
-
for (const item of decisions) {
|
|
57
|
-
if (!["retry", "escalate", "block"].includes(item.action)) continue;
|
|
58
|
-
const scenario = scenarioForPolicyReason(item.reason);
|
|
59
|
-
const recipe = recipeFor(scenario);
|
|
60
|
-
const priorAttempts = entries.filter((entry) => entry.scenario === scenario && entry.taskId === item.taskId).length;
|
|
61
|
-
const attempt = priorAttempts + 1;
|
|
62
|
-
entries.push({
|
|
63
|
-
scenario,
|
|
64
|
-
taskId: item.taskId,
|
|
65
|
-
decisionReason: item.reason,
|
|
66
|
-
attempt,
|
|
67
|
-
state: attempt <= recipe.maxAttempts && item.action !== "block" ? "planned" : "escalation_required",
|
|
68
|
-
steps: attempt <= recipe.maxAttempts ? recipe.steps : ["escalate_to_human"],
|
|
69
|
-
message: item.message,
|
|
70
|
-
createdAt: new Date().toISOString(),
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
return { entries };
|
|
74
|
-
}
|
|
1
|
+
import type { PolicyDecision, PolicyDecisionReason } from "../state/types.ts";
|
|
2
|
+
|
|
3
|
+
export type FailureScenario = "trust_prompt_unresolved" | "prompt_misdelivery" | "stale_branch" | "compile_red_cross_crate" | "mcp_handshake_failure" | "partial_plugin_startup" | "provider_failure" | "task_failed" | "worker_stale" | "green_unsatisfied";
|
|
4
|
+
export type RecoveryStep = "accept_trust_prompt" | "redirect_prompt_to_agent" | "rebase_branch" | "clean_build" | "retry_mcp_handshake" | "restart_plugin" | "restart_worker" | "rerun_task" | "collect_verification_evidence" | "escalate_to_human";
|
|
5
|
+
export type RecoveryResultState = "planned" | "skipped" | "escalation_required";
|
|
6
|
+
|
|
7
|
+
export interface RecoveryRecipe {
|
|
8
|
+
scenario: FailureScenario;
|
|
9
|
+
steps: RecoveryStep[];
|
|
10
|
+
maxAttempts: number;
|
|
11
|
+
escalationPolicy: "alert_human" | "log_and_continue" | "abort";
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface RecoveryLedgerEntry {
|
|
15
|
+
scenario: FailureScenario;
|
|
16
|
+
taskId?: string;
|
|
17
|
+
decisionReason: PolicyDecisionReason;
|
|
18
|
+
attempt: number;
|
|
19
|
+
state: RecoveryResultState;
|
|
20
|
+
steps: RecoveryStep[];
|
|
21
|
+
message: string;
|
|
22
|
+
createdAt: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface RecoveryLedger {
|
|
26
|
+
entries: RecoveryLedgerEntry[];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function scenarioForPolicyReason(reason: PolicyDecisionReason): FailureScenario {
|
|
30
|
+
switch (reason) {
|
|
31
|
+
case "branch_stale": return "stale_branch";
|
|
32
|
+
case "worker_stale": return "worker_stale";
|
|
33
|
+
case "green_unsatisfied": return "green_unsatisfied";
|
|
34
|
+
case "task_failed": return "task_failed";
|
|
35
|
+
default: return "provider_failure";
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function recipeFor(scenario: FailureScenario): RecoveryRecipe {
|
|
40
|
+
switch (scenario) {
|
|
41
|
+
case "trust_prompt_unresolved": return { scenario, steps: ["accept_trust_prompt"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
42
|
+
case "prompt_misdelivery": return { scenario, steps: ["redirect_prompt_to_agent"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
43
|
+
case "stale_branch": return { scenario, steps: ["rebase_branch", "clean_build"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
44
|
+
case "compile_red_cross_crate": return { scenario, steps: ["clean_build"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
45
|
+
case "mcp_handshake_failure": return { scenario, steps: ["retry_mcp_handshake"], maxAttempts: 1, escalationPolicy: "abort" };
|
|
46
|
+
case "partial_plugin_startup": return { scenario, steps: ["restart_plugin", "retry_mcp_handshake"], maxAttempts: 1, escalationPolicy: "log_and_continue" };
|
|
47
|
+
case "worker_stale": return { scenario, steps: ["restart_worker"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
48
|
+
case "green_unsatisfied": return { scenario, steps: ["collect_verification_evidence"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
49
|
+
case "task_failed": return { scenario, steps: ["rerun_task"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
50
|
+
case "provider_failure": return { scenario, steps: ["restart_worker"], maxAttempts: 1, escalationPolicy: "alert_human" };
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function buildRecoveryLedger(decisions: PolicyDecision[], previous: RecoveryLedger = { entries: [] }): RecoveryLedger {
|
|
55
|
+
const entries = [...previous.entries];
|
|
56
|
+
for (const item of decisions) {
|
|
57
|
+
if (!["retry", "escalate", "block"].includes(item.action)) continue;
|
|
58
|
+
const scenario = scenarioForPolicyReason(item.reason);
|
|
59
|
+
const recipe = recipeFor(scenario);
|
|
60
|
+
const priorAttempts = entries.filter((entry) => entry.scenario === scenario && entry.taskId === item.taskId).length;
|
|
61
|
+
const attempt = priorAttempts + 1;
|
|
62
|
+
entries.push({
|
|
63
|
+
scenario,
|
|
64
|
+
taskId: item.taskId,
|
|
65
|
+
decisionReason: item.reason,
|
|
66
|
+
attempt,
|
|
67
|
+
state: attempt <= recipe.maxAttempts && item.action !== "block" ? "planned" : "escalation_required",
|
|
68
|
+
steps: attempt <= recipe.maxAttempts ? recipe.steps : ["escalate_to_human"],
|
|
69
|
+
message: item.message,
|
|
70
|
+
createdAt: new Date().toISOString(),
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
return { entries };
|
|
74
|
+
}
|
|
@@ -1,59 +1,59 @@
|
|
|
1
|
-
import { sleep } from "../utils/sleep.ts";
|
|
2
|
-
|
|
3
|
-
export interface RetryPolicy {
|
|
4
|
-
maxAttempts: number;
|
|
5
|
-
backoffMs: number;
|
|
6
|
-
jitterRatio: number;
|
|
7
|
-
exponentialFactor: number;
|
|
8
|
-
retryableErrors?: string[];
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
export interface RetryHooks {
|
|
12
|
-
onAttemptFailed?: (attempt: number, error: Error, nextDelayMs: number) => void;
|
|
13
|
-
onRetryGivenUp?: (attempts: number, error: Error) => void;
|
|
14
|
-
signal?: AbortSignal;
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export const DEFAULT_RETRY_POLICY: RetryPolicy = { maxAttempts: 3, backoffMs: 1000, jitterRatio: 0.3, exponentialFactor: 2 };
|
|
18
|
-
|
|
19
|
-
function asError(error: unknown): Error {
|
|
20
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function globToRegex(pattern: string): RegExp {
|
|
24
|
-
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
25
|
-
return new RegExp(`^${escaped}$`, "i");
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
function isRetryable(error: Error, policy: RetryPolicy): boolean {
|
|
29
|
-
const patterns = policy.retryableErrors ?? [];
|
|
30
|
-
if (!patterns.length) return true;
|
|
31
|
-
return patterns.some((pattern) => globToRegex(pattern).test(error.message));
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
export function calculateRetryDelay(attempt: number, policy: RetryPolicy = DEFAULT_RETRY_POLICY, random = Math.random): number {
|
|
35
|
-
const base = policy.backoffMs * Math.pow(policy.exponentialFactor, Math.max(0, attempt - 1));
|
|
36
|
-
const jitter = (random() * 2 - 1) * policy.jitterRatio * base;
|
|
37
|
-
return Math.max(0, base + jitter);
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export async function executeWithRetry<T>(fn: (attempt: number) => Promise<T>, policy: RetryPolicy = DEFAULT_RETRY_POLICY, hooks: RetryHooks = {}): Promise<T> {
|
|
41
|
-
const normalized: RetryPolicy = { ...DEFAULT_RETRY_POLICY, ...policy, maxAttempts: Math.max(1, policy.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts) };
|
|
42
|
-
let lastError: Error | undefined;
|
|
43
|
-
for (let attempt = 1; attempt <= normalized.maxAttempts; attempt += 1) {
|
|
44
|
-
if (hooks.signal?.aborted) throw new Error("Retry aborted.");
|
|
45
|
-
try {
|
|
46
|
-
return await fn(attempt);
|
|
47
|
-
} catch (error) {
|
|
48
|
-
lastError = asError(error);
|
|
49
|
-
if (attempt >= normalized.maxAttempts || !isRetryable(lastError, normalized)) {
|
|
50
|
-
hooks.onRetryGivenUp?.(attempt, lastError);
|
|
51
|
-
throw lastError;
|
|
52
|
-
}
|
|
53
|
-
const delay = calculateRetryDelay(attempt, normalized);
|
|
54
|
-
hooks.onAttemptFailed?.(attempt, lastError, delay);
|
|
55
|
-
await sleep(delay, hooks.signal);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
throw lastError ?? new Error("Retry failed without error.");
|
|
59
|
-
}
|
|
1
|
+
import { sleep } from "../utils/sleep.ts";
|
|
2
|
+
|
|
3
|
+
export interface RetryPolicy {
|
|
4
|
+
maxAttempts: number;
|
|
5
|
+
backoffMs: number;
|
|
6
|
+
jitterRatio: number;
|
|
7
|
+
exponentialFactor: number;
|
|
8
|
+
retryableErrors?: string[];
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface RetryHooks {
|
|
12
|
+
onAttemptFailed?: (attempt: number, error: Error, nextDelayMs: number) => void;
|
|
13
|
+
onRetryGivenUp?: (attempts: number, error: Error) => void;
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const DEFAULT_RETRY_POLICY: RetryPolicy = { maxAttempts: 3, backoffMs: 1000, jitterRatio: 0.3, exponentialFactor: 2 };
|
|
18
|
+
|
|
19
|
+
function asError(error: unknown): Error {
|
|
20
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function globToRegex(pattern: string): RegExp {
|
|
24
|
+
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
25
|
+
return new RegExp(`^${escaped}$`, "i");
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function isRetryable(error: Error, policy: RetryPolicy): boolean {
|
|
29
|
+
const patterns = policy.retryableErrors ?? [];
|
|
30
|
+
if (!patterns.length) return true;
|
|
31
|
+
return patterns.some((pattern) => globToRegex(pattern).test(error.message));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function calculateRetryDelay(attempt: number, policy: RetryPolicy = DEFAULT_RETRY_POLICY, random = Math.random): number {
|
|
35
|
+
const base = policy.backoffMs * Math.pow(policy.exponentialFactor, Math.max(0, attempt - 1));
|
|
36
|
+
const jitter = (random() * 2 - 1) * policy.jitterRatio * base;
|
|
37
|
+
return Math.max(0, base + jitter);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export async function executeWithRetry<T>(fn: (attempt: number) => Promise<T>, policy: RetryPolicy = DEFAULT_RETRY_POLICY, hooks: RetryHooks = {}): Promise<T> {
|
|
41
|
+
const normalized: RetryPolicy = { ...DEFAULT_RETRY_POLICY, ...policy, maxAttempts: Math.max(1, policy.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts) };
|
|
42
|
+
let lastError: Error | undefined;
|
|
43
|
+
for (let attempt = 1; attempt <= normalized.maxAttempts; attempt += 1) {
|
|
44
|
+
if (hooks.signal?.aborted) throw new Error("Retry aborted.");
|
|
45
|
+
try {
|
|
46
|
+
return await fn(attempt);
|
|
47
|
+
} catch (error) {
|
|
48
|
+
lastError = asError(error);
|
|
49
|
+
if (attempt >= normalized.maxAttempts || !isRetryable(lastError, normalized)) {
|
|
50
|
+
hooks.onRetryGivenUp?.(attempt, lastError);
|
|
51
|
+
throw lastError;
|
|
52
|
+
}
|
|
53
|
+
const delay = calculateRetryDelay(attempt, normalized);
|
|
54
|
+
hooks.onAttemptFailed?.(attempt, lastError, delay);
|
|
55
|
+
await sleep(delay, hooks.signal);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
throw lastError ?? new Error("Retry failed without error.");
|
|
59
|
+
}
|