plugin-agent-orchestrator 1.1.3 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.js +1 -1
- package/dist/client-v2/214.b063f33ba044a808.js +10 -0
- package/dist/client-v2/253.59577b40d9875be0.js +10 -0
- package/dist/client-v2/264.7ed8bee71fa5935c.js +10 -0
- package/dist/client-v2/{41.ba6e080cc0488143.js → 41.3deb9e57a0f64fc4.js} +1 -1
- package/dist/client-v2/418.58fa350f5496efd3.js +10 -0
- package/dist/client-v2/{53.7c29c28043e9112d.js → 53.9a10455323390f14.js} +1 -1
- package/dist/client-v2/619.0a2144aaffefe018.js +10 -0
- package/dist/client-v2/{70.bda9518881c05360.js → 70.ffff2997c10fd4b0.js} +1 -1
- package/dist/client-v2/{794.9bbb8d3a61b2bdf2.js → 794.a67a2614405ba627.js} +1 -1
- package/dist/client-v2/925.e5925bf21ed68e7a.js +10 -0
- package/dist/client-v2/{993.82778b6fd6db3cb1.js → 993.b0688fdc5ccb000a.js} +1 -1
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +8 -6
- package/dist/locale/en-US.json +48 -28
- package/dist/locale/vi-VN.json +48 -28
- package/dist/locale/zh-CN.json +48 -28
- package/dist/server/collections/agent-execution-spans.js +47 -0
- package/dist/server/collections/agent-harness-profile-versions.js +56 -0
- package/dist/server/collections/agent-harness-profiles.js +12 -0
- package/dist/server/collections/agent-loop-action-approvals.js +68 -0
- package/dist/server/collections/agent-loop-artifacts.js +55 -0
- package/dist/server/collections/agent-loop-circuit-states.js +53 -0
- package/dist/server/collections/agent-loop-control-settings.js +51 -0
- package/dist/server/collections/agent-loop-events.js +27 -0
- package/dist/server/collections/agent-loop-path-locks.js +57 -0
- package/dist/server/collections/agent-loop-patterns.js +72 -0
- package/dist/server/collections/agent-loop-runs.js +163 -2
- package/dist/server/collections/agent-loop-steps.js +69 -0
- package/dist/server/collections/agent-loop-usage-buckets.js +52 -0
- package/dist/server/collections/agent-loop-worktrees.js +55 -0
- package/dist/server/migrations/{20260427000000-change-packages-to-text.js → 20260427000002-change-packages-to-text.js} +2 -0
- package/dist/server/migrations/20260429000000-add-llm-fields.js +3 -2
- package/dist/server/migrations/{20260429000000-fix-inputargs-json-to-text.js → 20260429000001-fix-inputargs-json-to-text.js} +14 -6
- package/dist/server/migrations/20260503000000-add-orchestrator-trace-fields.js +2 -1
- package/dist/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.js +2 -1
- package/dist/server/migrations/20260803010000-backfill-loop-control-plane-data.js +262 -0
- package/dist/server/plugin.js +52 -150
- package/dist/server/resources/agent-loop-approval.js +83 -0
- package/dist/server/resources/agent-loop-artifact.js +86 -0
- package/dist/server/resources/agent-loop-events-stream.js +130 -0
- package/dist/server/resources/agent-loop.js +181 -139
- package/dist/server/resources/resource-helpers.js +63 -0
- package/dist/server/resources/tracing.js +121 -107
- package/dist/server/services/DurableCircuitBreakerService.js +229 -0
- package/dist/server/services/HarnessCompiler.js +142 -0
- package/dist/server/services/HarnessProfileService.js +168 -0
- package/dist/server/services/HarnessSchema.js +167 -0
- package/dist/server/services/LoopBudgetService.js +269 -0
- package/dist/server/services/LoopControlService.js +139 -0
- package/dist/server/services/LoopPatternSchema.js +180 -0
- package/dist/server/services/LoopPatternService.js +132 -0
- package/dist/server/services/LoopRunRepository.js +205 -0
- package/dist/server/services/LoopRunStateMachine.js +389 -0
- package/dist/server/services/LoopSchedulerService.js +193 -0
- package/dist/server/services/LoopTriggerService.js +128 -0
- package/dist/server/services/LoopWorkerService.js +461 -0
- package/dist/server/services/PathLockService.js +288 -0
- package/dist/server/services/PluginAiRuntimeAdapter.js +215 -0
- package/dist/server/services/VerificationSchema.js +59 -0
- package/dist/server/services/VerificationService.js +169 -0
- package/dist/server/skill-hub/actions/git-import.js +13 -7
- package/dist/server/skill-hub/services/RegistrySkillSnapshotService.js +1 -5
- package/dist/server/utils/ensure-indexes.js +111 -0
- package/dist/server/utils/loop-pattern-context.js +56 -0
- package/package.json +1 -1
- package/src/client/OrchestratorSettings.tsx +44 -0
- package/src/client-v2/components/AgentRunsTab.tsx +576 -494
- package/src/client-v2/components/HarnessProfilesTab.tsx +13 -8
- package/src/client-v2/components/MemoryInspectorTab.tsx +19 -3
- package/src/client-v2/components/TracingTab.tsx +515 -432
- package/src/client-v2/hooks/useApiRequest.ts +141 -121
- package/src/client-v2/skill-hub/components/SkillMetrics.tsx +8 -6
- package/src/locale/en-US.json +52 -28
- package/src/locale/vi-VN.json +52 -28
- package/src/locale/zh-CN.json +52 -28
- package/src/server/__tests__/circuit-breaker.test.ts +225 -146
- package/src/server/__tests__/harness-compiler.test.ts +131 -0
- package/src/server/__tests__/harness-profile-service.test.ts +116 -0
- package/src/server/__tests__/harness-tool-plan.test.ts +148 -0
- package/src/server/__tests__/loop-budget-service.test.ts +255 -0
- package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -0
- package/src/server/__tests__/loop-pattern-schema.test.ts +56 -0
- package/src/server/__tests__/loop-pattern-service.test.ts +148 -0
- package/src/server/__tests__/loop-run-state-machine.test.ts +90 -0
- package/src/server/__tests__/loop-worker-claim.test.ts +341 -0
- package/src/server/__tests__/loop-worker-execution.test.ts +236 -0
- package/src/server/__tests__/path-lock-service.test.ts +254 -0
- package/src/server/__tests__/smoke.test.ts +62 -21
- package/src/server/collections/agent-execution-spans.ts +47 -0
- package/src/server/collections/agent-harness-profile-versions.ts +26 -0
- package/src/server/collections/agent-harness-profiles.ts +12 -0
- package/src/server/collections/agent-loop-action-approvals.ts +38 -0
- package/src/server/collections/agent-loop-artifacts.ts +25 -0
- package/src/server/collections/agent-loop-circuit-states.ts +23 -0
- package/src/server/collections/agent-loop-control-settings.ts +21 -0
- package/src/server/collections/agent-loop-events.ts +27 -0
- package/src/server/collections/agent-loop-path-locks.ts +27 -0
- package/src/server/collections/agent-loop-patterns.ts +42 -0
- package/src/server/collections/agent-loop-runs.ts +163 -2
- package/src/server/collections/agent-loop-steps.ts +69 -0
- package/src/server/collections/agent-loop-usage-buckets.ts +22 -0
- package/src/server/collections/agent-loop-worktrees.ts +25 -0
- package/src/server/migrations/{20260427000000-change-packages-to-text.ts → 20260427000002-change-packages-to-text.ts} +3 -0
- package/src/server/migrations/20260429000000-add-llm-fields.ts +3 -2
- package/src/server/migrations/{20260429000000-fix-inputargs-json-to-text.ts → 20260429000001-fix-inputargs-json-to-text.ts} +13 -6
- package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +2 -1
- package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +2 -1
- package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -0
- package/src/server/plugin.ts +53 -167
- package/src/server/resources/agent-loop-approval.ts +52 -0
- package/src/server/resources/agent-loop-artifact.ts +56 -0
- package/src/server/resources/agent-loop-events-stream.ts +105 -0
- package/src/server/resources/agent-loop.ts +202 -143
- package/src/server/resources/agent-monitor.ts +482 -482
- package/src/server/resources/resource-helpers.ts +33 -0
- package/src/server/resources/tracing.ts +145 -137
- package/src/server/services/DurableCircuitBreakerService.ts +239 -0
- package/src/server/services/HarnessCompiler.ts +133 -0
- package/src/server/services/HarnessProfileService.ts +157 -0
- package/src/server/services/HarnessSchema.ts +163 -0
- package/src/server/services/LoopBudgetService.ts +275 -0
- package/src/server/services/LoopControlService.ts +124 -0
- package/src/server/services/LoopPatternSchema.ts +167 -0
- package/src/server/services/LoopPatternService.ts +143 -0
- package/src/server/services/LoopRunRepository.ts +237 -0
- package/src/server/services/LoopRunStateMachine.ts +432 -0
- package/src/server/services/LoopSchedulerService.ts +199 -0
- package/src/server/services/LoopTriggerService.ts +115 -0
- package/src/server/services/LoopWorkerService.ts +522 -0
- package/src/server/services/PathLockService.ts +321 -0
- package/src/server/services/PluginAiRuntimeAdapter.ts +315 -0
- package/src/server/services/VerificationSchema.ts +31 -0
- package/src/server/services/VerificationService.ts +185 -0
- package/src/server/skill-hub/actions/__tests__/git-import.test.ts +15 -1
- package/src/server/skill-hub/actions/git-import.ts +9 -3
- package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +1 -5
- package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +1 -3
- package/src/server/utils/ensure-indexes.ts +96 -0
- package/src/server/utils/logging.ts +0 -1
- package/src/server/utils/loop-pattern-context.ts +20 -0
- package/dist/client-v2/214.79650a549273f163.js +0 -10
- package/dist/client-v2/253.9f0537713cee9e6c.js +0 -10
- package/dist/client-v2/264.718a107e43fc163c.js +0 -10
- package/dist/client-v2/418.ce8f5b2b8a229d1d.js +0 -10
- package/dist/client-v2/619.bd3c5698b40705c3.js +0 -10
- package/dist/client-v2/925.f5370de8f6632d65.js +0 -10
- package/dist/server/services/AgentHarness.js +0 -81
- package/dist/server/services/AgentLoopController.js +0 -1023
- package/dist/server/services/AgentLoopRepository.js +0 -192
- package/dist/server/services/AgentLoopService.js +0 -142
- package/dist/server/services/AgentPlanValidator.js +0 -99
- package/dist/server/services/AgentPlannerService.js +0 -99
- package/dist/server/services/CircuitBreaker.js +0 -125
- package/dist/server/services/ContextAggregator.js +0 -201
- package/dist/server/services/PlanningPromptService.js +0 -95
- package/dist/server/tools/agent-loop.js +0 -373
- package/dist/server/tools/orchestrator-plan.js +0 -254
- package/src/server/__tests__/agent-loop-controller.test.ts +0 -375
- package/src/server/__tests__/context-aggregator.test.ts +0 -222
- package/src/server/__tests__/parallel-execution.test.ts +0 -318
- package/src/server/__tests__/planning-prompt-service.test.ts +0 -23
- package/src/server/services/AgentHarness.ts +0 -67
- package/src/server/services/AgentLoopController.ts +0 -1263
- package/src/server/services/AgentLoopRepository.ts +0 -178
- package/src/server/services/AgentLoopService.ts +0 -177
- package/src/server/services/AgentPlanValidator.ts +0 -73
- package/src/server/services/AgentPlannerService.ts +0 -80
- package/src/server/services/CircuitBreaker.ts +0 -126
- package/src/server/services/ContextAggregator.ts +0 -239
- package/src/server/services/PlanningPromptService.ts +0 -72
- package/src/server/tools/agent-loop.ts +0 -360
- package/src/server/tools/orchestrator-plan.ts +0 -255
|
@@ -1,1263 +0,0 @@
|
|
|
1
|
-
import { AgentRegistryService } from './AgentRegistryService';
|
|
2
|
-
import { AgentPlannerService } from './AgentPlannerService';
|
|
3
|
-
import { AgentPlanValidator } from './AgentPlanValidator';
|
|
4
|
-
import { AgentLoopRepository } from './AgentLoopRepository';
|
|
5
|
-
import { AgentHarness } from './AgentHarness';
|
|
6
|
-
import { AgentLoopPolicy, AgentLoopPlanStepInput, AgentLoopRunStatus, AgentLoopStepStatus } from './AgentLoopService';
|
|
7
|
-
import { TokenTracker } from './TokenTracker';
|
|
8
|
-
import { getCircuitBreaker, subAgentCircuitKey } from './CircuitBreaker';
|
|
9
|
-
import { createHash } from 'crypto';
|
|
10
|
-
import { asObject, asArray, trimText, normalizeStepType, normalizePlanKey } from '../utils/ctx-utils';
|
|
11
|
-
|
|
12
|
-
const DEFAULT_POLICY: AgentLoopPolicy = {
|
|
13
|
-
maxIterations: 20,
|
|
14
|
-
maxStepAttempts: 2,
|
|
15
|
-
allowReplan: true,
|
|
16
|
-
requireVerification: true,
|
|
17
|
-
stopOnApprovalRequired: true,
|
|
18
|
-
maxContextTokens: 4000,
|
|
19
|
-
contextSummaryStrategy: 'last_n',
|
|
20
|
-
includeToolResults: false,
|
|
21
|
-
includeStepOutputs: true,
|
|
22
|
-
maxConcurrency: 5,
|
|
23
|
-
};
|
|
24
|
-
|
|
25
|
-
const TERMINAL_RUN_STATUSES = new Set<AgentLoopRunStatus>(['succeeded', 'failed', 'rejected', 'canceled']);
|
|
26
|
-
const TERMINAL_STEP_STATUSES = new Set<AgentLoopStepStatus>(['succeeded', 'skipped']);
|
|
27
|
-
const ORCHESTRATOR_CONTROLLER_MAX_STEPS = 100;
|
|
28
|
-
|
|
29
|
-
function now() {
|
|
30
|
-
return new Date();
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
function createRootRunId(seed = '') {
|
|
34
|
-
const hash = createHash('sha1').update(`${Date.now()}::${Math.random()}::${seed}`).digest('hex').slice(0, 10);
|
|
35
|
-
return `loop_${Date.now()}_${hash}`;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
function normalizePolicy(policy?: Partial<AgentLoopPolicy>): AgentLoopPolicy {
|
|
39
|
-
const next = { ...DEFAULT_POLICY, ...(policy || {}) };
|
|
40
|
-
next.maxIterations = Math.max(1, Number(next.maxIterations || DEFAULT_POLICY.maxIterations));
|
|
41
|
-
next.maxStepAttempts = Math.max(1, Number(next.maxStepAttempts || DEFAULT_POLICY.maxStepAttempts));
|
|
42
|
-
next.allowReplan = next.allowReplan !== false;
|
|
43
|
-
next.requireVerification = next.requireVerification !== false;
|
|
44
|
-
next.stopOnApprovalRequired = next.stopOnApprovalRequired !== false;
|
|
45
|
-
next.maxContextTokens = next.maxContextTokens || DEFAULT_POLICY.maxContextTokens;
|
|
46
|
-
next.contextSummaryStrategy = next.contextSummaryStrategy || DEFAULT_POLICY.contextSummaryStrategy;
|
|
47
|
-
next.includeToolResults = next.includeToolResults !== false;
|
|
48
|
-
next.includeStepOutputs = next.includeStepOutputs !== false;
|
|
49
|
-
next.maxConcurrency = Math.max(1, Number(next.maxConcurrency || DEFAULT_POLICY.maxConcurrency));
|
|
50
|
-
return next;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export class AgentLoopController {
|
|
54
|
-
constructor(
|
|
55
|
-
private readonly registryService: AgentRegistryService,
|
|
56
|
-
private readonly plannerService: AgentPlannerService,
|
|
57
|
-
private readonly validator: AgentPlanValidator,
|
|
58
|
-
private readonly repository: AgentLoopRepository,
|
|
59
|
-
private readonly harness: AgentHarness,
|
|
60
|
-
private readonly tokenTracker: TokenTracker | null = null,
|
|
61
|
-
) {}
|
|
62
|
-
|
|
63
|
-
async createRun(options: {
|
|
64
|
-
goal: string;
|
|
65
|
-
leaderUsername?: string;
|
|
66
|
-
sessionId?: string;
|
|
67
|
-
messageId?: string;
|
|
68
|
-
userId?: string | number;
|
|
69
|
-
policy?: Partial<AgentLoopPolicy>;
|
|
70
|
-
metadata?: any;
|
|
71
|
-
plan?: AgentLoopPlanStepInput[];
|
|
72
|
-
}) {
|
|
73
|
-
const goal = String(options.goal || '').trim();
|
|
74
|
-
if (!goal) {
|
|
75
|
-
throw new Error('Agent loop goal is required.');
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const policy = normalizePolicy(options.policy);
|
|
79
|
-
const rootRunId = createRootRunId(options.leaderUsername || goal);
|
|
80
|
-
const run = await this.repository.createRun({
|
|
81
|
-
rootRunId,
|
|
82
|
-
sessionId: options.sessionId,
|
|
83
|
-
messageId: options.messageId,
|
|
84
|
-
leaderUsername: options.leaderUsername,
|
|
85
|
-
goal,
|
|
86
|
-
status: 'planning',
|
|
87
|
-
policy,
|
|
88
|
-
iterationCount: 0,
|
|
89
|
-
metadata: options.metadata || {},
|
|
90
|
-
userId: options.userId,
|
|
91
|
-
startedAt: now(),
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
await this.repository.createEvent({
|
|
95
|
-
runId: run.id,
|
|
96
|
-
type: 'created',
|
|
97
|
-
title: 'Agent loop created',
|
|
98
|
-
content: goal,
|
|
99
|
-
status: 'planning',
|
|
100
|
-
userId: options.userId,
|
|
101
|
-
payload: { rootRunId },
|
|
102
|
-
});
|
|
103
|
-
|
|
104
|
-
if (Array.isArray(options.plan) && options.plan.length > 0) {
|
|
105
|
-
this.validator.validate(options.plan);
|
|
106
|
-
await this.replacePlan(run.id, options.plan, {
|
|
107
|
-
userId: options.userId,
|
|
108
|
-
mode: 'append',
|
|
109
|
-
markRunning: true,
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
return this.getRunSnapshot(run.id);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
async planGoal(options: {
|
|
117
|
-
goal: string;
|
|
118
|
-
leaderUsername?: string;
|
|
119
|
-
sessionId?: string;
|
|
120
|
-
messageId?: string;
|
|
121
|
-
userId?: string | number;
|
|
122
|
-
policy?: Partial<AgentLoopPolicy>;
|
|
123
|
-
metadata?: any;
|
|
124
|
-
plan?: AgentLoopPlanStepInput[];
|
|
125
|
-
planSource?: string;
|
|
126
|
-
plannerModel?: string;
|
|
127
|
-
harnessTag?: string;
|
|
128
|
-
targetAgent?: string;
|
|
129
|
-
runId?: string | number;
|
|
130
|
-
}) {
|
|
131
|
-
const plan = this.plannerService.buildPlan(options.goal, options.plan, options);
|
|
132
|
-
this.validator.validate(plan);
|
|
133
|
-
|
|
134
|
-
const harnessTag = String(options.harnessTag || options.metadata?.harnessTag || 'default').trim() || 'default';
|
|
135
|
-
const harnessProfile = await this.registryService.getHarnessProfile(harnessTag);
|
|
136
|
-
const harnessSettings = asObject(harnessProfile?.settings);
|
|
137
|
-
|
|
138
|
-
if (options.runId) {
|
|
139
|
-
return this.revisePlanGoal(options.runId, plan, {
|
|
140
|
-
goal: options.goal,
|
|
141
|
-
userId: options.userId,
|
|
142
|
-
metadata: options.metadata,
|
|
143
|
-
planSource:
|
|
144
|
-
options.planSource || (Array.isArray(options.plan) && options.plan.length ? 'provided' : 'template'),
|
|
145
|
-
plannerModel: options.plannerModel,
|
|
146
|
-
harnessTag,
|
|
147
|
-
harnessProfileId: harnessProfile?.id,
|
|
148
|
-
harnessSettings,
|
|
149
|
-
});
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
const snapshot = await this.createRun({
|
|
153
|
-
goal: options.goal,
|
|
154
|
-
leaderUsername: options.leaderUsername,
|
|
155
|
-
sessionId: options.sessionId,
|
|
156
|
-
messageId: options.messageId,
|
|
157
|
-
userId: options.userId,
|
|
158
|
-
policy: options.policy,
|
|
159
|
-
metadata: {
|
|
160
|
-
...asObject(options.metadata),
|
|
161
|
-
harnessTag,
|
|
162
|
-
harnessProfileId: harnessProfile?.id,
|
|
163
|
-
harnessSettings,
|
|
164
|
-
approvalMode: 'plan_first',
|
|
165
|
-
},
|
|
166
|
-
plan: [],
|
|
167
|
-
});
|
|
168
|
-
|
|
169
|
-
const runId = snapshot.run.id;
|
|
170
|
-
await this.replacePlan(runId, plan, {
|
|
171
|
-
userId: options.userId,
|
|
172
|
-
mode: 'append',
|
|
173
|
-
markRunning: false,
|
|
174
|
-
});
|
|
175
|
-
|
|
176
|
-
await this.repository.updateRun(runId, {
|
|
177
|
-
status: 'waiting_plan_approval',
|
|
178
|
-
approvalStatus: 'pending',
|
|
179
|
-
planVersion: 1,
|
|
180
|
-
planSource: options.planSource || (Array.isArray(options.plan) && options.plan.length ? 'provided' : 'template'),
|
|
181
|
-
plannerModel: options.plannerModel || '',
|
|
182
|
-
updatedAt: now(),
|
|
183
|
-
});
|
|
184
|
-
|
|
185
|
-
await this.repository.createEvent({
|
|
186
|
-
runId,
|
|
187
|
-
type: 'plan_approval_requested',
|
|
188
|
-
title: 'Plan approval requested',
|
|
189
|
-
content: `Waiting for user approval before executing ${plan.length} step(s).`,
|
|
190
|
-
status: 'waiting_plan_approval',
|
|
191
|
-
userId: options.userId,
|
|
192
|
-
payload: {
|
|
193
|
-
planVersion: 1,
|
|
194
|
-
harnessTag,
|
|
195
|
-
steps: plan.map((step, index) => ({
|
|
196
|
-
planKey: normalizePlanKey(step, index),
|
|
197
|
-
title: step.title || `Step ${index + 1}`,
|
|
198
|
-
type: normalizeStepType(step.type),
|
|
199
|
-
target: step.target || '',
|
|
200
|
-
dependsOn: asArray(step.dependsOn).map(String),
|
|
201
|
-
})),
|
|
202
|
-
},
|
|
203
|
-
});
|
|
204
|
-
|
|
205
|
-
return this.getRunDetail(runId);
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
async revisePlanGoal(
|
|
209
|
-
runId: string | number,
|
|
210
|
-
plan: AgentLoopPlanStepInput[],
|
|
211
|
-
options: {
|
|
212
|
-
goal?: string;
|
|
213
|
-
userId?: string | number;
|
|
214
|
-
metadata?: any;
|
|
215
|
-
planSource?: string;
|
|
216
|
-
plannerModel?: string;
|
|
217
|
-
harnessTag?: string;
|
|
218
|
-
harnessProfileId?: string | number;
|
|
219
|
-
harnessSettings?: any;
|
|
220
|
-
} = {},
|
|
221
|
-
) {
|
|
222
|
-
this.validator.validate(plan);
|
|
223
|
-
const run = await this.repository.requireRun(runId);
|
|
224
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
225
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
226
|
-
}
|
|
227
|
-
if (!['waiting_plan_approval', 'needs_replan'].includes(run.status)) {
|
|
228
|
-
throw new Error(`Run ${run.id} is not waiting for plan revision.`);
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
await this.replacePlan(run.id, plan, {
|
|
232
|
-
userId: options.userId,
|
|
233
|
-
mode: 'replace_pending',
|
|
234
|
-
reason: 'Plan revised',
|
|
235
|
-
markRunning: false,
|
|
236
|
-
});
|
|
237
|
-
|
|
238
|
-
const nextPlanVersion = Number(run.planVersion || 1) + 1;
|
|
239
|
-
await this.repository.updateRun(run.id, {
|
|
240
|
-
goal: options.goal || run.goal,
|
|
241
|
-
status: 'waiting_plan_approval',
|
|
242
|
-
approvalStatus: 'pending',
|
|
243
|
-
planVersion: nextPlanVersion,
|
|
244
|
-
planSource: options.planSource || run.planSource || 'provided',
|
|
245
|
-
plannerModel: options.plannerModel || run.plannerModel || '',
|
|
246
|
-
rejectionReason: '',
|
|
247
|
-
changeRequest: '',
|
|
248
|
-
metadata: {
|
|
249
|
-
...asObject(run.metadata),
|
|
250
|
-
...asObject(options.metadata),
|
|
251
|
-
harnessTag: options.harnessTag || asObject(run.metadata).harnessTag || 'default',
|
|
252
|
-
harnessProfileId: options.harnessProfileId || asObject(run.metadata).harnessProfileId,
|
|
253
|
-
harnessSettings: asObject(options.harnessSettings || asObject(run.metadata).harnessSettings),
|
|
254
|
-
approvalMode: 'plan_first',
|
|
255
|
-
},
|
|
256
|
-
updatedAt: now(),
|
|
257
|
-
});
|
|
258
|
-
|
|
259
|
-
await this.repository.createEvent({
|
|
260
|
-
runId: run.id,
|
|
261
|
-
type: 'plan_revision_requested',
|
|
262
|
-
title: 'Plan revised for approval',
|
|
263
|
-
content: `Waiting for approval of plan version ${nextPlanVersion}.`,
|
|
264
|
-
status: 'waiting_plan_approval',
|
|
265
|
-
userId: options.userId,
|
|
266
|
-
payload: {
|
|
267
|
-
planVersion: nextPlanVersion,
|
|
268
|
-
steps: plan.map((step, index) => normalizePlanKey(step, index)),
|
|
269
|
-
},
|
|
270
|
-
});
|
|
271
|
-
|
|
272
|
-
return this.getRunDetail(run.id);
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
async approvePlan(
|
|
276
|
-
runId: string | number,
|
|
277
|
-
options: { userId?: string | number; reason?: string } = {},
|
|
278
|
-
) {
|
|
279
|
-
const run = await this.repository.requireRun(runId);
|
|
280
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
281
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
282
|
-
}
|
|
283
|
-
if (!['waiting_plan_approval', 'approved', 'needs_replan'].includes(run.status)) {
|
|
284
|
-
throw new Error(`Run ${run.id} is not waiting for plan approval.`);
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
await this.repository.updateRun(run.id, {
|
|
288
|
-
status: 'approved',
|
|
289
|
-
approvalStatus: 'approved',
|
|
290
|
-
approvedById: options.userId,
|
|
291
|
-
approvedAt: now(),
|
|
292
|
-
rejectionReason: '',
|
|
293
|
-
changeRequest: '',
|
|
294
|
-
updatedAt: now(),
|
|
295
|
-
});
|
|
296
|
-
|
|
297
|
-
await this.repository.createEvent({
|
|
298
|
-
runId: run.id,
|
|
299
|
-
type: 'plan_approved',
|
|
300
|
-
title: 'Plan approved',
|
|
301
|
-
content: options.reason || '',
|
|
302
|
-
status: 'approved',
|
|
303
|
-
userId: options.userId,
|
|
304
|
-
payload: { planVersion: run.planVersion || 1 },
|
|
305
|
-
});
|
|
306
|
-
|
|
307
|
-
return this.getRunDetail(run.id);
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
/**
|
|
311
|
-
* @deprecated Use approvePlan() instead. Auto-execution via AgentHarness is retired;
|
|
312
|
-
* step execution is now driven by the AI via agent_loop_update_step tools.
|
|
313
|
-
*/
|
|
314
|
-
async approvePlanAndExecute(
|
|
315
|
-
runId: string | number,
|
|
316
|
-
options: { userId?: string | number; ctx?: any; reason?: string } = {},
|
|
317
|
-
) {
|
|
318
|
-
const run = await this.repository.requireRun(runId);
|
|
319
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
320
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
321
|
-
}
|
|
322
|
-
if (!['waiting_plan_approval', 'approved'].includes(run.status)) {
|
|
323
|
-
throw new Error(`Run ${run.id} is not waiting for plan approval.`);
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
await this.repository.updateRun(run.id, {
|
|
327
|
-
status: 'approved',
|
|
328
|
-
approvalStatus: 'approved',
|
|
329
|
-
approvedById: options.userId,
|
|
330
|
-
approvedAt: now(),
|
|
331
|
-
rejectionReason: '',
|
|
332
|
-
changeRequest: '',
|
|
333
|
-
updatedAt: now(),
|
|
334
|
-
});
|
|
335
|
-
|
|
336
|
-
await this.repository.createEvent({
|
|
337
|
-
runId: run.id,
|
|
338
|
-
type: 'plan_approved',
|
|
339
|
-
title: 'Plan approved',
|
|
340
|
-
content: options.reason || '',
|
|
341
|
-
status: 'approved',
|
|
342
|
-
userId: options.userId,
|
|
343
|
-
payload: { planVersion: run.planVersion || 1 },
|
|
344
|
-
});
|
|
345
|
-
|
|
346
|
-
return this.executeApprovedPlan(run.id, options);
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
async rejectPlan(runId: string | number, options: { userId?: string | number; reason?: string } = {}) {
|
|
350
|
-
const run = await this.repository.requireRun(runId);
|
|
351
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
352
|
-
return this.getRunSnapshot(run.id);
|
|
353
|
-
}
|
|
354
|
-
if (!['waiting_plan_approval', 'approved', 'needs_replan'].includes(run.status)) {
|
|
355
|
-
throw new Error(`Run ${run.id} is not waiting for plan approval.`);
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
const steps = await this.repository.getSteps(run.id);
|
|
359
|
-
for (const step of steps) {
|
|
360
|
-
if (!TERMINAL_STEP_STATUSES.has(step.status) && step.status !== 'failed') {
|
|
361
|
-
await this.repository.updateStep(step.id, {
|
|
362
|
-
status: 'skipped',
|
|
363
|
-
error: options.reason || 'Plan rejected by user.',
|
|
364
|
-
endedAt: now(),
|
|
365
|
-
updatedAt: now(),
|
|
366
|
-
});
|
|
367
|
-
}
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
await this.repository.updateRun(run.id, {
|
|
371
|
-
status: 'rejected',
|
|
372
|
-
approvalStatus: 'rejected',
|
|
373
|
-
rejectionReason: options.reason || '',
|
|
374
|
-
currentStepId: null,
|
|
375
|
-
endedAt: now(),
|
|
376
|
-
updatedAt: now(),
|
|
377
|
-
});
|
|
378
|
-
|
|
379
|
-
await this.repository.createEvent({
|
|
380
|
-
runId: run.id,
|
|
381
|
-
type: 'plan_rejected',
|
|
382
|
-
title: 'Plan rejected',
|
|
383
|
-
content: options.reason || '',
|
|
384
|
-
status: 'rejected',
|
|
385
|
-
userId: options.userId,
|
|
386
|
-
});
|
|
387
|
-
|
|
388
|
-
return this.getRunSnapshot(run.id);
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
async requestPlanChanges(runId: string | number, options: { userId?: string | number; feedback?: string } = {}) {
|
|
392
|
-
const run = await this.repository.requireRun(runId);
|
|
393
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
394
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
395
|
-
}
|
|
396
|
-
if (!['waiting_plan_approval', 'needs_replan'].includes(run.status)) {
|
|
397
|
-
throw new Error(`Run ${run.id} is not waiting for plan changes.`);
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
await this.repository.updateRun(run.id, {
|
|
401
|
-
status: 'needs_replan',
|
|
402
|
-
approvalStatus: 'changes_requested',
|
|
403
|
-
changeRequest: options.feedback || '',
|
|
404
|
-
updatedAt: now(),
|
|
405
|
-
});
|
|
406
|
-
|
|
407
|
-
await this.repository.createEvent({
|
|
408
|
-
runId: run.id,
|
|
409
|
-
type: 'plan_changes_requested',
|
|
410
|
-
title: 'Plan changes requested',
|
|
411
|
-
content: options.feedback || '',
|
|
412
|
-
status: 'needs_replan',
|
|
413
|
-
userId: options.userId,
|
|
414
|
-
});
|
|
415
|
-
|
|
416
|
-
return this.getRunDetail(run.id);
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
async replacePlan(
|
|
420
|
-
runId: string | number,
|
|
421
|
-
plan: AgentLoopPlanStepInput[],
|
|
422
|
-
options: {
|
|
423
|
-
userId?: string | number;
|
|
424
|
-
mode?: 'append' | 'replace_pending';
|
|
425
|
-
reason?: string;
|
|
426
|
-
markRunning?: boolean;
|
|
427
|
-
} = {},
|
|
428
|
-
) {
|
|
429
|
-
if (!Array.isArray(plan) || plan.length === 0) {
|
|
430
|
-
throw new Error('Plan must include at least one step.');
|
|
431
|
-
}
|
|
432
|
-
this.validator.validate(plan);
|
|
433
|
-
|
|
434
|
-
const run = await this.repository.requireRun(runId);
|
|
435
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
436
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
const policy = normalizePolicy(run.policy);
|
|
440
|
-
|
|
441
|
-
if (options.mode === 'replace_pending') {
|
|
442
|
-
const existing = await this.repository.getSteps(run.id);
|
|
443
|
-
for (const step of existing) {
|
|
444
|
-
if (!TERMINAL_STEP_STATUSES.has(step.status) && step.status !== 'running') {
|
|
445
|
-
await this.repository.updateStep(step.id, {
|
|
446
|
-
status: 'skipped',
|
|
447
|
-
error: options.reason || 'Replanned',
|
|
448
|
-
endedAt: now(),
|
|
449
|
-
});
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
const existingSteps = await this.repository.getSteps(run.id);
|
|
455
|
-
const indexStart = existingSteps.reduce((max, step) => Math.max(max, Number(step.index || 0)), -1) + 1;
|
|
456
|
-
const createdSteps = [];
|
|
457
|
-
|
|
458
|
-
for (let i = 0; i < plan.length; i++) {
|
|
459
|
-
const step = plan[i] || {};
|
|
460
|
-
const created = await this.repository.createStep({
|
|
461
|
-
runId: run.id,
|
|
462
|
-
parentStepId: step.parentStepId,
|
|
463
|
-
planKey: normalizePlanKey(step, i),
|
|
464
|
-
index: indexStart + i,
|
|
465
|
-
title: step.title || `Step ${indexStart + i + 1}`,
|
|
466
|
-
description: step.description || '',
|
|
467
|
-
type: normalizeStepType(step.type),
|
|
468
|
-
target: step.target || '',
|
|
469
|
-
input: step.input || {},
|
|
470
|
-
output: {},
|
|
471
|
-
status: 'pending',
|
|
472
|
-
attempt: 0,
|
|
473
|
-
maxAttempts: step.maxAttempts || policy.maxStepAttempts,
|
|
474
|
-
dependsOn: asArray(step.dependsOn).map(String),
|
|
475
|
-
dependencyPolicy: step.dependencyPolicy || step.metadata?.dependencyPolicy || 'require_success',
|
|
476
|
-
metadata: asObject(step.metadata),
|
|
477
|
-
});
|
|
478
|
-
createdSteps.push(created);
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
const nextStatus = options.markRunning === false ? run.status : 'running';
|
|
482
|
-
await this.repository.updateRun(run.id, {
|
|
483
|
-
status: nextStatus,
|
|
484
|
-
updatedAt: now(),
|
|
485
|
-
});
|
|
486
|
-
|
|
487
|
-
await this.repository.createEvent({
|
|
488
|
-
runId: run.id,
|
|
489
|
-
type: options.reason ? 'replanned' : 'planned',
|
|
490
|
-
title: options.reason ? 'Plan updated' : 'Plan created',
|
|
491
|
-
content: options.reason || `Created ${createdSteps.length} step(s).`,
|
|
492
|
-
status: nextStatus,
|
|
493
|
-
userId: options.userId,
|
|
494
|
-
payload: { mode: options.mode || 'append', steps: createdSteps.map((step) => step.planKey) },
|
|
495
|
-
});
|
|
496
|
-
|
|
497
|
-
return createdSteps;
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
async replan(
|
|
501
|
-
runId: string | number,
|
|
502
|
-
plan: AgentLoopPlanStepInput[],
|
|
503
|
-
options: { reason?: string; mode?: 'append' | 'replace_pending'; userId?: string | number } = {},
|
|
504
|
-
) {
|
|
505
|
-
if (!Array.isArray(plan) || plan.length === 0) {
|
|
506
|
-
throw new Error('Plan must include at least one step.');
|
|
507
|
-
}
|
|
508
|
-
|
|
509
|
-
const run = await this.repository.requireRun(runId);
|
|
510
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
511
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
const policy = normalizePolicy(run.policy);
|
|
515
|
-
if (!policy.allowReplan) {
|
|
516
|
-
throw new Error('Replanning is disabled for this run.');
|
|
517
|
-
}
|
|
518
|
-
if (Number(run.iterationCount || 0) >= policy.maxIterations) {
|
|
519
|
-
throw new Error(`Agent loop reached maxIterations=${policy.maxIterations}.`);
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
await this.repository.updateRun(run.id, {
|
|
523
|
-
iterationCount: Number(run.iterationCount || 0) + 1,
|
|
524
|
-
status: 'running',
|
|
525
|
-
updatedAt: now(),
|
|
526
|
-
});
|
|
527
|
-
|
|
528
|
-
return this.replacePlan(run.id, plan, {
|
|
529
|
-
mode: options.mode || 'replace_pending',
|
|
530
|
-
reason: options.reason || 'Replan requested',
|
|
531
|
-
userId: options.userId,
|
|
532
|
-
});
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
async startStep(
|
|
536
|
-
stepId: string | number,
|
|
537
|
-
options: { userId?: string | number; agentExecutionSpanId?: string | number } = {},
|
|
538
|
-
) {
|
|
539
|
-
const step = await this.repository.requireStep(stepId);
|
|
540
|
-
const run = await this.repository.requireRun(step.runId);
|
|
541
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
542
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
if (step.status !== 'pending' && step.status !== 'failed') {
|
|
546
|
-
throw new Error(`Step ${step.id} cannot start from status "${step.status}".`);
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
const policy = normalizePolicy(run.policy);
|
|
550
|
-
if (Number(step.attempt || 0) >= Number(step.maxAttempts || policy.maxStepAttempts)) {
|
|
551
|
-
throw new Error(`Step ${step.id} reached maxAttempts=${step.maxAttempts}.`);
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
const nextAttempt = Number(step.attempt || 0) + 1;
|
|
555
|
-
await this.repository.updateStep(step.id, {
|
|
556
|
-
status: 'running',
|
|
557
|
-
attempt: nextAttempt,
|
|
558
|
-
error: '',
|
|
559
|
-
agentExecutionSpanId: options.agentExecutionSpanId || step.agentExecutionSpanId,
|
|
560
|
-
startedAt: now(),
|
|
561
|
-
updatedAt: now(),
|
|
562
|
-
});
|
|
563
|
-
|
|
564
|
-
await this.repository.updateRun(run.id, {
|
|
565
|
-
status: 'running',
|
|
566
|
-
updatedAt: now(),
|
|
567
|
-
});
|
|
568
|
-
|
|
569
|
-
await this.repository.createEvent({
|
|
570
|
-
runId: run.id,
|
|
571
|
-
stepId: step.id,
|
|
572
|
-
type: 'step_started',
|
|
573
|
-
title: `Started: ${step.title || step.planKey}`,
|
|
574
|
-
status: 'running',
|
|
575
|
-
userId: options.userId,
|
|
576
|
-
payload: { attempt: nextAttempt },
|
|
577
|
-
});
|
|
578
|
-
|
|
579
|
-
if (['skill', 'tool', 'sub_agent'].includes(step.type)) {
|
|
580
|
-
await this.repository.createEvent({
|
|
581
|
-
runId: run.id,
|
|
582
|
-
stepId: step.id,
|
|
583
|
-
type: 'tool_called',
|
|
584
|
-
title: `Calling ${step.type}: ${step.target || step.title || step.planKey}`,
|
|
585
|
-
status: 'running',
|
|
586
|
-
userId: options.userId,
|
|
587
|
-
payload: {
|
|
588
|
-
type: step.type,
|
|
589
|
-
target: step.target,
|
|
590
|
-
attempt: nextAttempt,
|
|
591
|
-
},
|
|
592
|
-
});
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
return this.getRunSnapshot(run.id);
|
|
596
|
-
}
|
|
597
|
-
|
|
598
|
-
async completeStep(
|
|
599
|
-
stepId: string | number,
|
|
600
|
-
output: any,
|
|
601
|
-
options: {
|
|
602
|
-
userId?: string | number;
|
|
603
|
-
skillExecutionId?: string | number;
|
|
604
|
-
agentExecutionSpanId?: string | number;
|
|
605
|
-
metadata?: any;
|
|
606
|
-
} = {},
|
|
607
|
-
) {
|
|
608
|
-
const step = await this.repository.requireStep(stepId);
|
|
609
|
-
const run = await this.repository.requireRun(step.runId);
|
|
610
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
611
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
if (step.status !== 'running') {
|
|
615
|
-
throw new Error(`Step ${step.id} cannot complete from status "${step.status}".`);
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
await this.repository.updateStep(step.id, {
|
|
619
|
-
status: 'succeeded',
|
|
620
|
-
output: output === undefined ? {} : output,
|
|
621
|
-
error: '',
|
|
622
|
-
skillExecutionId: options.skillExecutionId || step.skillExecutionId,
|
|
623
|
-
agentExecutionSpanId: options.agentExecutionSpanId || step.agentExecutionSpanId,
|
|
624
|
-
metadata: { ...asObject(step.metadata), ...asObject(options.metadata) },
|
|
625
|
-
endedAt: now(),
|
|
626
|
-
updatedAt: now(),
|
|
627
|
-
});
|
|
628
|
-
|
|
629
|
-
await this.repository.updateRun(run.id, {
|
|
630
|
-
status: 'running',
|
|
631
|
-
updatedAt: now(),
|
|
632
|
-
});
|
|
633
|
-
|
|
634
|
-
await this.repository.createEvent({
|
|
635
|
-
runId: run.id,
|
|
636
|
-
stepId: step.id,
|
|
637
|
-
type: 'step_succeeded',
|
|
638
|
-
title: `Completed: ${step.title || step.planKey}`,
|
|
639
|
-
content: trimText(output, 2000),
|
|
640
|
-
status: 'succeeded',
|
|
641
|
-
userId: options.userId,
|
|
642
|
-
});
|
|
643
|
-
|
|
644
|
-
// Circuit breaker: record success for sub_agent steps
|
|
645
|
-
if (step.type === 'sub_agent' && step.target && run.leaderUsername) {
|
|
646
|
-
getCircuitBreaker().recordSuccess(subAgentCircuitKey(run.leaderUsername, step.target));
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
return this.getRunSnapshot(run.id);
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
async failStep(stepId: string | number, error: string, options: { userId?: string | number; metadata?: any } = {}) {
|
|
653
|
-
const step = await this.repository.requireStep(stepId);
|
|
654
|
-
const run = await this.repository.requireRun(step.runId);
|
|
655
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
656
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
657
|
-
}
|
|
658
|
-
|
|
659
|
-
if (step.status !== 'running') {
|
|
660
|
-
throw new Error(`Step ${step.id} cannot fail from status "${step.status}".`);
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
const policy = normalizePolicy(run.policy);
|
|
664
|
-
|
|
665
|
-
await this.repository.updateStep(step.id, {
|
|
666
|
-
status: 'failed',
|
|
667
|
-
error: trimText(error, 10000),
|
|
668
|
-
metadata: { ...asObject(step.metadata), ...asObject(options.metadata) },
|
|
669
|
-
endedAt: now(),
|
|
670
|
-
updatedAt: now(),
|
|
671
|
-
});
|
|
672
|
-
|
|
673
|
-
await this.repository.updateRun(run.id, {
|
|
674
|
-
status: 'running',
|
|
675
|
-
updatedAt: now(),
|
|
676
|
-
});
|
|
677
|
-
|
|
678
|
-
await this.repository.createEvent({
|
|
679
|
-
runId: run.id,
|
|
680
|
-
stepId: step.id,
|
|
681
|
-
type: 'step_failed',
|
|
682
|
-
title: `Failed: ${step.title || step.planKey}`,
|
|
683
|
-
content: error,
|
|
684
|
-
status: 'failed',
|
|
685
|
-
userId: options.userId,
|
|
686
|
-
payload: {
|
|
687
|
-
retryable: Number(step.attempt || 0) < Number(step.maxAttempts || policy.maxStepAttempts),
|
|
688
|
-
},
|
|
689
|
-
});
|
|
690
|
-
|
|
691
|
-
// Circuit breaker: record failure for sub_agent steps
|
|
692
|
-
if (step.type === 'sub_agent' && step.target && run.leaderUsername) {
|
|
693
|
-
getCircuitBreaker().recordFailure(subAgentCircuitKey(run.leaderUsername, step.target));
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
return this.getRunSnapshot(run.id);
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
async skipStep(stepId: string | number, reason = 'Skipped', options: { userId?: string | number } = {}) {
|
|
700
|
-
const step = await this.repository.requireStep(stepId);
|
|
701
|
-
const run = await this.repository.requireRun(step.runId);
|
|
702
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
703
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
if (!['pending', 'running', 'failed'].includes(step.status)) {
|
|
707
|
-
throw new Error(`Step ${step.id} cannot skip from status "${step.status}".`);
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
await this.repository.updateStep(step.id, {
|
|
711
|
-
status: 'skipped',
|
|
712
|
-
error: reason,
|
|
713
|
-
endedAt: now(),
|
|
714
|
-
updatedAt: now(),
|
|
715
|
-
});
|
|
716
|
-
|
|
717
|
-
await this.repository.createEvent({
|
|
718
|
-
runId: run.id,
|
|
719
|
-
stepId: step.id,
|
|
720
|
-
type: 'step_skipped',
|
|
721
|
-
title: `Skipped: ${step.title || step.planKey}`,
|
|
722
|
-
content: reason,
|
|
723
|
-
status: 'skipped',
|
|
724
|
-
userId: options.userId,
|
|
725
|
-
});
|
|
726
|
-
|
|
727
|
-
return this.getRunSnapshot(run.id);
|
|
728
|
-
}
|
|
729
|
-
|
|
730
|
-
async requestApproval(
|
|
731
|
-
stepId: string | number,
|
|
732
|
-
approval: any,
|
|
733
|
-
options: { userId?: string | number; reason?: string } = {},
|
|
734
|
-
) {
|
|
735
|
-
const step = await this.repository.requireStep(stepId);
|
|
736
|
-
const run = await this.repository.requireRun(step.runId);
|
|
737
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
738
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
739
|
-
}
|
|
740
|
-
|
|
741
|
-
if (!['pending', 'running'].includes(step.status)) {
|
|
742
|
-
throw new Error(`Step ${step.id} cannot request approval for status "${step.status}".`);
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
await this.repository.updateStep(step.id, {
|
|
746
|
-
status: 'waiting_user',
|
|
747
|
-
approval: approval || {},
|
|
748
|
-
updatedAt: now(),
|
|
749
|
-
});
|
|
750
|
-
|
|
751
|
-
await this.repository.updateRun(run.id, {
|
|
752
|
-
status: 'waiting_user',
|
|
753
|
-
currentStepId: step.id,
|
|
754
|
-
updatedAt: now(),
|
|
755
|
-
});
|
|
756
|
-
|
|
757
|
-
await this.repository.createEvent({
|
|
758
|
-
runId: run.id,
|
|
759
|
-
stepId: step.id,
|
|
760
|
-
type: 'approval_requested',
|
|
761
|
-
title: `Approval requested: ${step.title || step.planKey}`,
|
|
762
|
-
content: options.reason || approval?.prompt || '',
|
|
763
|
-
status: 'waiting_user',
|
|
764
|
-
userId: options.userId,
|
|
765
|
-
payload: approval || {},
|
|
766
|
-
});
|
|
767
|
-
|
|
768
|
-
return this.getRunSnapshot(run.id);
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
async resumeRun(
|
|
772
|
-
runId: string | number,
|
|
773
|
-
options: {
|
|
774
|
-
stepId?: string | number;
|
|
775
|
-
approved: boolean;
|
|
776
|
-
editedInput?: any;
|
|
777
|
-
userId?: string | number;
|
|
778
|
-
ctx?: any;
|
|
779
|
-
},
|
|
780
|
-
) {
|
|
781
|
-
const run = await this.repository.requireRun(runId);
|
|
782
|
-
const stepId = options.stepId || run.currentStepId;
|
|
783
|
-
if (!stepId) {
|
|
784
|
-
throw new Error('No waiting step found for this run.');
|
|
785
|
-
}
|
|
786
|
-
const step = await this.repository.requireStep(stepId);
|
|
787
|
-
if (String(step.runId) !== String(run.id)) {
|
|
788
|
-
throw new Error('Step does not belong to the run.');
|
|
789
|
-
}
|
|
790
|
-
if (run.status !== 'waiting_user' || step.status !== 'waiting_user') {
|
|
791
|
-
throw new Error('Run is not waiting for user approval.');
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
if (options.approved) {
|
|
795
|
-
const nextValues: any = {
|
|
796
|
-
status: 'pending',
|
|
797
|
-
approval: { ...(step.approval || {}), approved: true, resolvedAt: now().toISOString() },
|
|
798
|
-
updatedAt: now(),
|
|
799
|
-
};
|
|
800
|
-
if (options.editedInput !== undefined) {
|
|
801
|
-
nextValues.input = options.editedInput;
|
|
802
|
-
}
|
|
803
|
-
await this.repository.updateStep(step.id, nextValues);
|
|
804
|
-
await this.repository.updateRun(run.id, {
|
|
805
|
-
status: 'running',
|
|
806
|
-
currentStepId: null,
|
|
807
|
-
updatedAt: now(),
|
|
808
|
-
});
|
|
809
|
-
} else {
|
|
810
|
-
await this.repository.updateStep(step.id, {
|
|
811
|
-
status: 'failed',
|
|
812
|
-
approval: { ...(step.approval || {}), approved: false, resolvedAt: now().toISOString() },
|
|
813
|
-
error: 'User rejected this step.',
|
|
814
|
-
endedAt: now(),
|
|
815
|
-
updatedAt: now(),
|
|
816
|
-
});
|
|
817
|
-
await this.repository.updateRun(run.id, {
|
|
818
|
-
status: 'running',
|
|
819
|
-
currentStepId: null,
|
|
820
|
-
updatedAt: now(),
|
|
821
|
-
});
|
|
822
|
-
}
|
|
823
|
-
|
|
824
|
-
await this.repository.createEvent({
|
|
825
|
-
runId: run.id,
|
|
826
|
-
stepId: step.id,
|
|
827
|
-
type: 'approval_resolved',
|
|
828
|
-
title: options.approved ? 'Approval granted' : 'Approval rejected',
|
|
829
|
-
status: options.approved ? 'running' : 'failed',
|
|
830
|
-
userId: options.userId,
|
|
831
|
-
payload: {
|
|
832
|
-
approved: options.approved,
|
|
833
|
-
editedInput: options.editedInput,
|
|
834
|
-
},
|
|
835
|
-
});
|
|
836
|
-
|
|
837
|
-
if (options.approved) {
|
|
838
|
-
return this.getRunSnapshot(run.id);
|
|
839
|
-
}
|
|
840
|
-
return this.getRunSnapshot(run.id);
|
|
841
|
-
}
|
|
842
|
-
|
|
843
|
-
async retryStep(stepId: string | number, options: { userId?: string | number } = {}) {
|
|
844
|
-
const step = await this.repository.requireStep(stepId);
|
|
845
|
-
const run = await this.repository.requireRun(step.runId);
|
|
846
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
847
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
848
|
-
}
|
|
849
|
-
|
|
850
|
-
if (step.status !== 'failed') {
|
|
851
|
-
throw new Error('Only failed steps can be retried.');
|
|
852
|
-
}
|
|
853
|
-
|
|
854
|
-
const policy = normalizePolicy(run.policy);
|
|
855
|
-
if (Number(step.attempt || 0) >= Number(step.maxAttempts || policy.maxStepAttempts)) {
|
|
856
|
-
throw new Error(`Step ${step.id} reached maxAttempts=${step.maxAttempts}.`);
|
|
857
|
-
}
|
|
858
|
-
|
|
859
|
-
// ── Smart retry routing (Phase 8b) ─────────────────────────────────
|
|
860
|
-
// For sub_agent steps, check circuit breaker state on the target.
|
|
861
|
-
// If the circuit is open or has multiple failures, route to an alternative.
|
|
862
|
-
let routedTarget: string | undefined;
|
|
863
|
-
let routeReason: string | undefined;
|
|
864
|
-
if (step.type === 'sub_agent' && run.leaderUsername) {
|
|
865
|
-
const target = step.target || '';
|
|
866
|
-
if (target) {
|
|
867
|
-
const circuitBreaker = getCircuitBreaker();
|
|
868
|
-
const state = circuitBreaker.getState(subAgentCircuitKey(run.leaderUsername, target));
|
|
869
|
-
const attempt = Number(step.attempt || 0);
|
|
870
|
-
|
|
871
|
-
// Route away if: circuit is open, or 2+ failures and at least 1 retry already attempted
|
|
872
|
-
const shouldRoute = state?.state === 'open' || (state && state.failures >= 2 && attempt >= 1);
|
|
873
|
-
|
|
874
|
-
if (shouldRoute) {
|
|
875
|
-
const alternatives = await this.registryService.findAlternativeSubAgents(run.leaderUsername, target);
|
|
876
|
-
if (alternatives.length > 0) {
|
|
877
|
-
const chosen = alternatives[0]; // pick first alternative
|
|
878
|
-
routedTarget = chosen.username;
|
|
879
|
-
routeReason =
|
|
880
|
-
`Sub-agent "${target}" has ${state?.failures || 0} failure(s) (circuit: ${state?.state || 'closed'}). ` +
|
|
881
|
-
`Routing retry to alternative "${routedTarget}".`;
|
|
882
|
-
}
|
|
883
|
-
}
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
|
|
887
|
-
const updateValues: any = {
|
|
888
|
-
status: 'pending',
|
|
889
|
-
error: '',
|
|
890
|
-
endedAt: null,
|
|
891
|
-
updatedAt: now(),
|
|
892
|
-
};
|
|
893
|
-
if (routedTarget) {
|
|
894
|
-
updateValues.target = routedTarget;
|
|
895
|
-
}
|
|
896
|
-
|
|
897
|
-
await this.repository.updateStep(step.id, updateValues);
|
|
898
|
-
|
|
899
|
-
await this.repository.updateRun(run.id, {
|
|
900
|
-
status: 'running',
|
|
901
|
-
updatedAt: now(),
|
|
902
|
-
});
|
|
903
|
-
|
|
904
|
-
await this.repository.createEvent({
|
|
905
|
-
runId: run.id,
|
|
906
|
-
stepId: step.id,
|
|
907
|
-
type: 'step_retry',
|
|
908
|
-
title: routedTarget
|
|
909
|
-
? `Retry routed: ${step.title || step.planKey} → ${routedTarget}`
|
|
910
|
-
: `Retry queued: ${step.title || step.planKey}`,
|
|
911
|
-
content: routeReason || '',
|
|
912
|
-
status: 'pending',
|
|
913
|
-
userId: options.userId,
|
|
914
|
-
payload: routedTarget ? { originalTarget: step.target, routedTarget, reason: routeReason } : undefined,
|
|
915
|
-
});
|
|
916
|
-
|
|
917
|
-
return this.getRunSnapshot(run.id);
|
|
918
|
-
}
|
|
919
|
-
|
|
920
|
-
async finishRun(
|
|
921
|
-
runId: string | number,
|
|
922
|
-
finalAnswer: string,
|
|
923
|
-
options: {
|
|
924
|
-
status?: Extract<AgentLoopRunStatus, 'succeeded' | 'failed'>;
|
|
925
|
-
summary?: string;
|
|
926
|
-
evidence?: any;
|
|
927
|
-
userId?: string | number;
|
|
928
|
-
} = {},
|
|
929
|
-
) {
|
|
930
|
-
const run = await this.repository.requireRun(runId);
|
|
931
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
932
|
-
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
933
|
-
}
|
|
934
|
-
|
|
935
|
-
const status = options.status || 'succeeded';
|
|
936
|
-
if (status === 'succeeded') {
|
|
937
|
-
const steps = await this.repository.getSteps(run.id);
|
|
938
|
-
const unfinished = steps.filter((step) => !TERMINAL_STEP_STATUSES.has(step.status));
|
|
939
|
-
if (unfinished.length) {
|
|
940
|
-
throw new Error(`Cannot finish run ${run.id}: ${unfinished.length} step(s) are not complete.`);
|
|
941
|
-
}
|
|
942
|
-
const policy = normalizePolicy(run.policy);
|
|
943
|
-
const verificationPassed = steps.some((step) => step.type === 'verification' && step.status === 'succeeded');
|
|
944
|
-
if (policy.requireVerification && !verificationPassed) {
|
|
945
|
-
throw new Error('Cannot finish run: policy requires a succeeded verification step.');
|
|
946
|
-
}
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
await this.repository.updateRun(run.id, {
|
|
950
|
-
status,
|
|
951
|
-
finalAnswer: finalAnswer || '',
|
|
952
|
-
summary: options.summary || run.summary,
|
|
953
|
-
currentStepId: null,
|
|
954
|
-
metadata: { ...asObject(run.metadata), evidence: options.evidence },
|
|
955
|
-
endedAt: now(),
|
|
956
|
-
updatedAt: now(),
|
|
957
|
-
});
|
|
958
|
-
|
|
959
|
-
await this.repository.createEvent({
|
|
960
|
-
runId: run.id,
|
|
961
|
-
type: 'finished',
|
|
962
|
-
title: status === 'succeeded' ? 'Agent loop finished' : 'Agent loop failed',
|
|
963
|
-
content: finalAnswer,
|
|
964
|
-
status,
|
|
965
|
-
userId: options.userId,
|
|
966
|
-
payload: { evidence: options.evidence },
|
|
967
|
-
});
|
|
968
|
-
|
|
969
|
-
return this.getRunSnapshot(run.id);
|
|
970
|
-
}
|
|
971
|
-
|
|
972
|
-
async stepFeedback(
|
|
973
|
-
stepId: string | number,
|
|
974
|
-
feedback: { rating: 'positive' | 'negative'; comment?: string; category?: string },
|
|
975
|
-
options: { userId?: string | number } = {},
|
|
976
|
-
) {
|
|
977
|
-
const step = await this.repository.requireStep(stepId);
|
|
978
|
-
const run = await this.repository.requireRun(step.runId);
|
|
979
|
-
|
|
980
|
-
if (step.status !== 'succeeded' && step.status !== 'failed') {
|
|
981
|
-
throw new Error(`Cannot provide feedback for step ${step.id} in status "${step.status}".`);
|
|
982
|
-
}
|
|
983
|
-
|
|
984
|
-
await this.repository.createEvent({
|
|
985
|
-
runId: run.id,
|
|
986
|
-
stepId: step.id,
|
|
987
|
-
type: 'step_feedback',
|
|
988
|
-
title: `Feedback: ${feedback.rating === 'positive' ? '👍' : '👎'} ${step.title || step.planKey}`,
|
|
989
|
-
content: feedback.comment || '',
|
|
990
|
-
status: step.status,
|
|
991
|
-
userId: options.userId,
|
|
992
|
-
payload: {
|
|
993
|
-
rating: feedback.rating,
|
|
994
|
-
comment: feedback.comment || '',
|
|
995
|
-
category: feedback.category || '',
|
|
996
|
-
stepType: step.type,
|
|
997
|
-
stepPlanKey: step.planKey,
|
|
998
|
-
stepTarget: step.target || '',
|
|
999
|
-
attempt: step.attempt || 0,
|
|
1000
|
-
},
|
|
1001
|
-
});
|
|
1002
|
-
|
|
1003
|
-
return this.getRunSnapshot(run.id);
|
|
1004
|
-
}
|
|
1005
|
-
|
|
1006
|
-
async cancelRun(runId: string | number, options: { userId?: string | number; reason?: string } = {}) {
|
|
1007
|
-
const run = await this.repository.requireRun(runId);
|
|
1008
|
-
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
1009
|
-
return this.getRunSnapshot(run.id);
|
|
1010
|
-
}
|
|
1011
|
-
|
|
1012
|
-
const steps = await this.repository.getSteps(run.id);
|
|
1013
|
-
for (const step of steps) {
|
|
1014
|
-
if (!TERMINAL_STEP_STATUSES.has(step.status) && step.status !== 'failed') {
|
|
1015
|
-
await this.repository.updateStep(step.id, {
|
|
1016
|
-
status: 'skipped',
|
|
1017
|
-
error: options.reason || 'Run canceled.',
|
|
1018
|
-
endedAt: now(),
|
|
1019
|
-
updatedAt: now(),
|
|
1020
|
-
});
|
|
1021
|
-
}
|
|
1022
|
-
}
|
|
1023
|
-
|
|
1024
|
-
await this.repository.updateRun(run.id, {
|
|
1025
|
-
status: 'canceled',
|
|
1026
|
-
currentStepId: null,
|
|
1027
|
-
endedAt: now(),
|
|
1028
|
-
updatedAt: now(),
|
|
1029
|
-
});
|
|
1030
|
-
|
|
1031
|
-
await this.repository.createEvent({
|
|
1032
|
-
runId: run.id,
|
|
1033
|
-
type: 'canceled',
|
|
1034
|
-
title: 'Agent loop canceled',
|
|
1035
|
-
content: options.reason || '',
|
|
1036
|
-
status: 'canceled',
|
|
1037
|
-
userId: options.userId,
|
|
1038
|
-
});
|
|
1039
|
-
|
|
1040
|
-
return this.getRunSnapshot(run.id);
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
async executeApprovedPlan(runId: string | number, options: { userId?: string | number; ctx?: any } = {}) {
|
|
1044
|
-
// Concurrency check using a unique process-specific database lock token
|
|
1045
|
-
const lockToken = `exec-${runId}-${Math.random().toString(36).slice(2, 10)}`;
|
|
1046
|
-
const acquired = await this.repository.lockRun(runId, lockToken, 300000); // 5 mins lock
|
|
1047
|
-
if (!acquired) {
|
|
1048
|
-
throw new Error(`Run ${runId} is currently locked and executing in another process.`);
|
|
1049
|
-
}
|
|
1050
|
-
|
|
1051
|
-
try {
|
|
1052
|
-
let iterations = 0;
|
|
1053
|
-
let snapshot = await this.getRunSnapshot(runId);
|
|
1054
|
-
const harnessSettings = asObject(snapshot.run?.metadata?.harnessSettings);
|
|
1055
|
-
const maxControllerSteps = Math.max(
|
|
1056
|
-
1,
|
|
1057
|
-
Math.min(
|
|
1058
|
-
ORCHESTRATOR_CONTROLLER_MAX_STEPS,
|
|
1059
|
-
Number(harnessSettings.maxControllerSteps || ORCHESTRATOR_CONTROLLER_MAX_STEPS),
|
|
1060
|
-
),
|
|
1061
|
-
);
|
|
1062
|
-
|
|
1063
|
-
const policy = normalizePolicy(snapshot.run.policy);
|
|
1064
|
-
const maxConcurrency = policy.maxConcurrency;
|
|
1065
|
-
|
|
1066
|
-
while (snapshot.nextSteps && snapshot.nextSteps.length > 0 && iterations < maxControllerSteps) {
|
|
1067
|
-
const batch = snapshot.nextSteps.slice(0, maxConcurrency);
|
|
1068
|
-
iterations += batch.length;
|
|
1069
|
-
|
|
1070
|
-
// Budget check before batch
|
|
1071
|
-
if (this.tokenTracker) {
|
|
1072
|
-
const budgetCheck = await this.tokenTracker.checkBudget(runId);
|
|
1073
|
-
if (!budgetCheck.allowed) {
|
|
1074
|
-
return this.finishRun(runId, budgetCheck.reason, {
|
|
1075
|
-
status: 'failed' as const,
|
|
1076
|
-
summary: budgetCheck.reason,
|
|
1077
|
-
userId: options.userId,
|
|
1078
|
-
});
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
1081
|
-
|
|
1082
|
-
// Start all steps concurrently
|
|
1083
|
-
await Promise.all(batch.map((step: any) => this.startStep(step.id, { userId: options.userId })));
|
|
1084
|
-
|
|
1085
|
-
// Refresh snapshot to get updated step states
|
|
1086
|
-
snapshot = await this.getRunSnapshot(runId);
|
|
1087
|
-
const runningSteps = batch.map(
|
|
1088
|
-
(step: any) => snapshot.steps.find((s: any) => String(s.id) === String(step.id)) || step,
|
|
1089
|
-
);
|
|
1090
|
-
|
|
1091
|
-
// Execute all steps concurrently via harness
|
|
1092
|
-
const results = await Promise.allSettled(
|
|
1093
|
-
runningSteps.map((step: any) => this.harness.executeStep(snapshot.run, step, options)),
|
|
1094
|
-
);
|
|
1095
|
-
|
|
1096
|
-
// Process results. Failed-but-retryable steps need a backoff before the
|
|
1097
|
-
// next iteration picks them up again. We compute the LARGEST backoff in
|
|
1098
|
-
// the batch and sleep once after the loop — never sum per-step sleeps.
|
|
1099
|
-
// Summing held the run-lock for up to maxConcurrency × 60s (= the whole
|
|
1100
|
-
// 5-minute lock when a full batch fails), starving other steps.
|
|
1101
|
-
let batchBackoffMs = 0;
|
|
1102
|
-
let approvalBreak = false;
|
|
1103
|
-
for (let i = 0; i < results.length; i++) {
|
|
1104
|
-
const runningStep = runningSteps[i];
|
|
1105
|
-
const result = results[i];
|
|
1106
|
-
|
|
1107
|
-
if (result.status === 'fulfilled') {
|
|
1108
|
-
snapshot = await this.completeStep(runningStep.id, result.value, {
|
|
1109
|
-
userId: options.userId,
|
|
1110
|
-
metadata: { controller: 'agent-loop-service' },
|
|
1111
|
-
});
|
|
1112
|
-
} else {
|
|
1113
|
-
const error = result.reason;
|
|
1114
|
-
if (error?.message === 'requires_approval') {
|
|
1115
|
-
snapshot = await this.requestApproval(
|
|
1116
|
-
runningStep.id,
|
|
1117
|
-
{ prompt: `Execution of step "${runningStep.title}" requires permission.` },
|
|
1118
|
-
{ userId: options.userId, reason: 'Dynamic tool approval required by policy.' },
|
|
1119
|
-
);
|
|
1120
|
-
// Stop processing more results; remaining steps are left pending
|
|
1121
|
-
approvalBreak = true;
|
|
1122
|
-
break;
|
|
1123
|
-
}
|
|
1124
|
-
|
|
1125
|
-
snapshot = await this.failStep(runningStep.id, error?.message || String(error), {
|
|
1126
|
-
userId: options.userId,
|
|
1127
|
-
metadata: { controller: 'agent-loop-service' },
|
|
1128
|
-
});
|
|
1129
|
-
const failedStep = snapshot.steps.find((s: any) => String(s.id) === String(runningStep.id));
|
|
1130
|
-
if (
|
|
1131
|
-
failedStep &&
|
|
1132
|
-
Number(failedStep.attempt || 0) < Number(failedStep.maxAttempts || policy.maxStepAttempts)
|
|
1133
|
-
) {
|
|
1134
|
-
// Exponential backoff — track the max; the step stays pending for retry.
|
|
1135
|
-
const attempt = Number(failedStep.attempt || 1);
|
|
1136
|
-
const baseDelay = 1000;
|
|
1137
|
-
const maxDelay = 60000;
|
|
1138
|
-
const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
|
|
1139
|
-
batchBackoffMs = Math.max(batchBackoffMs, delay);
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1142
|
-
}
|
|
1143
|
-
|
|
1144
|
-
// Single consolidated backoff for the whole batch (bounded at maxDelay).
|
|
1145
|
-
if (!approvalBreak && batchBackoffMs > 0) {
|
|
1146
|
-
await new Promise((resolve) => setTimeout(resolve, batchBackoffMs));
|
|
1147
|
-
}
|
|
1148
|
-
|
|
1149
|
-
// Re-evaluate the plan — newly unblocked steps become eligible
|
|
1150
|
-
snapshot = await this.getRunSnapshot(runId);
|
|
1151
|
-
|
|
1152
|
-
// If we broke for approval, exit loop
|
|
1153
|
-
if (snapshot.run.status === 'waiting_user') {
|
|
1154
|
-
break;
|
|
1155
|
-
}
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
snapshot = await this.getRunSnapshot(runId);
|
|
1159
|
-
if (iterations >= maxControllerSteps && snapshot.nextSteps && snapshot.nextSteps.length > 0) {
|
|
1160
|
-
return this.finishRun(runId, `Agent loop stopped after ${maxControllerSteps} controller steps.`, {
|
|
1161
|
-
status: 'failed',
|
|
1162
|
-
summary: 'Controller iteration limit reached.',
|
|
1163
|
-
userId: options.userId,
|
|
1164
|
-
});
|
|
1165
|
-
}
|
|
1166
|
-
|
|
1167
|
-
// If waiting for approval or user, exit loop but don't finish
|
|
1168
|
-
if (snapshot.run.status === 'waiting_user') {
|
|
1169
|
-
return snapshot;
|
|
1170
|
-
}
|
|
1171
|
-
|
|
1172
|
-
const steps = snapshot.steps || [];
|
|
1173
|
-
const failed = steps.filter((step: any) => step.status === 'failed');
|
|
1174
|
-
const unfinished = steps.filter(
|
|
1175
|
-
(step: any) => !TERMINAL_STEP_STATUSES.has(step.status) && step.status !== 'failed',
|
|
1176
|
-
);
|
|
1177
|
-
if (failed.length || unfinished.length) {
|
|
1178
|
-
return this.finishRun(
|
|
1179
|
-
runId,
|
|
1180
|
-
failed.length
|
|
1181
|
-
? `Agent loop failed at ${failed.length} step(s).`
|
|
1182
|
-
: `Agent loop stopped with ${unfinished.length} unfinished step(s).`,
|
|
1183
|
-
{
|
|
1184
|
-
status: 'failed',
|
|
1185
|
-
summary: failed[0]?.error || 'No executable step is available.',
|
|
1186
|
-
evidence: {
|
|
1187
|
-
failedStepIds: failed.map((step: any) => step.id),
|
|
1188
|
-
unfinishedStepIds: unfinished.map((step: any) => step.id),
|
|
1189
|
-
},
|
|
1190
|
-
userId: options.userId,
|
|
1191
|
-
},
|
|
1192
|
-
);
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
const verification = steps.find((step: any) => step.type === 'verification' && step.status === 'succeeded');
|
|
1196
|
-
return this.finishRun(runId, 'Agent loop completed after plan execution.', {
|
|
1197
|
-
status: 'succeeded',
|
|
1198
|
-
summary: verification?.output?.summary || 'All approved plan steps completed.',
|
|
1199
|
-
evidence: { stepCount: steps.length, controllerIterations: iterations },
|
|
1200
|
-
userId: options.userId,
|
|
1201
|
-
});
|
|
1202
|
-
} finally {
|
|
1203
|
-
// Unlock only if we held this specific lockToken
|
|
1204
|
-
const current = await this.repository.getRun(runId);
|
|
1205
|
-
if (current && current.lockedBy === lockToken) {
|
|
1206
|
-
await this.repository.unlockRun(runId);
|
|
1207
|
-
}
|
|
1208
|
-
}
|
|
1209
|
-
}
|
|
1210
|
-
|
|
1211
|
-
async getRunSnapshot(runId: string | number) {
|
|
1212
|
-
const run = await this.repository.requireRun(runId);
|
|
1213
|
-
const steps = await this.repository.getSteps(run.id);
|
|
1214
|
-
const nextSteps = this.pickNextSteps(steps, run.policy);
|
|
1215
|
-
const runningStepIds = steps.filter((s) => s.status === 'running').map((s) => s.id);
|
|
1216
|
-
return {
|
|
1217
|
-
run: { ...run, runningStepIds },
|
|
1218
|
-
steps,
|
|
1219
|
-
nextSteps,
|
|
1220
|
-
};
|
|
1221
|
-
}
|
|
1222
|
-
|
|
1223
|
-
async getRunDetail(runId: string | number) {
|
|
1224
|
-
const snapshot = await this.getRunSnapshot(runId);
|
|
1225
|
-
const events = await this.repository.getEvents(snapshot.run.id);
|
|
1226
|
-
const spans = await this.repository.getLinkedSpans(snapshot.run.id, snapshot.run.rootRunId);
|
|
1227
|
-
const skillExecutions = await this.repository.getLinkedSkillExecutions(snapshot.run.id, snapshot.steps);
|
|
1228
|
-
return {
|
|
1229
|
-
...snapshot,
|
|
1230
|
-
events,
|
|
1231
|
-
spans,
|
|
1232
|
-
skillExecutions,
|
|
1233
|
-
};
|
|
1234
|
-
}
|
|
1235
|
-
|
|
1236
|
-
pickNextSteps(steps: any[], runPolicy?: any) {
|
|
1237
|
-
const byPlanKey = new Map(steps.map((step) => [String(step.planKey), step]));
|
|
1238
|
-
const policy = normalizePolicy(runPolicy);
|
|
1239
|
-
|
|
1240
|
-
const candidates = steps
|
|
1241
|
-
.filter(
|
|
1242
|
-
(step) =>
|
|
1243
|
-
step.status === 'pending' ||
|
|
1244
|
-
(step.status === 'failed' && Number(step.attempt || 0) < Number(step.maxAttempts || policy.maxStepAttempts)),
|
|
1245
|
-
)
|
|
1246
|
-
.sort((a, b) => Number(a.index || 0) - Number(b.index || 0));
|
|
1247
|
-
|
|
1248
|
-
const ready: any[] = [];
|
|
1249
|
-
for (const step of candidates) {
|
|
1250
|
-
const dependencies = asArray(step.dependsOn).map(String);
|
|
1251
|
-
const allowSkipped =
|
|
1252
|
-
step.dependencyPolicy === 'allow_skipped' || step.metadata?.dependencyPolicy === 'allow_skipped';
|
|
1253
|
-
const allDepsReady = dependencies.every((key) => {
|
|
1254
|
-
const dependency = byPlanKey.get(key);
|
|
1255
|
-
return dependency?.status === 'succeeded' || (allowSkipped && dependency?.status === 'skipped');
|
|
1256
|
-
});
|
|
1257
|
-
if (allDepsReady) {
|
|
1258
|
-
ready.push(step);
|
|
1259
|
-
}
|
|
1260
|
-
}
|
|
1261
|
-
return ready;
|
|
1262
|
-
}
|
|
1263
|
-
}
|