plugin-agent-orchestrator 1.1.4 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -114
- package/dist/client/index.js +1 -1
- package/dist/client-v2/117.7565cff57c39318a.js +10 -0
- package/dist/client-v2/214.106635011859dede.js +10 -0
- package/dist/client-v2/418.41ed200e7d6da332.js +10 -0
- package/dist/client-v2/53.349fff8db80b19d4.js +10 -0
- package/dist/client-v2/981.e4a1f41cfc94a856.js +10 -0
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +7 -8
- package/dist/locale/en-US.json +226 -170
- package/dist/locale/vi-VN.json +226 -170
- package/dist/locale/zh-CN.json +226 -170
- package/dist/server/collections/agent-execution-spans.js +8 -0
- package/dist/server/collections/agent-loop-action-approvals.js +1 -0
- package/dist/server/collections/agent-loop-runs.js +7 -1
- package/dist/server/collections/agent-loop-steps.js +7 -0
- package/dist/server/collections/skill-executions.js +8 -0
- package/dist/server/plugin.js +3 -1
- package/dist/server/resources/agent-harness-profile.js +117 -0
- package/dist/server/resources/agent-knowledge-insights.js +1 -3
- package/dist/server/resources/agent-loop-approval.js +25 -0
- package/dist/server/resources/agent-loop.js +3 -1
- package/dist/server/services/AgentMemoryContextService.js +34 -0
- package/dist/server/services/HarnessCompiler.js +35 -2
- package/dist/server/services/HarnessProfileService.js +74 -9
- package/dist/server/services/HarnessSchema.js +30 -3
- package/dist/server/services/LoopPatternSchema.js +16 -1
- package/dist/server/services/LoopRunRepository.js +80 -0
- package/dist/server/services/LoopRunStateMachine.js +92 -3
- package/dist/server/services/LoopTriggerService.js +37 -0
- package/dist/server/services/LoopWorkerService.js +261 -23
- package/dist/server/services/NativeSubAgentObserver.js +165 -23
- package/dist/server/services/PluginAiRuntimeAdapter.js +16 -5
- package/dist/server/services/ToolLoopDetectionService.js +210 -0
- package/dist/server/services/VerificationService.js +66 -0
- package/dist/server/services/harness-runtime-policy.js +139 -0
- package/dist/server/skill-hub/plugin.js +36 -3
- package/dist/server/skill-hub/tasks/SkillExecutionTask.js +29 -0
- package/package.json +53 -53
- package/src/client/ApprovalsTab.tsx +2 -0
- package/src/client/OrchestratorSettings.tsx +11 -0
- package/src/client/skill-hub/components/GitSkillImport.tsx +3 -3
- package/src/client/skill-hub/components/SkillMetrics.tsx +15 -10
- package/src/client/skill-hub/index.tsx +0 -52
- package/src/client-v2/components/AgentRunsTab.tsx +576 -576
- package/src/client-v2/components/ApprovalsTab.tsx +337 -0
- package/src/client-v2/components/HarnessProfilesTab.tsx +385 -105
- package/src/client-v2/components/MemoryInspectorTab.tsx +141 -141
- package/src/client-v2/components/TracingTab.tsx +515 -515
- package/src/client-v2/hooks/useApiRequest.ts +141 -141
- package/src/client-v2/pages/ApprovalsPage.tsx +10 -0
- package/src/client-v2/plugin.tsx +6 -6
- package/src/client-v2/skill-hub/components/SkillMetrics.tsx +5 -2
- package/src/locale/en-US.json +226 -174
- package/src/locale/vi-VN.json +226 -174
- package/src/locale/zh-CN.json +226 -174
- package/src/server/__tests__/circuit-breaker.test.ts +248 -248
- package/src/server/__tests__/harness-compiler.test.ts +83 -0
- package/src/server/__tests__/harness-profile-service.test.ts +51 -0
- package/src/server/__tests__/harness-runtime-policy.test.ts +293 -0
- package/src/server/__tests__/harness-tool-plan.test.ts +24 -0
- package/src/server/__tests__/loop-approval-decision.test.ts +512 -0
- package/src/server/__tests__/loop-approval-expiry.test.ts +258 -0
- package/src/server/__tests__/loop-budget-service.test.ts +255 -255
- package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -266
- package/src/server/__tests__/loop-pattern-schema.test.ts +18 -1
- package/src/server/__tests__/loop-pattern-service.test.ts +148 -148
- package/src/server/__tests__/loop-run-state-machine.test.ts +11 -0
- package/src/server/__tests__/loop-trigger-service.test.ts +148 -0
- package/src/server/__tests__/loop-worker-claim.test.ts +341 -341
- package/src/server/__tests__/loop-worker-execution.test.ts +82 -1
- package/src/server/__tests__/native-sub-agent-observer.test.ts +288 -4
- package/src/server/__tests__/path-lock-service.test.ts +254 -254
- package/src/server/__tests__/skill-execution-task-policy.test.ts +140 -0
- package/src/server/__tests__/smoke.test.ts +162 -162
- package/src/server/__tests__/tool-loop-detection.test.ts +275 -0
- package/src/server/collections/agent-execution-spans.ts +8 -0
- package/src/server/collections/agent-harness-profile-versions.ts +26 -26
- package/src/server/collections/agent-harness-profiles.ts +71 -71
- package/src/server/collections/agent-loop-action-approvals.ts +1 -0
- package/src/server/collections/agent-loop-artifacts.ts +25 -25
- package/src/server/collections/agent-loop-circuit-states.ts +23 -23
- package/src/server/collections/agent-loop-control-settings.ts +21 -21
- package/src/server/collections/agent-loop-events.ts +98 -98
- package/src/server/collections/agent-loop-path-locks.ts +27 -27
- package/src/server/collections/agent-loop-patterns.ts +42 -42
- package/src/server/collections/agent-loop-runs.ts +7 -1
- package/src/server/collections/agent-loop-steps.ts +7 -0
- package/src/server/collections/agent-loop-usage-buckets.ts +22 -22
- package/src/server/collections/agent-loop-worktrees.ts +25 -25
- package/src/server/collections/skill-executions.ts +8 -0
- package/src/server/migrations/20260427000002-change-packages-to-text.ts +52 -52
- package/src/server/migrations/20260429000000-add-llm-fields.ts +47 -47
- package/src/server/migrations/20260429000001-fix-inputargs-json-to-text.ts +45 -45
- package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +33 -33
- package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +31 -31
- package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -293
- package/src/server/plugin.ts +3 -1
- package/src/server/resources/agent-harness-profile.ts +94 -0
- package/src/server/resources/agent-knowledge-insights.ts +5 -3
- package/src/server/resources/agent-loop-approval.ts +32 -0
- package/src/server/resources/agent-loop-artifact.ts +56 -56
- package/src/server/resources/agent-loop-events-stream.ts +105 -105
- package/src/server/resources/agent-loop.ts +3 -1
- package/src/server/resources/resource-helpers.ts +33 -33
- package/src/server/resources/tracing.ts +266 -266
- package/src/server/services/AgentMemoryContextService.ts +44 -0
- package/src/server/services/DurableCircuitBreakerService.ts +239 -239
- package/src/server/services/ExecutionSpanService.ts +2 -0
- package/src/server/services/HarnessCompiler.ts +37 -0
- package/src/server/services/HarnessProfileService.ts +92 -10
- package/src/server/services/HarnessSchema.ts +31 -0
- package/src/server/services/LoopBudgetService.ts +275 -275
- package/src/server/services/LoopControlService.ts +124 -124
- package/src/server/services/LoopPatternSchema.ts +19 -1
- package/src/server/services/LoopPatternService.ts +143 -143
- package/src/server/services/LoopRunRepository.ts +99 -0
- package/src/server/services/LoopRunStateMachine.ts +100 -3
- package/src/server/services/LoopSchedulerService.ts +199 -199
- package/src/server/services/LoopTriggerService.ts +42 -0
- package/src/server/services/LoopWorkerService.ts +327 -27
- package/src/server/services/NativeSubAgentObserver.ts +186 -21
- package/src/server/services/PathLockService.ts +321 -321
- package/src/server/services/PluginAiRuntimeAdapter.ts +24 -7
- package/src/server/services/ToolLoopDetectionService.ts +213 -0
- package/src/server/services/VerificationSchema.ts +31 -31
- package/src/server/services/VerificationService.ts +81 -2
- package/src/server/services/harness-runtime-policy.ts +141 -0
- package/src/server/skill-hub/plugin.ts +41 -3
- package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +235 -235
- package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +61 -61
- package/src/server/skill-hub/tasks/SkillExecutionTask.ts +43 -2
- package/src/server/utils/ensure-indexes.ts +96 -96
- package/src/server/utils/logging.ts +62 -62
- package/src/server/utils/loop-pattern-context.ts +20 -20
- package/dist/client-v2/214.b063f33ba044a808.js +0 -10
- package/dist/client-v2/373.f5d5292e53c4e832.js +0 -10
- package/dist/client-v2/418.58fa350f5496efd3.js +0 -10
- package/dist/client-v2/53.9a10455323390f14.js +0 -10
- package/dist/client-v2/677.a991ce0250ff5c77.js +0 -10
- package/dist/client-v2/925.e5925bf21ed68e7a.js +0 -10
- package/src/client/AIEmployeeSelect.tsx +0 -47
- package/src/client/AIEmployeesContext.tsx +0 -102
- package/src/client/RulesTab.tsx +0 -2
- package/src/client/TracingTab.tsx +0 -436
- package/src/client/skill-hub/components/ExecutionProgress.tsx +0 -51
- package/src/client-v2/components/RulesTab.tsx +0 -2
- package/src/client-v2/pages/AgentRunsPage.tsx +0 -13
|
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import type { Database, Model } from '@nocobase/database';
|
|
3
3
|
import { LoopControlService } from './LoopControlService';
|
|
4
4
|
import { LoopPatternService } from './LoopPatternService';
|
|
5
|
+
import type { CompiledPatternSnapshot, HarnessSnapshot } from './LoopPatternService';
|
|
5
6
|
|
|
6
7
|
type TriggerType = 'manual' | 'cron' | 'event';
|
|
7
8
|
|
|
@@ -20,6 +21,22 @@ function isUniqueConstraintError(error: unknown) {
|
|
|
20
21
|
return error instanceof Error && error.name === 'SequelizeUniqueConstraintError';
|
|
21
22
|
}
|
|
22
23
|
|
|
24
|
+
// One audit row per role naming the exact profile version the run was frozen with. The snapshot
|
|
25
|
+
// on the run row is authoritative for execution; these events make the application itself auditable
|
|
26
|
+
// without diffing snapshots.
|
|
27
|
+
function harnessAppliedEvents(compiled: CompiledPatternSnapshot) {
|
|
28
|
+
const roles: Array<{ role: 'leader' | 'maker' | 'verifier'; username: string; snapshot: HarnessSnapshot }> = [
|
|
29
|
+
{ role: 'leader', username: compiled.roleBindings.leader, snapshot: compiled.leaderHarness },
|
|
30
|
+
...compiled.roleBindings.makers.map((username) => ({
|
|
31
|
+
role: 'maker' as const,
|
|
32
|
+
username,
|
|
33
|
+
snapshot: compiled.makerHarnesses[username],
|
|
34
|
+
})),
|
|
35
|
+
{ role: 'verifier', username: compiled.roleBindings.verifier, snapshot: compiled.verifierHarness },
|
|
36
|
+
];
|
|
37
|
+
return roles.filter((entry) => entry.snapshot);
|
|
38
|
+
}
|
|
39
|
+
|
|
23
40
|
export class LoopTriggerService {
|
|
24
41
|
private readonly control: LoopControlService;
|
|
25
42
|
|
|
@@ -101,6 +118,31 @@ export class LoopTriggerService {
|
|
|
101
118
|
},
|
|
102
119
|
transaction,
|
|
103
120
|
});
|
|
121
|
+
for (const entry of harnessAppliedEvents(compiled)) {
|
|
122
|
+
await this.database.getRepository('agentLoopEvents').create({
|
|
123
|
+
values: {
|
|
124
|
+
runId,
|
|
125
|
+
type: 'harness_applied',
|
|
126
|
+
title: `Harness ${entry.snapshot.tag}@v${entry.snapshot.version} applied to ${entry.role}`,
|
|
127
|
+
content: `${entry.role} ${entry.username} executes under harness profile ${entry.snapshot.tag} version ${entry.snapshot.version}.`,
|
|
128
|
+
status: 'queued',
|
|
129
|
+
payload: {
|
|
130
|
+
role: entry.role,
|
|
131
|
+
username: entry.username,
|
|
132
|
+
tag: entry.snapshot.tag,
|
|
133
|
+
version: entry.snapshot.version,
|
|
134
|
+
versionId: entry.snapshot.versionId,
|
|
135
|
+
schemaVersion: entry.snapshot.schemaVersion,
|
|
136
|
+
},
|
|
137
|
+
actorType: input.triggerType === 'manual' ? 'user' : 'system',
|
|
138
|
+
actorIdentity: input.userId ? String(input.userId) : input.triggerType,
|
|
139
|
+
correlationKey: `harness:${entry.role}:${entry.username}`,
|
|
140
|
+
userId: input.userId || null,
|
|
141
|
+
createdAt: now,
|
|
142
|
+
},
|
|
143
|
+
transaction,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
104
146
|
return { created: true, run: plain(run) };
|
|
105
147
|
});
|
|
106
148
|
} catch (error) {
|
|
@@ -11,7 +11,8 @@ import { LoopRunStateMachine } from './LoopRunStateMachine';
|
|
|
11
11
|
import type { ClaimedLoopRun } from './LoopRunStateMachine';
|
|
12
12
|
import { PathLockService } from './PathLockService';
|
|
13
13
|
import { PluginAiRuntimeAdapter } from './PluginAiRuntimeAdapter';
|
|
14
|
-
import type { InvocationOutcome } from './PluginAiRuntimeAdapter';
|
|
14
|
+
import type { InvocationOutcome, ToolCallDecision } from './PluginAiRuntimeAdapter';
|
|
15
|
+
import { ToolLoopDetectionService, toolLoopNotice, toolLoopReason } from './ToolLoopDetectionService';
|
|
15
16
|
import { VerificationService } from './VerificationService';
|
|
16
17
|
|
|
17
18
|
const POLL_INTERVAL_MS = 3_000;
|
|
@@ -19,6 +20,11 @@ const LEASE_MS = 120_000;
|
|
|
19
20
|
const HEARTBEAT_MS = 30_000;
|
|
20
21
|
const PATH_LOCK_TTL_MS = 900_000;
|
|
21
22
|
|
|
23
|
+
const LEADER_SYSTEM_MESSAGE =
|
|
24
|
+
'You lead an autonomous run. Break the goal into concrete work for your makers, then finish with a concise plan of what must be built and which evidence will prove it. You do not modify anything yourself.';
|
|
25
|
+
const MAKER_SYSTEM_MESSAGE =
|
|
26
|
+
'You are a maker on an autonomous run. Do the work assigned by the leader using only the tools you were granted, and report exactly what you changed and the evidence for it.';
|
|
27
|
+
|
|
22
28
|
export type LoopWorkerSyncMessage = {
|
|
23
29
|
type: 'loop-run-abort-requested';
|
|
24
30
|
runId: number;
|
|
@@ -28,6 +34,15 @@ export function loopWorkerAbortMessage(runId: number): LoopWorkerSyncMessage {
|
|
|
28
34
|
return { type: 'loop-run-abort-requested', runId };
|
|
29
35
|
}
|
|
30
36
|
|
|
37
|
+
type ResumeContext = {
|
|
38
|
+
role: 'leader' | 'maker';
|
|
39
|
+
username: string;
|
|
40
|
+
sessionId: string;
|
|
41
|
+
messageId: string;
|
|
42
|
+
makerReports: string[];
|
|
43
|
+
makerQueue: string[];
|
|
44
|
+
};
|
|
45
|
+
|
|
31
46
|
type RunSnapshot = {
|
|
32
47
|
id: number;
|
|
33
48
|
patternId: number;
|
|
@@ -43,6 +58,8 @@ type RunSnapshot = {
|
|
|
43
58
|
repositoryKey: string;
|
|
44
59
|
actingOn: string[];
|
|
45
60
|
userId?: number;
|
|
61
|
+
approvalStatus: string;
|
|
62
|
+
resumeContext: ResumeContext | null;
|
|
46
63
|
};
|
|
47
64
|
|
|
48
65
|
function read(record: Model | Record<string, unknown>, key: string) {
|
|
@@ -69,6 +86,24 @@ function harnessFrom(value: unknown): CompiledHarness {
|
|
|
69
86
|
return effective as unknown as CompiledHarness;
|
|
70
87
|
}
|
|
71
88
|
|
|
89
|
+
function resumeContextFrom(value: unknown): ResumeContext | null {
|
|
90
|
+
const context = asObject(value);
|
|
91
|
+
if (!context) return null;
|
|
92
|
+
const role = context.role === 'leader' || context.role === 'maker' ? context.role : null;
|
|
93
|
+
const username = typeof context.username === 'string' ? context.username : '';
|
|
94
|
+
const sessionId = typeof context.sessionId === 'string' ? context.sessionId : '';
|
|
95
|
+
const messageId = typeof context.messageId === 'string' ? context.messageId : '';
|
|
96
|
+
if (!role || !username || !sessionId || !messageId) return null;
|
|
97
|
+
return {
|
|
98
|
+
role,
|
|
99
|
+
username,
|
|
100
|
+
sessionId,
|
|
101
|
+
messageId,
|
|
102
|
+
makerReports: asStringArray(context.makerReports),
|
|
103
|
+
makerQueue: asStringArray(context.makerQueue),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
72
107
|
function inputHash(value: unknown) {
|
|
73
108
|
return createHash('sha256')
|
|
74
109
|
.update(JSON.stringify(value ?? null))
|
|
@@ -83,6 +118,7 @@ export class LoopWorkerService {
|
|
|
83
118
|
private readonly pathLocks: PathLockService;
|
|
84
119
|
private readonly runtime: PluginAiRuntimeAdapter;
|
|
85
120
|
private readonly verification: VerificationService;
|
|
121
|
+
private readonly loopGuard: ToolLoopDetectionService;
|
|
86
122
|
private readonly workerId: string;
|
|
87
123
|
private readonly active = new Map<number, AbortController>();
|
|
88
124
|
private readonly heldPathLocks = new Map<number, string>();
|
|
@@ -105,6 +141,7 @@ export class LoopWorkerService {
|
|
|
105
141
|
this.pathLocks = new PathLockService(database, distributedLock);
|
|
106
142
|
this.runtime = new PluginAiRuntimeAdapter(app);
|
|
107
143
|
this.verification = new VerificationService(database, this.runtime, this.stateMachine);
|
|
144
|
+
this.loopGuard = new ToolLoopDetectionService(database);
|
|
108
145
|
this.workerId = `${app.name}:${process.pid}:${randomUUID()}`;
|
|
109
146
|
}
|
|
110
147
|
|
|
@@ -152,6 +189,9 @@ export class LoopWorkerService {
|
|
|
152
189
|
|
|
153
190
|
private async poll() {
|
|
154
191
|
if (!this.running) return;
|
|
192
|
+
await this.stateMachine.expireOverdueApprovals(new Date()).catch((error) => {
|
|
193
|
+
this.app.logger.error('[AgentOrchestrator] Approval expiry sweep failed.', { error });
|
|
194
|
+
});
|
|
155
195
|
const claimed = await this.stateMachine.claimNext(this.workerId, LEASE_MS);
|
|
156
196
|
if (!claimed) return;
|
|
157
197
|
await this.execute(claimed);
|
|
@@ -211,6 +251,24 @@ export class LoopWorkerService {
|
|
|
211
251
|
|
|
212
252
|
if (!(await this.acquirePaths(snapshot, leaseToken))) return;
|
|
213
253
|
|
|
254
|
+
// A run requeued from a pause while its approvals are still pending has nothing to execute:
|
|
255
|
+
// the decisions never arrived, so it parks again instead of restarting from the leader.
|
|
256
|
+
if (snapshot.resumeContext && snapshot.approvalStatus === 'pending') {
|
|
257
|
+
await this.stateMachine.transition({
|
|
258
|
+
runId: snapshot.id,
|
|
259
|
+
to: 'waiting_approval',
|
|
260
|
+
actorType: 'worker',
|
|
261
|
+
actorIdentity: this.workerId,
|
|
262
|
+
leaseToken,
|
|
263
|
+
eventType: 'run_waiting_approval',
|
|
264
|
+
title: 'Waiting for action approval',
|
|
265
|
+
content: 'Run requeued while its approval decisions are still pending.',
|
|
266
|
+
correlationKey: `approval-repark:${snapshot.id}`,
|
|
267
|
+
values: { approvalStatus: 'pending' },
|
|
268
|
+
});
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
|
|
214
272
|
await this.stateMachine.transition({
|
|
215
273
|
runId: snapshot.id,
|
|
216
274
|
to: 'running',
|
|
@@ -218,35 +276,91 @@ export class LoopWorkerService {
|
|
|
218
276
|
actorIdentity: this.workerId,
|
|
219
277
|
leaseToken,
|
|
220
278
|
eventType: 'run_started',
|
|
221
|
-
title: 'Run started',
|
|
222
|
-
content:
|
|
279
|
+
title: snapshot.resumeContext ? 'Run resumed' : 'Run started',
|
|
280
|
+
content: snapshot.resumeContext
|
|
281
|
+
? `${snapshot.resumeContext.username} resumed after approval decisions.`
|
|
282
|
+
: `Leader ${snapshot.leaderUsername} began working on the goal.`,
|
|
223
283
|
correlationKey: `started:${snapshot.id}`,
|
|
224
|
-
values: { currentRole: 'leader' },
|
|
284
|
+
values: { currentRole: snapshot.resumeContext ? snapshot.resumeContext.role : 'leader' },
|
|
225
285
|
});
|
|
226
286
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
harness: snapshot.leaderHarness,
|
|
231
|
-
systemMessage:
|
|
232
|
-
'You lead an autonomous run. Break the goal into concrete work for your makers, then finish with a concise plan of what must be built and which evidence will prove it. You do not modify anything yourself.',
|
|
233
|
-
prompt: snapshot.goal,
|
|
234
|
-
leaseToken,
|
|
235
|
-
signal,
|
|
236
|
-
});
|
|
287
|
+
// Loops recorded before this claim (for example before a park for approval) keep warning the
|
|
288
|
+
// prompts composed later in the run.
|
|
289
|
+
const loopNotices = await this.loopGuard.existingNotices(snapshot.id);
|
|
237
290
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
291
|
+
let makerReports: string[];
|
|
292
|
+
let makerQueue: string[];
|
|
293
|
+
|
|
294
|
+
if (snapshot.resumeContext) {
|
|
295
|
+
const resume = snapshot.resumeContext;
|
|
296
|
+
const decisions = await this.consumeDecisions(snapshot.id);
|
|
297
|
+
const resumed = await this.resumeRole(snapshot, resume, decisions, signal);
|
|
298
|
+
|
|
299
|
+
const resumeGuard = await this.enforceToolLoopGuard(
|
|
300
|
+
snapshot,
|
|
301
|
+
leaseToken,
|
|
302
|
+
{ role: resume.role, username: resume.username },
|
|
303
|
+
loopNotices,
|
|
304
|
+
);
|
|
305
|
+
if (resumeGuard === 'halt') return;
|
|
306
|
+
|
|
307
|
+
// A resumed conversation can hit another approval gate; park with the same role context so
|
|
308
|
+
// the next resume continues exactly where this one stopped.
|
|
309
|
+
if (resumed.interrupted.length) {
|
|
310
|
+
await this.parkForApproval(snapshot, resumed, leaseToken, {
|
|
311
|
+
role: resume.role,
|
|
312
|
+
username: resume.username,
|
|
313
|
+
makerReports: resume.makerReports,
|
|
314
|
+
makerQueue: resume.makerQueue,
|
|
315
|
+
});
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
if (resume.role === 'leader') {
|
|
319
|
+
makerReports = [resumed.content];
|
|
320
|
+
makerQueue = [...snapshot.makerUsernames];
|
|
321
|
+
} else {
|
|
322
|
+
makerReports = [...resume.makerReports, `# ${resume.username}\n${resumed.content}`];
|
|
323
|
+
makerQueue = [...resume.makerQueue];
|
|
324
|
+
}
|
|
325
|
+
} else {
|
|
326
|
+
const leader = await this.invokeRole(snapshot, {
|
|
327
|
+
role: 'leader',
|
|
328
|
+
username: snapshot.leaderUsername,
|
|
329
|
+
harness: snapshot.leaderHarness,
|
|
330
|
+
systemMessage: LEADER_SYSTEM_MESSAGE,
|
|
331
|
+
prompt: snapshot.goal,
|
|
332
|
+
leaseToken,
|
|
333
|
+
signal,
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
const leaderGuard = await this.enforceToolLoopGuard(
|
|
337
|
+
snapshot,
|
|
338
|
+
leaseToken,
|
|
339
|
+
{ role: 'leader', username: snapshot.leaderUsername },
|
|
340
|
+
loopNotices,
|
|
341
|
+
);
|
|
342
|
+
if (leaderGuard === 'halt') return;
|
|
343
|
+
|
|
344
|
+
// plugin-ai stops the graph on an approval-gated tool. Approval is a human decision, so the
|
|
345
|
+
// run parks here instead of the worker deciding on the operator's behalf.
|
|
346
|
+
if (leader.interrupted.length) {
|
|
347
|
+
await this.parkForApproval(snapshot, leader, leaseToken, {
|
|
348
|
+
role: 'leader',
|
|
349
|
+
username: snapshot.leaderUsername,
|
|
350
|
+
makerReports: [],
|
|
351
|
+
makerQueue: snapshot.makerUsernames,
|
|
352
|
+
});
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
makerReports = [leader.content];
|
|
356
|
+
makerQueue = [...snapshot.makerUsernames];
|
|
243
357
|
}
|
|
244
358
|
|
|
245
359
|
// Each maker executes with its own compiled harness snapshot. Reusing one harness would apply
|
|
246
360
|
// the wrong per-employee tool grants, so the snapshot is looked up by username and a maker with
|
|
247
361
|
// no snapshot is a compile error, not a silent skip.
|
|
248
|
-
|
|
249
|
-
|
|
362
|
+
while (makerQueue.length) {
|
|
363
|
+
const username = makerQueue[0];
|
|
250
364
|
const harness = snapshot.makerHarnesses[username]?.effective;
|
|
251
365
|
if (!harness) {
|
|
252
366
|
throw new Error(`The claimed run has no harness snapshot for maker "${username}".`);
|
|
@@ -255,22 +369,35 @@ export class LoopWorkerService {
|
|
|
255
369
|
role: 'maker',
|
|
256
370
|
username,
|
|
257
371
|
harness,
|
|
258
|
-
systemMessage:
|
|
259
|
-
'You are a maker on an autonomous run. Do the work assigned by the leader using only the tools you were granted, and report exactly what you changed and the evidence for it.',
|
|
372
|
+
systemMessage: MAKER_SYSTEM_MESSAGE,
|
|
260
373
|
prompt: [
|
|
261
374
|
`Goal: ${snapshot.goal}`,
|
|
262
375
|
'',
|
|
263
376
|
"Leader's plan:",
|
|
264
|
-
|
|
377
|
+
makerReports[0] || '(the leader produced no textual plan)',
|
|
378
|
+
...loopNotices,
|
|
265
379
|
].join('\n'),
|
|
266
380
|
leaseToken,
|
|
267
381
|
signal,
|
|
268
382
|
});
|
|
383
|
+
const makerGuard = await this.enforceToolLoopGuard(
|
|
384
|
+
snapshot,
|
|
385
|
+
leaseToken,
|
|
386
|
+
{ role: 'maker', username },
|
|
387
|
+
loopNotices,
|
|
388
|
+
);
|
|
389
|
+
if (makerGuard === 'halt') return;
|
|
269
390
|
if (maker.interrupted.length) {
|
|
270
|
-
await this.parkForApproval(snapshot, maker, leaseToken
|
|
391
|
+
await this.parkForApproval(snapshot, maker, leaseToken, {
|
|
392
|
+
role: 'maker',
|
|
393
|
+
username,
|
|
394
|
+
makerReports,
|
|
395
|
+
makerQueue: makerQueue.slice(1),
|
|
396
|
+
});
|
|
271
397
|
return;
|
|
272
398
|
}
|
|
273
399
|
makerReports.push(`# ${username}\n${maker.content}`);
|
|
400
|
+
makerQueue = makerQueue.slice(1);
|
|
274
401
|
}
|
|
275
402
|
const makerSummary = makerReports.join('\n\n');
|
|
276
403
|
|
|
@@ -306,9 +433,14 @@ export class LoopWorkerService {
|
|
|
306
433
|
userId: snapshot.userId,
|
|
307
434
|
workerId: this.workerId,
|
|
308
435
|
leaseToken,
|
|
436
|
+
loopNotices,
|
|
309
437
|
signal,
|
|
310
438
|
});
|
|
311
439
|
|
|
440
|
+
// The tool loop guard already moved the run to blocked or waiting_human; the circuit breaker
|
|
441
|
+
// tracks failed attempts, not behavioral stops, so it must not see this as a failure.
|
|
442
|
+
if (outcome.finalStatus === 'halted') return;
|
|
443
|
+
|
|
312
444
|
if (outcome.finalStatus === 'failed') {
|
|
313
445
|
await this.circuits.recordFailure({
|
|
314
446
|
patternId: snapshot.patternId,
|
|
@@ -387,6 +519,7 @@ export class LoopWorkerService {
|
|
|
387
519
|
kind: 'invocation',
|
|
388
520
|
type: 'reasoning',
|
|
389
521
|
employeeUsername: input.username,
|
|
522
|
+
sessionId,
|
|
390
523
|
title: `${input.role} invocation`,
|
|
391
524
|
inputHash: inputHash(input.prompt),
|
|
392
525
|
status: 'running',
|
|
@@ -420,21 +553,166 @@ export class LoopWorkerService {
|
|
|
420
553
|
return outcome;
|
|
421
554
|
}
|
|
422
555
|
|
|
423
|
-
private async
|
|
556
|
+
private async resumeRole(
|
|
557
|
+
snapshot: RunSnapshot,
|
|
558
|
+
resume: ResumeContext,
|
|
559
|
+
decisions: Array<{ toolCallId: string; decision: ToolCallDecision }>,
|
|
560
|
+
signal: AbortSignal,
|
|
561
|
+
) {
|
|
562
|
+
await this.budgets.reserve({
|
|
563
|
+
runId: snapshot.id,
|
|
564
|
+
patternId: snapshot.patternId,
|
|
565
|
+
policy: snapshot.policy,
|
|
566
|
+
delta: { invocations: 1 },
|
|
567
|
+
});
|
|
568
|
+
const harness =
|
|
569
|
+
resume.role === 'leader' ? snapshot.leaderHarness : snapshot.makerHarnesses[resume.username]?.effective;
|
|
570
|
+
if (!harness) {
|
|
571
|
+
throw new Error(`The claimed run has no harness snapshot for ${resume.role} "${resume.username}".`);
|
|
572
|
+
}
|
|
573
|
+
const step = await this.database.getRepository('agentLoopSteps').create({
|
|
574
|
+
values: {
|
|
575
|
+
runId: snapshot.id,
|
|
576
|
+
runtimeVersion: 'control-plane-v2',
|
|
577
|
+
sequence: await this.nextSequence(snapshot.id),
|
|
578
|
+
role: resume.role,
|
|
579
|
+
kind: 'invocation',
|
|
580
|
+
type: 'resume',
|
|
581
|
+
employeeUsername: resume.username,
|
|
582
|
+
sessionId: resume.sessionId,
|
|
583
|
+
title: `${resume.role} resume after approval`,
|
|
584
|
+
inputHash: inputHash(decisions),
|
|
585
|
+
status: 'running',
|
|
586
|
+
startedAt: new Date(),
|
|
587
|
+
createdAt: new Date(),
|
|
588
|
+
},
|
|
589
|
+
});
|
|
590
|
+
|
|
591
|
+
const outcome = await this.runtime.resume({
|
|
592
|
+
username: resume.username,
|
|
593
|
+
sessionId: resume.sessionId,
|
|
594
|
+
messageId: resume.messageId,
|
|
595
|
+
userId: snapshot.userId,
|
|
596
|
+
systemMessage: resume.role === 'leader' ? LEADER_SYSTEM_MESSAGE : MAKER_SYSTEM_MESSAGE,
|
|
597
|
+
harness,
|
|
598
|
+
decisions,
|
|
599
|
+
signal,
|
|
600
|
+
});
|
|
601
|
+
|
|
602
|
+
await this.database.getRepository('agentLoopSteps').update({
|
|
603
|
+
filterByTk: read(step, 'id'),
|
|
604
|
+
values: {
|
|
605
|
+
status: outcome.interrupted.length ? 'waiting_user' : 'succeeded',
|
|
606
|
+
output: { content: outcome.content },
|
|
607
|
+
endedAt: new Date(),
|
|
608
|
+
},
|
|
609
|
+
});
|
|
610
|
+
await this.database.getRepository('agentLoopRuns').update({
|
|
611
|
+
filterByTk: snapshot.id,
|
|
612
|
+
values: { sessionId: outcome.sessionId, messageId: outcome.messageId },
|
|
613
|
+
});
|
|
614
|
+
return outcome;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
// Run-level scan over every tool call issued so far, evaluated at each pass boundary. A warning
|
|
618
|
+
// is injected into the prompts composed later in the run; block and escalate halt it at once.
|
|
619
|
+
private async enforceToolLoopGuard(
|
|
620
|
+
snapshot: RunSnapshot,
|
|
621
|
+
leaseToken: string,
|
|
622
|
+
pass: { role: 'leader' | 'maker'; username: string },
|
|
623
|
+
loopNotices: string[],
|
|
624
|
+
): Promise<'continue' | 'halt'> {
|
|
625
|
+
const finding = await this.loopGuard.scanRun(snapshot.id, snapshot.policy);
|
|
626
|
+
if (!finding) return 'continue';
|
|
627
|
+
|
|
628
|
+
await this.loopGuard.recordFinding(
|
|
629
|
+
{
|
|
630
|
+
runId: snapshot.id,
|
|
631
|
+
role: pass.role,
|
|
632
|
+
username: pass.username,
|
|
633
|
+
runStatus: 'running',
|
|
634
|
+
actorType: 'worker',
|
|
635
|
+
actorIdentity: this.workerId,
|
|
636
|
+
},
|
|
637
|
+
finding,
|
|
638
|
+
);
|
|
639
|
+
|
|
640
|
+
if (finding.level === 'warn') {
|
|
641
|
+
const notice = toolLoopNotice(finding.toolName, finding.count);
|
|
642
|
+
if (!loopNotices.includes(notice)) loopNotices.push(notice);
|
|
643
|
+
return 'continue';
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
const reason = toolLoopReason(finding);
|
|
647
|
+
await this.stateMachine.transition({
|
|
648
|
+
runId: snapshot.id,
|
|
649
|
+
to: finding.level === 'block' ? 'blocked' : 'waiting_human',
|
|
650
|
+
actorType: 'worker',
|
|
651
|
+
actorIdentity: this.workerId,
|
|
652
|
+
leaseToken,
|
|
653
|
+
eventType: finding.level === 'block' ? 'run_blocked' : 'run_escalated',
|
|
654
|
+
title: finding.level === 'block' ? 'Run blocked by tool loop detection' : 'Run escalated by tool loop detection',
|
|
655
|
+
content: reason,
|
|
656
|
+
correlationKey: `tool-loop-${finding.level}:${snapshot.id}:${finding.signature}`,
|
|
657
|
+
values: finding.level === 'block' ? { blockedReason: reason } : { escalationReason: reason },
|
|
658
|
+
});
|
|
659
|
+
return 'halt';
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
// Decisions are consumed exactly once: rows are marked consumed so a retry or a second resume
|
|
663
|
+
// can never replay a human decision against a fresh conversation.
|
|
664
|
+
private async consumeDecisions(runId: number) {
|
|
665
|
+
const repository = this.database.getRepository('agentLoopActionApprovals');
|
|
666
|
+
const approvals = await repository.find({ filter: { runId } });
|
|
667
|
+
const decisions: Array<{ toolCallId: string; decision: ToolCallDecision }> = [];
|
|
668
|
+
const now = new Date();
|
|
669
|
+
for (const approval of approvals) {
|
|
670
|
+
const status = String(read(approval, 'status'));
|
|
671
|
+
if (status !== 'approved' && status !== 'rejected') continue;
|
|
672
|
+
if (read(approval, 'consumedAt')) continue;
|
|
673
|
+
const editedInput = read(approval, 'editedInput');
|
|
674
|
+
const decision: ToolCallDecision =
|
|
675
|
+
status === 'approved'
|
|
676
|
+
? editedInput
|
|
677
|
+
? { type: 'edit', editedAction: { name: String(read(approval, 'toolName')), args: editedInput } }
|
|
678
|
+
: { type: 'approve' }
|
|
679
|
+
: { type: 'reject', message: String(read(approval, 'decisionNote') || '') || undefined };
|
|
680
|
+
decisions.push({ toolCallId: String(read(approval, 'toolCallId')), decision });
|
|
681
|
+
await repository.update({ filterByTk: Number(read(approval, 'id')), values: { consumedAt: now } });
|
|
682
|
+
}
|
|
683
|
+
if (!decisions.length) throw new Error('The resumed run has no recorded approval decisions.');
|
|
684
|
+
return decisions;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
private async parkForApproval(
|
|
688
|
+
snapshot: RunSnapshot,
|
|
689
|
+
outcome: InvocationOutcome,
|
|
690
|
+
leaseToken: string,
|
|
691
|
+
context: { role: 'leader' | 'maker'; username: string; makerReports: string[]; makerQueue: string[] },
|
|
692
|
+
) {
|
|
424
693
|
const now = new Date();
|
|
425
694
|
const expiresAt = new Date(now.getTime() + snapshot.policy.actions.approvalTimeoutMs);
|
|
426
695
|
const assignedToId = snapshot.policy.actions.approvalAssigneeIds[0] || snapshot.userId || null;
|
|
696
|
+
const harness =
|
|
697
|
+
context.role === 'leader' ? snapshot.leaderHarness : snapshot.makerHarnesses[context.username]?.effective;
|
|
698
|
+
const escalatable = new Set(harness?.tools.escalate ?? []);
|
|
427
699
|
for (const call of outcome.interrupted) {
|
|
700
|
+
// An escalation is a request for authority the harness never granted, widened for exactly
|
|
701
|
+
// one call; a plain tool_call approval gates a tool the harness allows but asks about.
|
|
702
|
+
const escalation = escalatable.has(call.toolName);
|
|
428
703
|
await this.database.getRepository('agentLoopActionApprovals').create({
|
|
429
704
|
values: {
|
|
430
705
|
runId: snapshot.id,
|
|
431
706
|
toolCallId: call.toolCallId,
|
|
432
707
|
toolName: call.toolName,
|
|
708
|
+
actionType: escalation ? 'escalation' : 'tool_call',
|
|
433
709
|
proposedInput: call.args,
|
|
434
710
|
inputHash: inputHash(call.args),
|
|
435
711
|
paths: snapshot.actingOn,
|
|
436
712
|
policyDecision: { decision: 'ask', interruptId: call.interruptId },
|
|
437
|
-
reason:
|
|
713
|
+
reason: escalation
|
|
714
|
+
? `Tool "${call.toolName}" is not granted by this run's harness. Approving widens authority for this single call only.`
|
|
715
|
+
: `Tool "${call.toolName}" requires human approval under this pattern's policy.`,
|
|
438
716
|
status: 'pending',
|
|
439
717
|
assignedToId,
|
|
440
718
|
requestedById: snapshot.userId || null,
|
|
@@ -444,6 +722,22 @@ export class LoopWorkerService {
|
|
|
444
722
|
},
|
|
445
723
|
});
|
|
446
724
|
}
|
|
725
|
+
await this.database.getRepository('agentLoopRuns').update({
|
|
726
|
+
filterByTk: snapshot.id,
|
|
727
|
+
values: {
|
|
728
|
+
// The resume point is durable: which role stopped, in which conversation, with which
|
|
729
|
+
// reports already collected and which makers still queued. Without it a resumed run
|
|
730
|
+
// would restart from the leader and redo every completed invocation.
|
|
731
|
+
resumeContext: {
|
|
732
|
+
role: context.role,
|
|
733
|
+
username: context.username,
|
|
734
|
+
sessionId: outcome.sessionId,
|
|
735
|
+
messageId: outcome.messageId,
|
|
736
|
+
makerReports: context.makerReports,
|
|
737
|
+
makerQueue: context.makerQueue,
|
|
738
|
+
},
|
|
739
|
+
},
|
|
740
|
+
});
|
|
447
741
|
await this.stateMachine.transition({
|
|
448
742
|
runId: snapshot.id,
|
|
449
743
|
to: 'waiting_approval',
|
|
@@ -502,6 +796,7 @@ export class LoopWorkerService {
|
|
|
502
796
|
}
|
|
503
797
|
const roleBindings = asObject(run.roleBindingsSnapshot);
|
|
504
798
|
const makerSnapshots = asObject(run.makerHarnessSnapshot);
|
|
799
|
+
const approvalStatus = String(run.approvalStatus || 'none');
|
|
505
800
|
return {
|
|
506
801
|
id: Number(run.id),
|
|
507
802
|
patternId,
|
|
@@ -517,6 +812,11 @@ export class LoopWorkerService {
|
|
|
517
812
|
repositoryKey: String(run.repositoryKey || ''),
|
|
518
813
|
actingOn: asStringArray(run.actingOn),
|
|
519
814
|
userId: Number(run.userId) || undefined,
|
|
815
|
+
approvalStatus,
|
|
816
|
+
// Only a live approval state may resume. Stale context left over from an expired window or a
|
|
817
|
+
// cleared retry would otherwise replay an old conversation against fresh approvals.
|
|
818
|
+
resumeContext:
|
|
819
|
+
approvalStatus === 'decided' || approvalStatus === 'pending' ? resumeContextFrom(run.resumeContext) : null,
|
|
520
820
|
};
|
|
521
821
|
}
|
|
522
822
|
}
|