plugin-agent-orchestrator 1.1.4 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/README.md +204 -114
  2. package/dist/client/index.js +1 -1
  3. package/dist/client-v2/117.7565cff57c39318a.js +10 -0
  4. package/dist/client-v2/214.106635011859dede.js +10 -0
  5. package/dist/client-v2/418.41ed200e7d6da332.js +10 -0
  6. package/dist/client-v2/53.349fff8db80b19d4.js +10 -0
  7. package/dist/client-v2/981.e4a1f41cfc94a856.js +10 -0
  8. package/dist/client-v2/index.js +1 -1
  9. package/dist/externalVersion.js +7 -8
  10. package/dist/locale/en-US.json +226 -170
  11. package/dist/locale/vi-VN.json +226 -170
  12. package/dist/locale/zh-CN.json +226 -170
  13. package/dist/server/collections/agent-execution-spans.js +8 -0
  14. package/dist/server/collections/agent-loop-action-approvals.js +1 -0
  15. package/dist/server/collections/agent-loop-runs.js +7 -1
  16. package/dist/server/collections/agent-loop-steps.js +7 -0
  17. package/dist/server/collections/skill-executions.js +8 -0
  18. package/dist/server/plugin.js +3 -1
  19. package/dist/server/resources/agent-harness-profile.js +117 -0
  20. package/dist/server/resources/agent-knowledge-insights.js +1 -3
  21. package/dist/server/resources/agent-loop-approval.js +25 -0
  22. package/dist/server/resources/agent-loop.js +3 -1
  23. package/dist/server/services/AgentMemoryContextService.js +34 -0
  24. package/dist/server/services/HarnessCompiler.js +35 -2
  25. package/dist/server/services/HarnessProfileService.js +74 -9
  26. package/dist/server/services/HarnessSchema.js +30 -3
  27. package/dist/server/services/LoopPatternSchema.js +16 -1
  28. package/dist/server/services/LoopRunRepository.js +80 -0
  29. package/dist/server/services/LoopRunStateMachine.js +92 -3
  30. package/dist/server/services/LoopTriggerService.js +37 -0
  31. package/dist/server/services/LoopWorkerService.js +261 -23
  32. package/dist/server/services/NativeSubAgentObserver.js +165 -23
  33. package/dist/server/services/PluginAiRuntimeAdapter.js +16 -5
  34. package/dist/server/services/ToolLoopDetectionService.js +210 -0
  35. package/dist/server/services/VerificationService.js +66 -0
  36. package/dist/server/services/harness-runtime-policy.js +139 -0
  37. package/dist/server/skill-hub/plugin.js +36 -3
  38. package/dist/server/skill-hub/tasks/SkillExecutionTask.js +29 -0
  39. package/package.json +53 -53
  40. package/src/client/ApprovalsTab.tsx +2 -0
  41. package/src/client/OrchestratorSettings.tsx +11 -0
  42. package/src/client/skill-hub/components/GitSkillImport.tsx +3 -3
  43. package/src/client/skill-hub/components/SkillMetrics.tsx +15 -10
  44. package/src/client/skill-hub/index.tsx +0 -52
  45. package/src/client-v2/components/AgentRunsTab.tsx +576 -576
  46. package/src/client-v2/components/ApprovalsTab.tsx +337 -0
  47. package/src/client-v2/components/HarnessProfilesTab.tsx +385 -105
  48. package/src/client-v2/components/MemoryInspectorTab.tsx +141 -141
  49. package/src/client-v2/components/TracingTab.tsx +515 -515
  50. package/src/client-v2/hooks/useApiRequest.ts +141 -141
  51. package/src/client-v2/pages/ApprovalsPage.tsx +10 -0
  52. package/src/client-v2/plugin.tsx +6 -6
  53. package/src/client-v2/skill-hub/components/SkillMetrics.tsx +5 -2
  54. package/src/locale/en-US.json +226 -174
  55. package/src/locale/vi-VN.json +226 -174
  56. package/src/locale/zh-CN.json +226 -174
  57. package/src/server/__tests__/circuit-breaker.test.ts +248 -248
  58. package/src/server/__tests__/harness-compiler.test.ts +83 -0
  59. package/src/server/__tests__/harness-profile-service.test.ts +51 -0
  60. package/src/server/__tests__/harness-runtime-policy.test.ts +293 -0
  61. package/src/server/__tests__/harness-tool-plan.test.ts +24 -0
  62. package/src/server/__tests__/loop-approval-decision.test.ts +512 -0
  63. package/src/server/__tests__/loop-approval-expiry.test.ts +258 -0
  64. package/src/server/__tests__/loop-budget-service.test.ts +255 -255
  65. package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -266
  66. package/src/server/__tests__/loop-pattern-schema.test.ts +18 -1
  67. package/src/server/__tests__/loop-pattern-service.test.ts +148 -148
  68. package/src/server/__tests__/loop-run-state-machine.test.ts +11 -0
  69. package/src/server/__tests__/loop-trigger-service.test.ts +148 -0
  70. package/src/server/__tests__/loop-worker-claim.test.ts +341 -341
  71. package/src/server/__tests__/loop-worker-execution.test.ts +82 -1
  72. package/src/server/__tests__/native-sub-agent-observer.test.ts +288 -4
  73. package/src/server/__tests__/path-lock-service.test.ts +254 -254
  74. package/src/server/__tests__/skill-execution-task-policy.test.ts +140 -0
  75. package/src/server/__tests__/smoke.test.ts +162 -162
  76. package/src/server/__tests__/tool-loop-detection.test.ts +275 -0
  77. package/src/server/collections/agent-execution-spans.ts +8 -0
  78. package/src/server/collections/agent-harness-profile-versions.ts +26 -26
  79. package/src/server/collections/agent-harness-profiles.ts +71 -71
  80. package/src/server/collections/agent-loop-action-approvals.ts +1 -0
  81. package/src/server/collections/agent-loop-artifacts.ts +25 -25
  82. package/src/server/collections/agent-loop-circuit-states.ts +23 -23
  83. package/src/server/collections/agent-loop-control-settings.ts +21 -21
  84. package/src/server/collections/agent-loop-events.ts +98 -98
  85. package/src/server/collections/agent-loop-path-locks.ts +27 -27
  86. package/src/server/collections/agent-loop-patterns.ts +42 -42
  87. package/src/server/collections/agent-loop-runs.ts +7 -1
  88. package/src/server/collections/agent-loop-steps.ts +7 -0
  89. package/src/server/collections/agent-loop-usage-buckets.ts +22 -22
  90. package/src/server/collections/agent-loop-worktrees.ts +25 -25
  91. package/src/server/collections/skill-executions.ts +8 -0
  92. package/src/server/migrations/20260427000002-change-packages-to-text.ts +52 -52
  93. package/src/server/migrations/20260429000000-add-llm-fields.ts +47 -47
  94. package/src/server/migrations/20260429000001-fix-inputargs-json-to-text.ts +45 -45
  95. package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +33 -33
  96. package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +31 -31
  97. package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -293
  98. package/src/server/plugin.ts +3 -1
  99. package/src/server/resources/agent-harness-profile.ts +94 -0
  100. package/src/server/resources/agent-knowledge-insights.ts +5 -3
  101. package/src/server/resources/agent-loop-approval.ts +32 -0
  102. package/src/server/resources/agent-loop-artifact.ts +56 -56
  103. package/src/server/resources/agent-loop-events-stream.ts +105 -105
  104. package/src/server/resources/agent-loop.ts +3 -1
  105. package/src/server/resources/resource-helpers.ts +33 -33
  106. package/src/server/resources/tracing.ts +266 -266
  107. package/src/server/services/AgentMemoryContextService.ts +44 -0
  108. package/src/server/services/DurableCircuitBreakerService.ts +239 -239
  109. package/src/server/services/ExecutionSpanService.ts +2 -0
  110. package/src/server/services/HarnessCompiler.ts +37 -0
  111. package/src/server/services/HarnessProfileService.ts +92 -10
  112. package/src/server/services/HarnessSchema.ts +31 -0
  113. package/src/server/services/LoopBudgetService.ts +275 -275
  114. package/src/server/services/LoopControlService.ts +124 -124
  115. package/src/server/services/LoopPatternSchema.ts +19 -1
  116. package/src/server/services/LoopPatternService.ts +143 -143
  117. package/src/server/services/LoopRunRepository.ts +99 -0
  118. package/src/server/services/LoopRunStateMachine.ts +100 -3
  119. package/src/server/services/LoopSchedulerService.ts +199 -199
  120. package/src/server/services/LoopTriggerService.ts +42 -0
  121. package/src/server/services/LoopWorkerService.ts +327 -27
  122. package/src/server/services/NativeSubAgentObserver.ts +186 -21
  123. package/src/server/services/PathLockService.ts +321 -321
  124. package/src/server/services/PluginAiRuntimeAdapter.ts +24 -7
  125. package/src/server/services/ToolLoopDetectionService.ts +213 -0
  126. package/src/server/services/VerificationSchema.ts +31 -31
  127. package/src/server/services/VerificationService.ts +81 -2
  128. package/src/server/services/harness-runtime-policy.ts +141 -0
  129. package/src/server/skill-hub/plugin.ts +41 -3
  130. package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +235 -235
  131. package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +61 -61
  132. package/src/server/skill-hub/tasks/SkillExecutionTask.ts +43 -2
  133. package/src/server/utils/ensure-indexes.ts +96 -96
  134. package/src/server/utils/logging.ts +62 -62
  135. package/src/server/utils/loop-pattern-context.ts +20 -20
  136. package/dist/client-v2/214.b063f33ba044a808.js +0 -10
  137. package/dist/client-v2/373.f5d5292e53c4e832.js +0 -10
  138. package/dist/client-v2/418.58fa350f5496efd3.js +0 -10
  139. package/dist/client-v2/53.9a10455323390f14.js +0 -10
  140. package/dist/client-v2/677.a991ce0250ff5c77.js +0 -10
  141. package/dist/client-v2/925.e5925bf21ed68e7a.js +0 -10
  142. package/src/client/AIEmployeeSelect.tsx +0 -47
  143. package/src/client/AIEmployeesContext.tsx +0 -102
  144. package/src/client/RulesTab.tsx +0 -2
  145. package/src/client/TracingTab.tsx +0 -436
  146. package/src/client/skill-hub/components/ExecutionProgress.tsx +0 -51
  147. package/src/client-v2/components/RulesTab.tsx +0 -2
  148. package/src/client-v2/pages/AgentRunsPage.tsx +0 -13
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
2
2
  import type { Database, Model } from '@nocobase/database';
3
3
  import { LoopControlService } from './LoopControlService';
4
4
  import { LoopPatternService } from './LoopPatternService';
5
+ import type { CompiledPatternSnapshot, HarnessSnapshot } from './LoopPatternService';
5
6
 
6
7
  type TriggerType = 'manual' | 'cron' | 'event';
7
8
 
@@ -20,6 +21,22 @@ function isUniqueConstraintError(error: unknown) {
20
21
  return error instanceof Error && error.name === 'SequelizeUniqueConstraintError';
21
22
  }
22
23
 
24
+ // One audit row per role naming the exact profile version the run was frozen with. The snapshot
25
+ // on the run row is authoritative for execution; these events make the application itself auditable
26
+ // without diffing snapshots.
27
+ function harnessAppliedEvents(compiled: CompiledPatternSnapshot) {
28
+ const roles: Array<{ role: 'leader' | 'maker' | 'verifier'; username: string; snapshot: HarnessSnapshot }> = [
29
+ { role: 'leader', username: compiled.roleBindings.leader, snapshot: compiled.leaderHarness },
30
+ ...compiled.roleBindings.makers.map((username) => ({
31
+ role: 'maker' as const,
32
+ username,
33
+ snapshot: compiled.makerHarnesses[username],
34
+ })),
35
+ { role: 'verifier', username: compiled.roleBindings.verifier, snapshot: compiled.verifierHarness },
36
+ ];
37
+ return roles.filter((entry) => entry.snapshot);
38
+ }
39
+
23
40
  export class LoopTriggerService {
24
41
  private readonly control: LoopControlService;
25
42
 
@@ -101,6 +118,31 @@ export class LoopTriggerService {
101
118
  },
102
119
  transaction,
103
120
  });
121
+ for (const entry of harnessAppliedEvents(compiled)) {
122
+ await this.database.getRepository('agentLoopEvents').create({
123
+ values: {
124
+ runId,
125
+ type: 'harness_applied',
126
+ title: `Harness ${entry.snapshot.tag}@v${entry.snapshot.version} applied to ${entry.role}`,
127
+ content: `${entry.role} ${entry.username} executes under harness profile ${entry.snapshot.tag} version ${entry.snapshot.version}.`,
128
+ status: 'queued',
129
+ payload: {
130
+ role: entry.role,
131
+ username: entry.username,
132
+ tag: entry.snapshot.tag,
133
+ version: entry.snapshot.version,
134
+ versionId: entry.snapshot.versionId,
135
+ schemaVersion: entry.snapshot.schemaVersion,
136
+ },
137
+ actorType: input.triggerType === 'manual' ? 'user' : 'system',
138
+ actorIdentity: input.userId ? String(input.userId) : input.triggerType,
139
+ correlationKey: `harness:${entry.role}:${entry.username}`,
140
+ userId: input.userId || null,
141
+ createdAt: now,
142
+ },
143
+ transaction,
144
+ });
145
+ }
104
146
  return { created: true, run: plain(run) };
105
147
  });
106
148
  } catch (error) {
@@ -11,7 +11,8 @@ import { LoopRunStateMachine } from './LoopRunStateMachine';
11
11
  import type { ClaimedLoopRun } from './LoopRunStateMachine';
12
12
  import { PathLockService } from './PathLockService';
13
13
  import { PluginAiRuntimeAdapter } from './PluginAiRuntimeAdapter';
14
- import type { InvocationOutcome } from './PluginAiRuntimeAdapter';
14
+ import type { InvocationOutcome, ToolCallDecision } from './PluginAiRuntimeAdapter';
15
+ import { ToolLoopDetectionService, toolLoopNotice, toolLoopReason } from './ToolLoopDetectionService';
15
16
  import { VerificationService } from './VerificationService';
16
17
 
17
18
  const POLL_INTERVAL_MS = 3_000;
@@ -19,6 +20,11 @@ const LEASE_MS = 120_000;
19
20
  const HEARTBEAT_MS = 30_000;
20
21
  const PATH_LOCK_TTL_MS = 900_000;
21
22
 
23
+ const LEADER_SYSTEM_MESSAGE =
24
+ 'You lead an autonomous run. Break the goal into concrete work for your makers, then finish with a concise plan of what must be built and which evidence will prove it. You do not modify anything yourself.';
25
+ const MAKER_SYSTEM_MESSAGE =
26
+ 'You are a maker on an autonomous run. Do the work assigned by the leader using only the tools you were granted, and report exactly what you changed and the evidence for it.';
27
+
22
28
  export type LoopWorkerSyncMessage = {
23
29
  type: 'loop-run-abort-requested';
24
30
  runId: number;
@@ -28,6 +34,15 @@ export function loopWorkerAbortMessage(runId: number): LoopWorkerSyncMessage {
28
34
  return { type: 'loop-run-abort-requested', runId };
29
35
  }
30
36
 
37
+ type ResumeContext = {
38
+ role: 'leader' | 'maker';
39
+ username: string;
40
+ sessionId: string;
41
+ messageId: string;
42
+ makerReports: string[];
43
+ makerQueue: string[];
44
+ };
45
+
31
46
  type RunSnapshot = {
32
47
  id: number;
33
48
  patternId: number;
@@ -43,6 +58,8 @@ type RunSnapshot = {
43
58
  repositoryKey: string;
44
59
  actingOn: string[];
45
60
  userId?: number;
61
+ approvalStatus: string;
62
+ resumeContext: ResumeContext | null;
46
63
  };
47
64
 
48
65
  function read(record: Model | Record<string, unknown>, key: string) {
@@ -69,6 +86,24 @@ function harnessFrom(value: unknown): CompiledHarness {
69
86
  return effective as unknown as CompiledHarness;
70
87
  }
71
88
 
89
+ function resumeContextFrom(value: unknown): ResumeContext | null {
90
+ const context = asObject(value);
91
+ if (!context) return null;
92
+ const role = context.role === 'leader' || context.role === 'maker' ? context.role : null;
93
+ const username = typeof context.username === 'string' ? context.username : '';
94
+ const sessionId = typeof context.sessionId === 'string' ? context.sessionId : '';
95
+ const messageId = typeof context.messageId === 'string' ? context.messageId : '';
96
+ if (!role || !username || !sessionId || !messageId) return null;
97
+ return {
98
+ role,
99
+ username,
100
+ sessionId,
101
+ messageId,
102
+ makerReports: asStringArray(context.makerReports),
103
+ makerQueue: asStringArray(context.makerQueue),
104
+ };
105
+ }
106
+
72
107
  function inputHash(value: unknown) {
73
108
  return createHash('sha256')
74
109
  .update(JSON.stringify(value ?? null))
@@ -83,6 +118,7 @@ export class LoopWorkerService {
83
118
  private readonly pathLocks: PathLockService;
84
119
  private readonly runtime: PluginAiRuntimeAdapter;
85
120
  private readonly verification: VerificationService;
121
+ private readonly loopGuard: ToolLoopDetectionService;
86
122
  private readonly workerId: string;
87
123
  private readonly active = new Map<number, AbortController>();
88
124
  private readonly heldPathLocks = new Map<number, string>();
@@ -105,6 +141,7 @@ export class LoopWorkerService {
105
141
  this.pathLocks = new PathLockService(database, distributedLock);
106
142
  this.runtime = new PluginAiRuntimeAdapter(app);
107
143
  this.verification = new VerificationService(database, this.runtime, this.stateMachine);
144
+ this.loopGuard = new ToolLoopDetectionService(database);
108
145
  this.workerId = `${app.name}:${process.pid}:${randomUUID()}`;
109
146
  }
110
147
 
@@ -152,6 +189,9 @@ export class LoopWorkerService {
152
189
 
153
190
  private async poll() {
154
191
  if (!this.running) return;
192
+ await this.stateMachine.expireOverdueApprovals(new Date()).catch((error) => {
193
+ this.app.logger.error('[AgentOrchestrator] Approval expiry sweep failed.', { error });
194
+ });
155
195
  const claimed = await this.stateMachine.claimNext(this.workerId, LEASE_MS);
156
196
  if (!claimed) return;
157
197
  await this.execute(claimed);
@@ -211,6 +251,24 @@ export class LoopWorkerService {
211
251
 
212
252
  if (!(await this.acquirePaths(snapshot, leaseToken))) return;
213
253
 
254
+ // A run requeued from a pause while its approvals are still pending has nothing to execute:
255
+ // the decisions never arrived, so it parks again instead of restarting from the leader.
256
+ if (snapshot.resumeContext && snapshot.approvalStatus === 'pending') {
257
+ await this.stateMachine.transition({
258
+ runId: snapshot.id,
259
+ to: 'waiting_approval',
260
+ actorType: 'worker',
261
+ actorIdentity: this.workerId,
262
+ leaseToken,
263
+ eventType: 'run_waiting_approval',
264
+ title: 'Waiting for action approval',
265
+ content: 'Run requeued while its approval decisions are still pending.',
266
+ correlationKey: `approval-repark:${snapshot.id}`,
267
+ values: { approvalStatus: 'pending' },
268
+ });
269
+ return;
270
+ }
271
+
214
272
  await this.stateMachine.transition({
215
273
  runId: snapshot.id,
216
274
  to: 'running',
@@ -218,35 +276,91 @@ export class LoopWorkerService {
218
276
  actorIdentity: this.workerId,
219
277
  leaseToken,
220
278
  eventType: 'run_started',
221
- title: 'Run started',
222
- content: `Leader ${snapshot.leaderUsername} began working on the goal.`,
279
+ title: snapshot.resumeContext ? 'Run resumed' : 'Run started',
280
+ content: snapshot.resumeContext
281
+ ? `${snapshot.resumeContext.username} resumed after approval decisions.`
282
+ : `Leader ${snapshot.leaderUsername} began working on the goal.`,
223
283
  correlationKey: `started:${snapshot.id}`,
224
- values: { currentRole: 'leader' },
284
+ values: { currentRole: snapshot.resumeContext ? snapshot.resumeContext.role : 'leader' },
225
285
  });
226
286
 
227
- const leader = await this.invokeRole(snapshot, {
228
- role: 'leader',
229
- username: snapshot.leaderUsername,
230
- harness: snapshot.leaderHarness,
231
- systemMessage:
232
- 'You lead an autonomous run. Break the goal into concrete work for your makers, then finish with a concise plan of what must be built and which evidence will prove it. You do not modify anything yourself.',
233
- prompt: snapshot.goal,
234
- leaseToken,
235
- signal,
236
- });
287
+ // Loops recorded before this claim (for example before a park for approval) keep warning the
288
+ // prompts composed later in the run.
289
+ const loopNotices = await this.loopGuard.existingNotices(snapshot.id);
237
290
 
238
- // plugin-ai stops the graph on an approval-gated tool. Approval is a human decision, so the
239
- // run parks here instead of the worker deciding on the operator's behalf.
240
- if (leader.interrupted.length) {
241
- await this.parkForApproval(snapshot, leader, leaseToken);
242
- return;
291
+ let makerReports: string[];
292
+ let makerQueue: string[];
293
+
294
+ if (snapshot.resumeContext) {
295
+ const resume = snapshot.resumeContext;
296
+ const decisions = await this.consumeDecisions(snapshot.id);
297
+ const resumed = await this.resumeRole(snapshot, resume, decisions, signal);
298
+
299
+ const resumeGuard = await this.enforceToolLoopGuard(
300
+ snapshot,
301
+ leaseToken,
302
+ { role: resume.role, username: resume.username },
303
+ loopNotices,
304
+ );
305
+ if (resumeGuard === 'halt') return;
306
+
307
+ // A resumed conversation can hit another approval gate; park with the same role context so
308
+ // the next resume continues exactly where this one stopped.
309
+ if (resumed.interrupted.length) {
310
+ await this.parkForApproval(snapshot, resumed, leaseToken, {
311
+ role: resume.role,
312
+ username: resume.username,
313
+ makerReports: resume.makerReports,
314
+ makerQueue: resume.makerQueue,
315
+ });
316
+ return;
317
+ }
318
+ if (resume.role === 'leader') {
319
+ makerReports = [resumed.content];
320
+ makerQueue = [...snapshot.makerUsernames];
321
+ } else {
322
+ makerReports = [...resume.makerReports, `# ${resume.username}\n${resumed.content}`];
323
+ makerQueue = [...resume.makerQueue];
324
+ }
325
+ } else {
326
+ const leader = await this.invokeRole(snapshot, {
327
+ role: 'leader',
328
+ username: snapshot.leaderUsername,
329
+ harness: snapshot.leaderHarness,
330
+ systemMessage: LEADER_SYSTEM_MESSAGE,
331
+ prompt: snapshot.goal,
332
+ leaseToken,
333
+ signal,
334
+ });
335
+
336
+ const leaderGuard = await this.enforceToolLoopGuard(
337
+ snapshot,
338
+ leaseToken,
339
+ { role: 'leader', username: snapshot.leaderUsername },
340
+ loopNotices,
341
+ );
342
+ if (leaderGuard === 'halt') return;
343
+
344
+ // plugin-ai stops the graph on an approval-gated tool. Approval is a human decision, so the
345
+ // run parks here instead of the worker deciding on the operator's behalf.
346
+ if (leader.interrupted.length) {
347
+ await this.parkForApproval(snapshot, leader, leaseToken, {
348
+ role: 'leader',
349
+ username: snapshot.leaderUsername,
350
+ makerReports: [],
351
+ makerQueue: snapshot.makerUsernames,
352
+ });
353
+ return;
354
+ }
355
+ makerReports = [leader.content];
356
+ makerQueue = [...snapshot.makerUsernames];
243
357
  }
244
358
 
245
359
  // Each maker executes with its own compiled harness snapshot. Reusing one harness would apply
246
360
  // the wrong per-employee tool grants, so the snapshot is looked up by username and a maker with
247
361
  // no snapshot is a compile error, not a silent skip.
248
- const makerReports: string[] = [leader.content];
249
- for (const username of snapshot.makerUsernames) {
362
+ while (makerQueue.length) {
363
+ const username = makerQueue[0];
250
364
  const harness = snapshot.makerHarnesses[username]?.effective;
251
365
  if (!harness) {
252
366
  throw new Error(`The claimed run has no harness snapshot for maker "${username}".`);
@@ -255,22 +369,35 @@ export class LoopWorkerService {
255
369
  role: 'maker',
256
370
  username,
257
371
  harness,
258
- systemMessage:
259
- 'You are a maker on an autonomous run. Do the work assigned by the leader using only the tools you were granted, and report exactly what you changed and the evidence for it.',
372
+ systemMessage: MAKER_SYSTEM_MESSAGE,
260
373
  prompt: [
261
374
  `Goal: ${snapshot.goal}`,
262
375
  '',
263
376
  "Leader's plan:",
264
- leader.content || '(the leader produced no textual plan)',
377
+ makerReports[0] || '(the leader produced no textual plan)',
378
+ ...loopNotices,
265
379
  ].join('\n'),
266
380
  leaseToken,
267
381
  signal,
268
382
  });
383
+ const makerGuard = await this.enforceToolLoopGuard(
384
+ snapshot,
385
+ leaseToken,
386
+ { role: 'maker', username },
387
+ loopNotices,
388
+ );
389
+ if (makerGuard === 'halt') return;
269
390
  if (maker.interrupted.length) {
270
- await this.parkForApproval(snapshot, maker, leaseToken);
391
+ await this.parkForApproval(snapshot, maker, leaseToken, {
392
+ role: 'maker',
393
+ username,
394
+ makerReports,
395
+ makerQueue: makerQueue.slice(1),
396
+ });
271
397
  return;
272
398
  }
273
399
  makerReports.push(`# ${username}\n${maker.content}`);
400
+ makerQueue = makerQueue.slice(1);
274
401
  }
275
402
  const makerSummary = makerReports.join('\n\n');
276
403
 
@@ -306,9 +433,14 @@ export class LoopWorkerService {
306
433
  userId: snapshot.userId,
307
434
  workerId: this.workerId,
308
435
  leaseToken,
436
+ loopNotices,
309
437
  signal,
310
438
  });
311
439
 
440
+ // The tool loop guard already moved the run to blocked or waiting_human; the circuit breaker
441
+ // tracks failed attempts, not behavioral stops, so it must not see this as a failure.
442
+ if (outcome.finalStatus === 'halted') return;
443
+
312
444
  if (outcome.finalStatus === 'failed') {
313
445
  await this.circuits.recordFailure({
314
446
  patternId: snapshot.patternId,
@@ -387,6 +519,7 @@ export class LoopWorkerService {
387
519
  kind: 'invocation',
388
520
  type: 'reasoning',
389
521
  employeeUsername: input.username,
522
+ sessionId,
390
523
  title: `${input.role} invocation`,
391
524
  inputHash: inputHash(input.prompt),
392
525
  status: 'running',
@@ -420,21 +553,166 @@ export class LoopWorkerService {
420
553
  return outcome;
421
554
  }
422
555
 
423
- private async parkForApproval(snapshot: RunSnapshot, outcome: InvocationOutcome, leaseToken: string) {
556
+ private async resumeRole(
557
+ snapshot: RunSnapshot,
558
+ resume: ResumeContext,
559
+ decisions: Array<{ toolCallId: string; decision: ToolCallDecision }>,
560
+ signal: AbortSignal,
561
+ ) {
562
+ await this.budgets.reserve({
563
+ runId: snapshot.id,
564
+ patternId: snapshot.patternId,
565
+ policy: snapshot.policy,
566
+ delta: { invocations: 1 },
567
+ });
568
+ const harness =
569
+ resume.role === 'leader' ? snapshot.leaderHarness : snapshot.makerHarnesses[resume.username]?.effective;
570
+ if (!harness) {
571
+ throw new Error(`The claimed run has no harness snapshot for ${resume.role} "${resume.username}".`);
572
+ }
573
+ const step = await this.database.getRepository('agentLoopSteps').create({
574
+ values: {
575
+ runId: snapshot.id,
576
+ runtimeVersion: 'control-plane-v2',
577
+ sequence: await this.nextSequence(snapshot.id),
578
+ role: resume.role,
579
+ kind: 'invocation',
580
+ type: 'resume',
581
+ employeeUsername: resume.username,
582
+ sessionId: resume.sessionId,
583
+ title: `${resume.role} resume after approval`,
584
+ inputHash: inputHash(decisions),
585
+ status: 'running',
586
+ startedAt: new Date(),
587
+ createdAt: new Date(),
588
+ },
589
+ });
590
+
591
+ const outcome = await this.runtime.resume({
592
+ username: resume.username,
593
+ sessionId: resume.sessionId,
594
+ messageId: resume.messageId,
595
+ userId: snapshot.userId,
596
+ systemMessage: resume.role === 'leader' ? LEADER_SYSTEM_MESSAGE : MAKER_SYSTEM_MESSAGE,
597
+ harness,
598
+ decisions,
599
+ signal,
600
+ });
601
+
602
+ await this.database.getRepository('agentLoopSteps').update({
603
+ filterByTk: read(step, 'id'),
604
+ values: {
605
+ status: outcome.interrupted.length ? 'waiting_user' : 'succeeded',
606
+ output: { content: outcome.content },
607
+ endedAt: new Date(),
608
+ },
609
+ });
610
+ await this.database.getRepository('agentLoopRuns').update({
611
+ filterByTk: snapshot.id,
612
+ values: { sessionId: outcome.sessionId, messageId: outcome.messageId },
613
+ });
614
+ return outcome;
615
+ }
616
+
617
+ // Run-level scan over every tool call issued so far, evaluated at each pass boundary. A warning
618
+ // is injected into the prompts composed later in the run; block and escalate halt it at once.
619
+ private async enforceToolLoopGuard(
620
+ snapshot: RunSnapshot,
621
+ leaseToken: string,
622
+ pass: { role: 'leader' | 'maker'; username: string },
623
+ loopNotices: string[],
624
+ ): Promise<'continue' | 'halt'> {
625
+ const finding = await this.loopGuard.scanRun(snapshot.id, snapshot.policy);
626
+ if (!finding) return 'continue';
627
+
628
+ await this.loopGuard.recordFinding(
629
+ {
630
+ runId: snapshot.id,
631
+ role: pass.role,
632
+ username: pass.username,
633
+ runStatus: 'running',
634
+ actorType: 'worker',
635
+ actorIdentity: this.workerId,
636
+ },
637
+ finding,
638
+ );
639
+
640
+ if (finding.level === 'warn') {
641
+ const notice = toolLoopNotice(finding.toolName, finding.count);
642
+ if (!loopNotices.includes(notice)) loopNotices.push(notice);
643
+ return 'continue';
644
+ }
645
+
646
+ const reason = toolLoopReason(finding);
647
+ await this.stateMachine.transition({
648
+ runId: snapshot.id,
649
+ to: finding.level === 'block' ? 'blocked' : 'waiting_human',
650
+ actorType: 'worker',
651
+ actorIdentity: this.workerId,
652
+ leaseToken,
653
+ eventType: finding.level === 'block' ? 'run_blocked' : 'run_escalated',
654
+ title: finding.level === 'block' ? 'Run blocked by tool loop detection' : 'Run escalated by tool loop detection',
655
+ content: reason,
656
+ correlationKey: `tool-loop-${finding.level}:${snapshot.id}:${finding.signature}`,
657
+ values: finding.level === 'block' ? { blockedReason: reason } : { escalationReason: reason },
658
+ });
659
+ return 'halt';
660
+ }
661
+
662
+ // Decisions are consumed exactly once: rows are marked consumed so a retry or a second resume
663
+ // can never replay a human decision against a fresh conversation.
664
+ private async consumeDecisions(runId: number) {
665
+ const repository = this.database.getRepository('agentLoopActionApprovals');
666
+ const approvals = await repository.find({ filter: { runId } });
667
+ const decisions: Array<{ toolCallId: string; decision: ToolCallDecision }> = [];
668
+ const now = new Date();
669
+ for (const approval of approvals) {
670
+ const status = String(read(approval, 'status'));
671
+ if (status !== 'approved' && status !== 'rejected') continue;
672
+ if (read(approval, 'consumedAt')) continue;
673
+ const editedInput = read(approval, 'editedInput');
674
+ const decision: ToolCallDecision =
675
+ status === 'approved'
676
+ ? editedInput
677
+ ? { type: 'edit', editedAction: { name: String(read(approval, 'toolName')), args: editedInput } }
678
+ : { type: 'approve' }
679
+ : { type: 'reject', message: String(read(approval, 'decisionNote') || '') || undefined };
680
+ decisions.push({ toolCallId: String(read(approval, 'toolCallId')), decision });
681
+ await repository.update({ filterByTk: Number(read(approval, 'id')), values: { consumedAt: now } });
682
+ }
683
+ if (!decisions.length) throw new Error('The resumed run has no recorded approval decisions.');
684
+ return decisions;
685
+ }
686
+
687
+ private async parkForApproval(
688
+ snapshot: RunSnapshot,
689
+ outcome: InvocationOutcome,
690
+ leaseToken: string,
691
+ context: { role: 'leader' | 'maker'; username: string; makerReports: string[]; makerQueue: string[] },
692
+ ) {
424
693
  const now = new Date();
425
694
  const expiresAt = new Date(now.getTime() + snapshot.policy.actions.approvalTimeoutMs);
426
695
  const assignedToId = snapshot.policy.actions.approvalAssigneeIds[0] || snapshot.userId || null;
696
+ const harness =
697
+ context.role === 'leader' ? snapshot.leaderHarness : snapshot.makerHarnesses[context.username]?.effective;
698
+ const escalatable = new Set(harness?.tools.escalate ?? []);
427
699
  for (const call of outcome.interrupted) {
700
+ // An escalation is a request for authority the harness never granted, widened for exactly
701
+ // one call; a plain tool_call approval gates a tool the harness allows but asks about.
702
+ const escalation = escalatable.has(call.toolName);
428
703
  await this.database.getRepository('agentLoopActionApprovals').create({
429
704
  values: {
430
705
  runId: snapshot.id,
431
706
  toolCallId: call.toolCallId,
432
707
  toolName: call.toolName,
708
+ actionType: escalation ? 'escalation' : 'tool_call',
433
709
  proposedInput: call.args,
434
710
  inputHash: inputHash(call.args),
435
711
  paths: snapshot.actingOn,
436
712
  policyDecision: { decision: 'ask', interruptId: call.interruptId },
437
- reason: `Tool "${call.toolName}" requires human approval under this pattern's policy.`,
713
+ reason: escalation
714
+ ? `Tool "${call.toolName}" is not granted by this run's harness. Approving widens authority for this single call only.`
715
+ : `Tool "${call.toolName}" requires human approval under this pattern's policy.`,
438
716
  status: 'pending',
439
717
  assignedToId,
440
718
  requestedById: snapshot.userId || null,
@@ -444,6 +722,22 @@ export class LoopWorkerService {
444
722
  },
445
723
  });
446
724
  }
725
+ await this.database.getRepository('agentLoopRuns').update({
726
+ filterByTk: snapshot.id,
727
+ values: {
728
+ // The resume point is durable: which role stopped, in which conversation, with which
729
+ // reports already collected and which makers still queued. Without it a resumed run
730
+ // would restart from the leader and redo every completed invocation.
731
+ resumeContext: {
732
+ role: context.role,
733
+ username: context.username,
734
+ sessionId: outcome.sessionId,
735
+ messageId: outcome.messageId,
736
+ makerReports: context.makerReports,
737
+ makerQueue: context.makerQueue,
738
+ },
739
+ },
740
+ });
447
741
  await this.stateMachine.transition({
448
742
  runId: snapshot.id,
449
743
  to: 'waiting_approval',
@@ -502,6 +796,7 @@ export class LoopWorkerService {
502
796
  }
503
797
  const roleBindings = asObject(run.roleBindingsSnapshot);
504
798
  const makerSnapshots = asObject(run.makerHarnessSnapshot);
799
+ const approvalStatus = String(run.approvalStatus || 'none');
505
800
  return {
506
801
  id: Number(run.id),
507
802
  patternId,
@@ -517,6 +812,11 @@ export class LoopWorkerService {
517
812
  repositoryKey: String(run.repositoryKey || ''),
518
813
  actingOn: asStringArray(run.actingOn),
519
814
  userId: Number(run.userId) || undefined,
815
+ approvalStatus,
816
+ // Only a live approval state may resume. Stale context left over from an expired window or a
817
+ // cleared retry would otherwise replay an old conversation against fresh approvals.
818
+ resumeContext:
819
+ approvalStatus === 'decided' || approvalStatus === 'pending' ? resumeContextFrom(run.resumeContext) : null,
520
820
  };
521
821
  }
522
822
  }