plugin-agent-orchestrator 1.1.3 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/README.md +204 -114
  2. package/dist/client/index.js +1 -1
  3. package/dist/client-v2/117.7565cff57c39318a.js +10 -0
  4. package/dist/client-v2/214.106635011859dede.js +10 -0
  5. package/dist/client-v2/253.59577b40d9875be0.js +10 -0
  6. package/dist/client-v2/264.7ed8bee71fa5935c.js +10 -0
  7. package/dist/client-v2/{41.ba6e080cc0488143.js → 41.3deb9e57a0f64fc4.js} +1 -1
  8. package/dist/client-v2/418.41ed200e7d6da332.js +10 -0
  9. package/dist/client-v2/53.349fff8db80b19d4.js +10 -0
  10. package/dist/client-v2/619.0a2144aaffefe018.js +10 -0
  11. package/dist/client-v2/{70.bda9518881c05360.js → 70.ffff2997c10fd4b0.js} +1 -1
  12. package/dist/client-v2/{794.9bbb8d3a61b2bdf2.js → 794.a67a2614405ba627.js} +1 -1
  13. package/dist/client-v2/981.e4a1f41cfc94a856.js +10 -0
  14. package/dist/client-v2/{993.82778b6fd6db3cb1.js → 993.b0688fdc5ccb000a.js} +1 -1
  15. package/dist/client-v2/index.js +1 -1
  16. package/dist/externalVersion.js +8 -7
  17. package/dist/locale/en-US.json +199 -123
  18. package/dist/locale/vi-VN.json +199 -123
  19. package/dist/locale/zh-CN.json +199 -123
  20. package/dist/server/collections/agent-execution-spans.js +55 -0
  21. package/dist/server/collections/agent-harness-profile-versions.js +56 -0
  22. package/dist/server/collections/agent-harness-profiles.js +12 -0
  23. package/dist/server/collections/agent-loop-action-approvals.js +69 -0
  24. package/dist/server/collections/agent-loop-artifacts.js +55 -0
  25. package/dist/server/collections/agent-loop-circuit-states.js +53 -0
  26. package/dist/server/collections/agent-loop-control-settings.js +51 -0
  27. package/dist/server/collections/agent-loop-events.js +27 -0
  28. package/dist/server/collections/agent-loop-path-locks.js +57 -0
  29. package/dist/server/collections/agent-loop-patterns.js +72 -0
  30. package/dist/server/collections/agent-loop-runs.js +170 -3
  31. package/dist/server/collections/agent-loop-steps.js +76 -0
  32. package/dist/server/collections/agent-loop-usage-buckets.js +52 -0
  33. package/dist/server/collections/agent-loop-worktrees.js +55 -0
  34. package/dist/server/collections/skill-executions.js +8 -0
  35. package/dist/server/migrations/{20260427000000-change-packages-to-text.js → 20260427000002-change-packages-to-text.js} +2 -0
  36. package/dist/server/migrations/20260429000000-add-llm-fields.js +3 -2
  37. package/dist/server/migrations/{20260429000000-fix-inputargs-json-to-text.js → 20260429000001-fix-inputargs-json-to-text.js} +14 -6
  38. package/dist/server/migrations/20260503000000-add-orchestrator-trace-fields.js +2 -1
  39. package/dist/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.js +2 -1
  40. package/dist/server/migrations/20260803010000-backfill-loop-control-plane-data.js +262 -0
  41. package/dist/server/plugin.js +54 -150
  42. package/dist/server/resources/agent-harness-profile.js +117 -0
  43. package/dist/server/resources/agent-knowledge-insights.js +1 -3
  44. package/dist/server/resources/agent-loop-approval.js +108 -0
  45. package/dist/server/resources/agent-loop-artifact.js +86 -0
  46. package/dist/server/resources/agent-loop-events-stream.js +130 -0
  47. package/dist/server/resources/agent-loop.js +183 -139
  48. package/dist/server/resources/resource-helpers.js +63 -0
  49. package/dist/server/resources/tracing.js +121 -107
  50. package/dist/server/services/AgentMemoryContextService.js +34 -0
  51. package/dist/server/services/DurableCircuitBreakerService.js +229 -0
  52. package/dist/server/services/HarnessCompiler.js +175 -0
  53. package/dist/server/services/HarnessProfileService.js +233 -0
  54. package/dist/server/services/HarnessSchema.js +194 -0
  55. package/dist/server/services/LoopBudgetService.js +269 -0
  56. package/dist/server/services/LoopControlService.js +139 -0
  57. package/dist/server/services/LoopPatternSchema.js +195 -0
  58. package/dist/server/services/LoopPatternService.js +132 -0
  59. package/dist/server/services/LoopRunRepository.js +285 -0
  60. package/dist/server/services/LoopRunStateMachine.js +478 -0
  61. package/dist/server/services/LoopSchedulerService.js +193 -0
  62. package/dist/server/services/LoopTriggerService.js +165 -0
  63. package/dist/server/services/LoopWorkerService.js +699 -0
  64. package/dist/server/services/NativeSubAgentObserver.js +165 -23
  65. package/dist/server/services/PathLockService.js +288 -0
  66. package/dist/server/services/PluginAiRuntimeAdapter.js +226 -0
  67. package/dist/server/services/ToolLoopDetectionService.js +210 -0
  68. package/dist/server/services/VerificationSchema.js +59 -0
  69. package/dist/server/services/VerificationService.js +235 -0
  70. package/dist/server/services/harness-runtime-policy.js +139 -0
  71. package/dist/server/skill-hub/actions/git-import.js +13 -7
  72. package/dist/server/skill-hub/plugin.js +36 -3
  73. package/dist/server/skill-hub/services/RegistrySkillSnapshotService.js +1 -5
  74. package/dist/server/skill-hub/tasks/SkillExecutionTask.js +29 -0
  75. package/dist/server/utils/ensure-indexes.js +111 -0
  76. package/dist/server/utils/loop-pattern-context.js +56 -0
  77. package/package.json +53 -53
  78. package/src/client/ApprovalsTab.tsx +2 -0
  79. package/src/client/OrchestratorSettings.tsx +55 -0
  80. package/src/client/skill-hub/components/GitSkillImport.tsx +3 -3
  81. package/src/client/skill-hub/components/SkillMetrics.tsx +15 -10
  82. package/src/client/skill-hub/index.tsx +0 -52
  83. package/src/client-v2/components/AgentRunsTab.tsx +100 -18
  84. package/src/client-v2/components/ApprovalsTab.tsx +337 -0
  85. package/src/client-v2/components/HarnessProfilesTab.tsx +385 -100
  86. package/src/client-v2/components/MemoryInspectorTab.tsx +141 -125
  87. package/src/client-v2/components/TracingTab.tsx +160 -77
  88. package/src/client-v2/hooks/useApiRequest.ts +28 -8
  89. package/src/client-v2/pages/ApprovalsPage.tsx +10 -0
  90. package/src/client-v2/plugin.tsx +6 -6
  91. package/src/client-v2/skill-hub/components/SkillMetrics.tsx +11 -6
  92. package/src/locale/en-US.json +199 -123
  93. package/src/locale/vi-VN.json +199 -123
  94. package/src/locale/zh-CN.json +199 -123
  95. package/src/server/__tests__/circuit-breaker.test.ts +248 -169
  96. package/src/server/__tests__/harness-compiler.test.ts +214 -0
  97. package/src/server/__tests__/harness-profile-service.test.ts +167 -0
  98. package/src/server/__tests__/harness-runtime-policy.test.ts +293 -0
  99. package/src/server/__tests__/harness-tool-plan.test.ts +172 -0
  100. package/src/server/__tests__/loop-approval-decision.test.ts +512 -0
  101. package/src/server/__tests__/loop-approval-expiry.test.ts +258 -0
  102. package/src/server/__tests__/loop-budget-service.test.ts +255 -0
  103. package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -0
  104. package/src/server/__tests__/loop-pattern-schema.test.ts +73 -0
  105. package/src/server/__tests__/loop-pattern-service.test.ts +148 -0
  106. package/src/server/__tests__/loop-run-state-machine.test.ts +101 -0
  107. package/src/server/__tests__/loop-trigger-service.test.ts +148 -0
  108. package/src/server/__tests__/loop-worker-claim.test.ts +341 -0
  109. package/src/server/__tests__/loop-worker-execution.test.ts +317 -0
  110. package/src/server/__tests__/native-sub-agent-observer.test.ts +288 -4
  111. package/src/server/__tests__/path-lock-service.test.ts +254 -0
  112. package/src/server/__tests__/skill-execution-task-policy.test.ts +140 -0
  113. package/src/server/__tests__/smoke.test.ts +162 -121
  114. package/src/server/__tests__/tool-loop-detection.test.ts +275 -0
  115. package/src/server/collections/agent-execution-spans.ts +55 -0
  116. package/src/server/collections/agent-harness-profile-versions.ts +26 -0
  117. package/src/server/collections/agent-harness-profiles.ts +71 -59
  118. package/src/server/collections/agent-loop-action-approvals.ts +39 -0
  119. package/src/server/collections/agent-loop-artifacts.ts +25 -0
  120. package/src/server/collections/agent-loop-circuit-states.ts +23 -0
  121. package/src/server/collections/agent-loop-control-settings.ts +21 -0
  122. package/src/server/collections/agent-loop-events.ts +98 -71
  123. package/src/server/collections/agent-loop-path-locks.ts +27 -0
  124. package/src/server/collections/agent-loop-patterns.ts +42 -0
  125. package/src/server/collections/agent-loop-runs.ts +170 -3
  126. package/src/server/collections/agent-loop-steps.ts +76 -0
  127. package/src/server/collections/agent-loop-usage-buckets.ts +22 -0
  128. package/src/server/collections/agent-loop-worktrees.ts +25 -0
  129. package/src/server/collections/skill-executions.ts +8 -0
  130. package/src/server/migrations/{20260427000000-change-packages-to-text.ts → 20260427000002-change-packages-to-text.ts} +52 -49
  131. package/src/server/migrations/20260429000000-add-llm-fields.ts +47 -46
  132. package/src/server/migrations/{20260429000000-fix-inputargs-json-to-text.ts → 20260429000001-fix-inputargs-json-to-text.ts} +45 -38
  133. package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +33 -32
  134. package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +31 -30
  135. package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -0
  136. package/src/server/plugin.ts +55 -167
  137. package/src/server/resources/agent-harness-profile.ts +94 -0
  138. package/src/server/resources/agent-knowledge-insights.ts +5 -3
  139. package/src/server/resources/agent-loop-approval.ts +84 -0
  140. package/src/server/resources/agent-loop-artifact.ts +56 -0
  141. package/src/server/resources/agent-loop-events-stream.ts +105 -0
  142. package/src/server/resources/agent-loop.ts +204 -143
  143. package/src/server/resources/agent-monitor.ts +482 -482
  144. package/src/server/resources/resource-helpers.ts +33 -0
  145. package/src/server/resources/tracing.ts +266 -258
  146. package/src/server/services/AgentMemoryContextService.ts +44 -0
  147. package/src/server/services/DurableCircuitBreakerService.ts +239 -0
  148. package/src/server/services/ExecutionSpanService.ts +2 -0
  149. package/src/server/services/HarnessCompiler.ts +170 -0
  150. package/src/server/services/HarnessProfileService.ts +239 -0
  151. package/src/server/services/HarnessSchema.ts +194 -0
  152. package/src/server/services/LoopBudgetService.ts +275 -0
  153. package/src/server/services/LoopControlService.ts +124 -0
  154. package/src/server/services/LoopPatternSchema.ts +185 -0
  155. package/src/server/services/LoopPatternService.ts +143 -0
  156. package/src/server/services/LoopRunRepository.ts +336 -0
  157. package/src/server/services/LoopRunStateMachine.ts +529 -0
  158. package/src/server/services/LoopSchedulerService.ts +199 -0
  159. package/src/server/services/LoopTriggerService.ts +157 -0
  160. package/src/server/services/LoopWorkerService.ts +822 -0
  161. package/src/server/services/NativeSubAgentObserver.ts +186 -21
  162. package/src/server/services/PathLockService.ts +321 -0
  163. package/src/server/services/PluginAiRuntimeAdapter.ts +332 -0
  164. package/src/server/services/ToolLoopDetectionService.ts +213 -0
  165. package/src/server/services/VerificationSchema.ts +31 -0
  166. package/src/server/services/VerificationService.ts +264 -0
  167. package/src/server/services/harness-runtime-policy.ts +141 -0
  168. package/src/server/skill-hub/actions/__tests__/git-import.test.ts +15 -1
  169. package/src/server/skill-hub/actions/git-import.ts +9 -3
  170. package/src/server/skill-hub/plugin.ts +41 -3
  171. package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +235 -239
  172. package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +61 -63
  173. package/src/server/skill-hub/tasks/SkillExecutionTask.ts +43 -2
  174. package/src/server/utils/ensure-indexes.ts +96 -0
  175. package/src/server/utils/logging.ts +62 -63
  176. package/src/server/utils/loop-pattern-context.ts +20 -0
  177. package/dist/client-v2/214.79650a549273f163.js +0 -10
  178. package/dist/client-v2/253.9f0537713cee9e6c.js +0 -10
  179. package/dist/client-v2/264.718a107e43fc163c.js +0 -10
  180. package/dist/client-v2/373.f5d5292e53c4e832.js +0 -10
  181. package/dist/client-v2/418.ce8f5b2b8a229d1d.js +0 -10
  182. package/dist/client-v2/53.7c29c28043e9112d.js +0 -10
  183. package/dist/client-v2/619.bd3c5698b40705c3.js +0 -10
  184. package/dist/client-v2/677.a991ce0250ff5c77.js +0 -10
  185. package/dist/client-v2/925.f5370de8f6632d65.js +0 -10
  186. package/dist/server/services/AgentHarness.js +0 -81
  187. package/dist/server/services/AgentLoopController.js +0 -1023
  188. package/dist/server/services/AgentLoopRepository.js +0 -192
  189. package/dist/server/services/AgentLoopService.js +0 -142
  190. package/dist/server/services/AgentPlanValidator.js +0 -99
  191. package/dist/server/services/AgentPlannerService.js +0 -99
  192. package/dist/server/services/CircuitBreaker.js +0 -125
  193. package/dist/server/services/ContextAggregator.js +0 -201
  194. package/dist/server/services/PlanningPromptService.js +0 -95
  195. package/dist/server/tools/agent-loop.js +0 -373
  196. package/dist/server/tools/orchestrator-plan.js +0 -254
  197. package/src/client/AIEmployeeSelect.tsx +0 -47
  198. package/src/client/AIEmployeesContext.tsx +0 -102
  199. package/src/client/RulesTab.tsx +0 -2
  200. package/src/client/TracingTab.tsx +0 -436
  201. package/src/client/skill-hub/components/ExecutionProgress.tsx +0 -51
  202. package/src/client-v2/components/RulesTab.tsx +0 -2
  203. package/src/client-v2/pages/AgentRunsPage.tsx +0 -13
  204. package/src/server/__tests__/agent-loop-controller.test.ts +0 -375
  205. package/src/server/__tests__/context-aggregator.test.ts +0 -222
  206. package/src/server/__tests__/parallel-execution.test.ts +0 -318
  207. package/src/server/__tests__/planning-prompt-service.test.ts +0 -23
  208. package/src/server/services/AgentHarness.ts +0 -67
  209. package/src/server/services/AgentLoopController.ts +0 -1263
  210. package/src/server/services/AgentLoopRepository.ts +0 -178
  211. package/src/server/services/AgentLoopService.ts +0 -177
  212. package/src/server/services/AgentPlanValidator.ts +0 -73
  213. package/src/server/services/AgentPlannerService.ts +0 -80
  214. package/src/server/services/CircuitBreaker.ts +0 -126
  215. package/src/server/services/ContextAggregator.ts +0 -239
  216. package/src/server/services/PlanningPromptService.ts +0 -72
  217. package/src/server/tools/agent-loop.ts +0 -360
  218. package/src/server/tools/orchestrator-plan.ts +0 -255
@@ -0,0 +1,529 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import type { Database, Model, Transaction } from '@nocobase/database';
3
+ import { LoopControlService } from './LoopControlService';
4
+ import { loopPatternPolicySchema } from './LoopPatternSchema';
5
+ import { getRunEventBus } from './RunEventBus';
6
+ import { isPassingVerification } from './VerificationSchema';
7
+
8
+ export const loopRunStatuses = [
9
+ 'queued',
10
+ 'preparing',
11
+ 'waiting_lock',
12
+ 'running',
13
+ 'waiting_approval',
14
+ 'verifying',
15
+ 'waiting_human',
16
+ 'paused',
17
+ 'blocked',
18
+ 'succeeded',
19
+ 'failed',
20
+ 'canceled',
21
+ ] as const;
22
+
23
+ export type LoopRunStatus = (typeof loopRunStatuses)[number];
24
+
25
+ const transitions: Record<LoopRunStatus, ReadonlySet<LoopRunStatus>> = {
26
+ queued: new Set(['preparing', 'paused', 'blocked', 'canceled']),
27
+ preparing: new Set([
28
+ 'queued',
29
+ 'waiting_lock',
30
+ 'waiting_approval',
31
+ 'running',
32
+ 'paused',
33
+ 'blocked',
34
+ 'failed',
35
+ 'canceled',
36
+ ]),
37
+ waiting_lock: new Set(['preparing', 'paused', 'blocked', 'canceled']),
38
+ running: new Set([
39
+ 'queued',
40
+ 'waiting_approval',
41
+ 'verifying',
42
+ 'waiting_human',
43
+ 'paused',
44
+ 'blocked',
45
+ 'failed',
46
+ 'canceled',
47
+ ]),
48
+ waiting_approval: new Set(['queued', 'running', 'waiting_human', 'paused', 'blocked', 'failed', 'canceled']),
49
+ verifying: new Set(['running', 'waiting_human', 'paused', 'blocked', 'succeeded', 'failed', 'canceled']),
50
+ waiting_human: new Set(['running', 'succeeded', 'failed', 'canceled']),
51
+ paused: new Set(['queued', 'preparing', 'running', 'waiting_approval', 'verifying', 'blocked', 'canceled']),
52
+ blocked: new Set(['queued', 'waiting_human', 'failed', 'canceled']),
53
+ succeeded: new Set(),
54
+ failed: new Set(['queued']),
55
+ canceled: new Set(),
56
+ };
57
+
58
+ export type TransitionInput = {
59
+ from: LoopRunStatus;
60
+ to: LoopRunStatus;
61
+ runtimeVersion?: string;
62
+ verifierEvidence?: unknown;
63
+ humanAccepted?: boolean;
64
+ };
65
+
66
+ export function assertRunTransition(input: TransitionInput) {
67
+ if (input.runtimeVersion && input.runtimeVersion !== 'control-plane-v2') {
68
+ throw new Error('Historical plan-era runs are read-only.');
69
+ }
70
+ if (input.from === input.to) {
71
+ throw new Error(`Run is already ${input.to}.`);
72
+ }
73
+ if (!transitions[input.from].has(input.to)) {
74
+ throw new Error(`Illegal run transition: ${input.from} -> ${input.to}.`);
75
+ }
76
+ if (input.to === 'succeeded') {
77
+ if (!isPassingVerification(input.verifierEvidence)) {
78
+ throw new Error('A run cannot succeed without a structured passing verifier verdict.');
79
+ }
80
+ if (input.from === 'waiting_human' && input.humanAccepted !== true) {
81
+ throw new Error('Human acceptance is required to finish a run waiting for human review.');
82
+ }
83
+ }
84
+ }
85
+
86
+ function read(record: Model | Record<string, unknown>, key: string) {
87
+ const model = record as Model & { get?: (name: string) => unknown };
88
+ return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
89
+ }
90
+
91
+ function asStatus(value: unknown): LoopRunStatus {
92
+ if (typeof value === 'string' && (loopRunStatuses as readonly string[]).includes(value)) {
93
+ return value as LoopRunStatus;
94
+ }
95
+ throw new Error(`Unknown Loop Control Plane run status: ${String(value)}.`);
96
+ }
97
+
98
+ const activeExecutionStatuses: LoopRunStatus[] = ['preparing', 'running', 'verifying'];
99
+ const claimableStatuses: LoopRunStatus[] = ['queued', 'waiting_lock'];
100
+
101
+ function asDate(value: unknown): Date | null {
102
+ if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value;
103
+ if (typeof value === 'string' || typeof value === 'number') {
104
+ const parsed = new Date(value);
105
+ return Number.isNaN(parsed.getTime()) ? null : parsed;
106
+ }
107
+ return null;
108
+ }
109
+
110
+ function leaseIsExpired(record: Model | Record<string, unknown>, now: Date) {
111
+ const lockedUntil = asDate(read(record, 'lockedUntil'));
112
+ return !lockedUntil || lockedUntil.getTime() <= now.getTime();
113
+ }
114
+
115
+ function leaseOwner(record: Model | Record<string, unknown>) {
116
+ const owner = read(record, 'lockedBy');
117
+ return typeof owner === 'string' && owner.trim() ? owner : null;
118
+ }
119
+
120
+ // Only a run someone actually claimed can lose its lease. An active run with no owner still
121
+ // occupies a concurrency slot: requeueing it would dispatch work that may already be running.
122
+ function leaseIsReclaimable(record: Model | Record<string, unknown>, now: Date) {
123
+ return leaseOwner(record) !== null && leaseIsExpired(record, now);
124
+ }
125
+
126
+ function asPositiveInteger(value: unknown, label: string) {
127
+ const parsed = Number(value);
128
+ if (!Number.isSafeInteger(parsed) || parsed <= 0) throw new Error(`${label} is invalid.`);
129
+ return parsed;
130
+ }
131
+
132
+ function asPlain(record: Model | Record<string, unknown>) {
133
+ return typeof (record as Model).toJSON === 'function'
134
+ ? ((record as Model).toJSON() as Record<string, unknown>)
135
+ : structuredClone(record as Record<string, unknown>);
136
+ }
137
+
138
+ export type ClaimedLoopRun = {
139
+ run: Record<string, unknown>;
140
+ leaseToken: string;
141
+ leaseUntil: Date;
142
+ };
143
+
144
+ export class LoopRunStateMachine {
145
+ private readonly control: LoopControlService;
146
+
147
+ constructor(private readonly database: Database) {
148
+ this.control = new LoopControlService(database);
149
+ }
150
+
151
+ async claimNext(workerId: string, leaseMs: number): Promise<ClaimedLoopRun | null> {
152
+ const normalizedWorkerId = workerId.trim();
153
+ if (!normalizedWorkerId) throw new Error('Worker identity is required.');
154
+ if (!Number.isSafeInteger(leaseMs) || leaseMs < 5_000) throw new Error('Worker lease must be at least 5000ms.');
155
+
156
+ const committed = await this.database.sequelize.transaction(async (transaction) => {
157
+ const control = await this.control.getForUpdate(transaction);
158
+ if (control.state !== 'running')
159
+ return { claimed: null, blockedEvents: [] as Array<{ runId: number; event: unknown }> };
160
+
161
+ const runs = this.database.getRepository('agentLoopRuns');
162
+ const now = new Date();
163
+ const reclaimedEvents: Array<{ runId: number; event: unknown }> = [];
164
+
165
+ // A worker that crashed mid-run leaves an active status behind with a lease that
166
+ // eventually expires. Those runs must be returned to the queue before concurrency is
167
+ // measured, otherwise each crash permanently consumes a slot.
168
+ const staleActive = await runs.find({
169
+ filter: { runtimeVersion: 'control-plane-v2', status: { $in: activeExecutionStatuses } },
170
+ transaction,
171
+ lock: transaction.LOCK.UPDATE,
172
+ });
173
+ for (const candidate of staleActive) {
174
+ if (!leaseIsReclaimable(candidate, now)) continue;
175
+ const runId = asPositiveInteger(read(candidate, 'id'), 'Run identifier');
176
+ const from = asStatus(read(candidate, 'status'));
177
+ assertRunTransition({
178
+ from,
179
+ to: 'queued',
180
+ runtimeVersion: String(read(candidate, 'runtimeVersion') || ''),
181
+ });
182
+ await runs.update({
183
+ filterByTk: runId,
184
+ values: { status: 'queued', lockedBy: null, lockedUntil: null },
185
+ transaction,
186
+ });
187
+ const event = await this.createEvent(
188
+ {
189
+ runId,
190
+ type: 'run_lease_expired',
191
+ title: 'Worker lease expired',
192
+ content: `Run returned to the queue after its ${from} worker lease expired.`,
193
+ status: 'queued',
194
+ payload: { from, previousOwner: read(candidate, 'lockedBy') ?? null },
195
+ actorType: 'system',
196
+ actorIdentity: normalizedWorkerId,
197
+ correlationKey: `lease-expired:${runId}:${String(read(candidate, 'lockedBy') || '')}`,
198
+ createdAt: now,
199
+ },
200
+ transaction,
201
+ );
202
+ reclaimedEvents.push({ runId, event });
203
+ }
204
+
205
+ const activeGlobal = await this.countActive(runs, {}, transaction, now);
206
+ if (activeGlobal >= control.globalMaxConcurrency) {
207
+ return { claimed: null, blockedEvents: reclaimedEvents };
208
+ }
209
+
210
+ const candidates = await runs.find({
211
+ filter: { runtimeVersion: 'control-plane-v2', status: { $in: claimableStatuses } },
212
+ sort: ['createdAt', 'id'],
213
+ limit: 100,
214
+ transaction,
215
+ lock: transaction.LOCK.UPDATE,
216
+ });
217
+ const blockedEvents: Array<{ runId: number; event: unknown }> = [...reclaimedEvents];
218
+
219
+ for (const candidate of candidates) {
220
+ const runId = asPositiveInteger(read(candidate, 'id'), 'Run identifier');
221
+ const patternId = Number(read(candidate, 'patternId'));
222
+ const pattern =
223
+ Number.isSafeInteger(patternId) && patternId > 0
224
+ ? await this.database.getRepository('agentLoopPatterns').findOne({
225
+ filterByTk: patternId,
226
+ transaction,
227
+ lock: transaction.LOCK.UPDATE,
228
+ })
229
+ : null;
230
+ let blockedReason = '';
231
+ if (!Number.isSafeInteger(patternId) || patternId <= 0) blockedReason = 'Run has no valid pattern snapshot.';
232
+ else if (!pattern) blockedReason = 'Loop pattern was deleted.';
233
+ else if (read(pattern, 'enabled') !== true) blockedReason = 'Loop pattern is disabled.';
234
+
235
+ const parsedPolicy = loopPatternPolicySchema.safeParse(read(candidate, 'policySnapshot'));
236
+ if (!blockedReason && !parsedPolicy.success) blockedReason = 'Run policy snapshot is invalid.';
237
+ if (blockedReason) {
238
+ assertRunTransition({
239
+ from: asStatus(read(candidate, 'status')),
240
+ to: 'blocked',
241
+ runtimeVersion: String(read(candidate, 'runtimeVersion') || ''),
242
+ });
243
+ await runs.update({
244
+ filterByTk: runId,
245
+ values: { status: 'blocked', blockedReason, lockedBy: null, lockedUntil: null },
246
+ transaction,
247
+ });
248
+ const event = await this.createEvent(
249
+ {
250
+ runId,
251
+ type: 'run_blocked',
252
+ title: 'Run blocked',
253
+ content: blockedReason,
254
+ status: 'blocked',
255
+ payload: { reason: blockedReason },
256
+ actorType: 'worker',
257
+ actorIdentity: normalizedWorkerId,
258
+ correlationKey: `claim-blocked:${runId}`,
259
+ createdAt: now,
260
+ },
261
+ transaction,
262
+ );
263
+ blockedEvents.push({ runId, event });
264
+ continue;
265
+ }
266
+
267
+ const activePattern = await this.countActive(runs, { patternId }, transaction, now);
268
+ if (activePattern >= parsedPolicy.data.maxConcurrency) continue;
269
+
270
+ const leaseToken = `${normalizedWorkerId}:${randomUUID()}`;
271
+ const leaseUntil = new Date(now.getTime() + leaseMs);
272
+ assertRunTransition({
273
+ from: asStatus(read(candidate, 'status')),
274
+ to: 'preparing',
275
+ runtimeVersion: String(read(candidate, 'runtimeVersion') || ''),
276
+ });
277
+ await runs.update({
278
+ filterByTk: runId,
279
+ values: { status: 'preparing', blockedReason: null, lockedBy: leaseToken, lockedUntil: leaseUntil },
280
+ transaction,
281
+ });
282
+ const event = await this.createEvent(
283
+ {
284
+ runId,
285
+ type: 'run_claimed',
286
+ title: 'Run claimed',
287
+ content: `Run claimed by worker ${normalizedWorkerId}.`,
288
+ status: 'preparing',
289
+ payload: { workerId: normalizedWorkerId, leaseUntil: leaseUntil.toISOString() },
290
+ actorType: 'worker',
291
+ actorIdentity: normalizedWorkerId,
292
+ correlationKey: `claim:${leaseToken}`,
293
+ createdAt: now,
294
+ },
295
+ transaction,
296
+ );
297
+ return {
298
+ claimed: {
299
+ run: { ...asPlain(candidate), status: 'preparing', lockedBy: leaseToken, lockedUntil: leaseUntil },
300
+ leaseToken,
301
+ leaseUntil,
302
+ runId,
303
+ event,
304
+ },
305
+ blockedEvents,
306
+ };
307
+ }
308
+ return { claimed: null, blockedEvents };
309
+ });
310
+
311
+ for (const blocked of committed.blockedEvents) getRunEventBus().emit(blocked.runId, blocked.event);
312
+ if (!committed.claimed) return null;
313
+ getRunEventBus().emit(committed.claimed.runId, committed.claimed.event);
314
+ return {
315
+ run: committed.claimed.run,
316
+ leaseToken: committed.claimed.leaseToken,
317
+ leaseUntil: committed.claimed.leaseUntil,
318
+ };
319
+ }
320
+
321
+ // Approval windows fail closed: an unanswered request blocks the run instead of silently
322
+ // turning into an approval. Runs only move while they are still `waiting_approval`, so a
323
+ // decision recorded through another path always wins the race against this sweep.
324
+ async expireOverdueApprovals(now: Date): Promise<number[]> {
325
+ const blockedRunIds: number[] = [];
326
+ const committedEvents: Array<{ runId: number; event: unknown }> = [];
327
+
328
+ await this.database.sequelize.transaction(async (transaction) => {
329
+ const approvals = this.database.getRepository('agentLoopActionApprovals');
330
+ const runs = this.database.getRepository('agentLoopRuns');
331
+ const pending = await approvals.find({ filter: { status: 'pending' }, transaction });
332
+
333
+ const overdueByRun = new Map<number, Array<Model | Record<string, unknown>>>();
334
+ for (const approval of pending) {
335
+ const expiresAt = asDate(read(approval, 'expiresAt'));
336
+ if (!expiresAt || expiresAt.getTime() > now.getTime()) continue;
337
+ const runId = Number(read(approval, 'runId'));
338
+ if (!Number.isSafeInteger(runId) || runId <= 0) continue;
339
+ const bucket = overdueByRun.get(runId) || [];
340
+ bucket.push(approval);
341
+ overdueByRun.set(runId, bucket);
342
+ }
343
+
344
+ for (const [runId, overdue] of overdueByRun) {
345
+ const run = await runs.findOne({ filterByTk: runId, transaction, lock: transaction.LOCK.UPDATE });
346
+ if (!run) continue;
347
+ if (String(read(run, 'runtimeVersion')) !== 'control-plane-v2') continue;
348
+ if (asStatus(read(run, 'status')) !== 'waiting_approval') continue;
349
+
350
+ assertRunTransition({
351
+ from: 'waiting_approval',
352
+ to: 'blocked',
353
+ runtimeVersion: String(read(run, 'runtimeVersion') || ''),
354
+ });
355
+
356
+ for (const approval of overdue) {
357
+ await approvals.update({
358
+ filterByTk: Number(read(approval, 'id')),
359
+ values: { status: 'expired', decidedAt: now },
360
+ transaction,
361
+ });
362
+ }
363
+
364
+ const reason = `${overdue.length} action approval(s) expired before a decision was made.`;
365
+ await runs.update({
366
+ filterByTk: runId,
367
+ values: {
368
+ status: 'blocked',
369
+ blockedReason: reason,
370
+ approvalStatus: 'expired',
371
+ lockedBy: null,
372
+ lockedUntil: null,
373
+ },
374
+ transaction,
375
+ });
376
+ const event = await this.createEvent(
377
+ {
378
+ runId,
379
+ type: 'run_approval_expired',
380
+ title: 'Approval window expired',
381
+ content: reason,
382
+ status: 'blocked',
383
+ payload: { reason, expiredApprovals: overdue.length },
384
+ actorType: 'system',
385
+ actorIdentity: 'approval-reaper',
386
+ correlationKey: `approval-expired:${runId}`,
387
+ createdAt: now,
388
+ },
389
+ transaction,
390
+ );
391
+ blockedRunIds.push(runId);
392
+ committedEvents.push({ runId, event });
393
+ }
394
+ });
395
+
396
+ for (const blocked of committedEvents) getRunEventBus().emit(blocked.runId, blocked.event);
397
+ return blockedRunIds;
398
+ }
399
+
400
+ async renewLease(runId: number, leaseToken: string, leaseMs: number) {
401
+ if (!leaseToken.trim()) throw new Error('Worker lease token is required.');
402
+ if (!Number.isSafeInteger(leaseMs) || leaseMs < 5_000) throw new Error('Worker lease must be at least 5000ms.');
403
+ return this.database.sequelize.transaction(async (transaction) => {
404
+ const runs = this.database.getRepository('agentLoopRuns');
405
+ const run = await runs.findOne({ filterByTk: runId, transaction, lock: transaction.LOCK.UPDATE });
406
+ if (!run || read(run, 'lockedBy') !== leaseToken)
407
+ throw new Error('Worker lease is no longer owned by this worker.');
408
+ if (String(read(run, 'runtimeVersion')) !== 'control-plane-v2')
409
+ throw new Error('Historical plan-era runs are read-only.');
410
+ if (!activeExecutionStatuses.includes(asStatus(read(run, 'status')))) {
411
+ throw new Error('Only actively executing runs can renew a worker lease.');
412
+ }
413
+ const leaseUntil = new Date(Date.now() + leaseMs);
414
+ await runs.update({ filterByTk: runId, values: { lockedUntil: leaseUntil }, transaction });
415
+ return leaseUntil;
416
+ });
417
+ }
418
+
419
+ async releaseLease(runId: number, leaseToken: string) {
420
+ if (!leaseToken.trim()) throw new Error('Worker lease token is required.');
421
+ return this.database.sequelize.transaction(async (transaction) => {
422
+ const runs = this.database.getRepository('agentLoopRuns');
423
+ const run = await runs.findOne({ filterByTk: runId, transaction, lock: transaction.LOCK.UPDATE });
424
+ if (!run) return false;
425
+ if (read(run, 'lockedBy') !== leaseToken) throw new Error('Worker lease is no longer owned by this worker.');
426
+ await runs.update({ filterByTk: runId, values: { lockedBy: null, lockedUntil: null }, transaction });
427
+ return true;
428
+ });
429
+ }
430
+
431
+ async transition(input: {
432
+ runId: number;
433
+ to: LoopRunStatus;
434
+ actorType: string;
435
+ actorIdentity: string;
436
+ eventType?: string;
437
+ title?: string;
438
+ content?: string;
439
+ correlationKey?: string;
440
+ leaseToken?: string;
441
+ values?: Record<string, unknown>;
442
+ humanAccepted?: boolean;
443
+ }) {
444
+ const committed = await this.database.sequelize.transaction(async (transaction) => {
445
+ const runs = this.database.getRepository('agentLoopRuns');
446
+ const run = await runs.findOne({
447
+ filterByTk: input.runId,
448
+ transaction,
449
+ lock: transaction.LOCK.UPDATE,
450
+ });
451
+ if (!run) throw new Error(`Agent loop run ${input.runId} was not found.`);
452
+
453
+ if (input.actorType === 'worker') {
454
+ if (!input.leaseToken?.trim() || read(run, 'lockedBy') !== input.leaseToken) {
455
+ throw new Error('Worker lease is no longer owned by this worker.');
456
+ }
457
+ if (leaseIsExpired(run, new Date())) throw new Error('Worker lease has expired.');
458
+ }
459
+
460
+ const from = asStatus(read(run, 'status'));
461
+
462
+ // The transition table permits `paused -> running/verifying` because that is how a human
463
+ // resumes a run. A worker must not use those edges: a run paused or canceled mid-invocation
464
+ // would otherwise be carried forward by the worker that was already executing it, and the
465
+ // pause would only take effect on the next claim.
466
+ if (input.actorType === 'worker' && !activeExecutionStatuses.includes(from)) {
467
+ throw new Error(`A worker cannot transition a run that is ${from}.`);
468
+ }
469
+
470
+ const verifierEvidence = input.values?.verifierEvidence ?? read(run, 'verifierEvidence');
471
+ assertRunTransition({
472
+ from,
473
+ to: input.to,
474
+ runtimeVersion: String(read(run, 'runtimeVersion') || ''),
475
+ verifierEvidence,
476
+ humanAccepted: input.humanAccepted,
477
+ });
478
+
479
+ const now = new Date();
480
+ await runs.update({
481
+ filterByTk: input.runId,
482
+ values: {
483
+ ...(input.values || {}),
484
+ status: input.to,
485
+ ...(input.to === 'running' && !read(run, 'startedAt') ? { startedAt: now } : {}),
486
+ ...(['succeeded', 'failed', 'canceled'].includes(input.to) ? { endedAt: now } : {}),
487
+ },
488
+ transaction,
489
+ });
490
+ const event = await this.createEvent(
491
+ {
492
+ runId: input.runId,
493
+ type: input.eventType || 'run_status_changed',
494
+ title: input.title || `Run ${input.to}`,
495
+ content: input.content || `${from} -> ${input.to}`,
496
+ status: input.to,
497
+ payload: { from, to: input.to },
498
+ actorType: input.actorType,
499
+ actorIdentity: input.actorIdentity,
500
+ correlationKey: input.correlationKey,
501
+ createdAt: now,
502
+ },
503
+ transaction,
504
+ );
505
+ return { from, to: input.to, event };
506
+ });
507
+
508
+ getRunEventBus().emit(input.runId, committed.event);
509
+ return committed;
510
+ }
511
+
512
+ private async countActive(
513
+ runs: ReturnType<Database['getRepository']>,
514
+ filter: Record<string, unknown>,
515
+ transaction: Transaction,
516
+ now: Date,
517
+ ) {
518
+ const rows = await runs.find({
519
+ filter: { ...filter, runtimeVersion: 'control-plane-v2', status: { $in: activeExecutionStatuses } },
520
+ transaction,
521
+ });
522
+ return rows.filter((row) => !leaseIsReclaimable(row, now)).length;
523
+ }
524
+
525
+ private async createEvent(values: Record<string, unknown>, transaction: Transaction) {
526
+ const record = await this.database.getRepository('agentLoopEvents').create({ values, transaction });
527
+ return typeof record.toJSON === 'function' ? record.toJSON() : record;
528
+ }
529
+ }